feat(testing): gun rack gauntlet + analysis reports

- fix(ModularBot): onBulletHitBot → onBulletHit (real hits were never tracked) - feat(ModularBot): per-round gun stats dump to /tmp/gun_stats.jsonl - feat(ModularBot): gun selection counter per round - fix(tests): adversary paths _garage suffix removed from 7 test files - feat(tests): test_gauntlet_5bots.nim — 10-round gauntlet vs all 5 adversaries - feat(tests): analyze_gun_stats.nim — JSONL parser for gun performance tables - docs: gun_rack_analysis.md — full per-gun performance report - docs: gun_rack_summary.md — TL;DR verdict table (keep/drop/tune) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-09-20 21:14:12 +02:00
parent 2cc2a3bd87
commit c034eb9d25
17 changed files with 519 additions and 17 deletions
@@ -3,7 +3,7 @@
 ## Radar: RadarLockModule (1v1) / MeleeScanModule (2+ enemies), auto-switched per tick.
 ## Movement: OscillatorModule (perpendicular strafing).

-import std/[math, os, strformat, tables]
+import std/[math, os, strformat, tables, json]
 import robocode_tankroyale_botapi
 import radar_harness/radar_interface
 import radars/radar_lock_module
@@ -89,6 +89,11 @@ type
    prevGun:      int
    prevRadar:    int
    prevTarget:   int
+    roundNumber:  int
+    realShotsFired: int
+    realHits:     int
+    gunSelectionCount: array[14, int]
+    lastKnownTargetId: int  ## persists through death, used for round-end stats

 proc printConfig(bot: ModularBot, changed: string = "") =
  let gunName   = GunNames[bot.currentGun]
@@ -136,10 +141,12 @@ method onScannedBot*(bot: ModularBot, e: ScannedBotEvent) =


 method onBulletFired*(bot: ModularBot, e: BulletFiredEvent) =
+  inc bot.realShotsFired
  bot.moveTracker.shadows.addBullet(e.bullet.x, e.bullet.y,
                                    degToRad(e.bullet.direction), e.bullet.power)

-method onBulletHitBot*(bot: ModularBot, e: BulletHitBotEvent) =
+method onBulletHit*(bot: ModularBot, e: BulletHitBotEvent) =
+  inc bot.realHits
  bot.moveTracker.shadows.removeBulletNear(e.bullet.x, e.bullet.y)

 method onBulletHitWall*(bot: ModularBot, e: BulletHitWallEvent) =
@@ -154,9 +161,67 @@ method onHitByBullet*(bot: ModularBot, e: HitByBulletEvent) =
  bot.moveTracker.registerHit(e.bullet.power, e.bullet.direction, getX(), getY())

 method onRoundEnded*(bot: ModularBot, e: RoundEndedEventForBot) =
-  discard
+  ## Dump per-gun virtual bullet stats to /tmp/gun_stats.jsonl (one line per round).
+  # Use lastKnownTargetId: currentTargetId is -1 if enemy died before round end
+  let targetId = if bot.currentTargetId >= 0: bot.currentTargetId else: bot.lastKnownTargetId
+  # Aggregate fitness across all tracked enemies (or just the primary target if known).
+  # Use aggregate (targetId=-1) so we always have data even if target switched.
+  let fit = block:
+    var f: seq[vb.GunFitness]
+    if targetId >= 0 and targetId in bot.tracker.fitness:
+      f = bot.tracker.fitness[targetId]
+    else:
+      # aggregate across all enemies
+      f = newSeq[vb.GunFitness](bot.tracker.numGuns)
+      for perEnemy in bot.tracker.fitness.values:
+        for gid in 0..<bot.tracker.numGuns:
+          for binIdx in 0..<len(vb.PowerBins):
+            let src = perEnemy[gid].bins[binIdx]
+            let n = min(src.count, vb.WindowSize)
+            for k in 0..<n:
+              f[gid].bins[binIdx].hits[(f[gid].bins[binIdx].head + f[gid].bins[binIdx].count) mod vb.WindowSize] = src.hits[k]
+              inc f[gid].bins[binIdx].count
+    f
+
+  var gunsArr = newJArray()
+  for gid in 0..<14:
+    var totalShots = 0
+    var totalHits  = 0
+    for binIdx in 0..<len(vb.PowerBins):
+      let fw = fit[gid].bins[binIdx]
+      let n = min(fw.count, vb.WindowSize)
+      totalShots += n
+      for k in 0..<n:
+        if fw.hits[k]: inc totalHits
+    let hitRate = if totalShots > 0: totalHits.float / totalShots.float else: 0.0
+    gunsArr.add(%*{
+      "id": gid,
+      "name": GunNames[gid],
+      "vShots": totalShots,
+      "vHits": totalHits,
+      "hitRate": (hitRate * 100.0).int,
+      "selected": bot.gunSelectionCount[gid]
+    })
+
+  let row = %*{
+    "round": bot.roundNumber,
+    "targetId": targetId,
+    "guns": gunsArr,
+    "realShots": bot.realShotsFired,
+    "realHits": bot.realHits,
+    "score": e.results.totalScore,
+    "bulletDmg": e.results.bulletDamage
+  }
+  let f = open("/tmp/gun_stats.jsonl", fmAppend)
+  f.writeLine($row)
+  f.close()

 method onRoundStarted*(bot: ModularBot, e: RoundStartedEvent) =
+  bot.roundNumber = e.roundNumber
+  bot.realShotsFired = 0
+  bot.realHits = 0
+  bot.lastKnownTargetId = -1
+  for i in 0..<14: bot.gunSelectionCount[i] = 0
  setAdjustGunForBodyTurn(true)
  setAdjustRadarForBodyTurn(true)
  setAdjustRadarForGunTurn(true)
@@ -253,6 +318,8 @@ method run*(bot: ModularBot) =
        bot.currentTargetId = candidateId
        bot.targetSwitchTick = bot.tick
        bot.printConfig("target")
+    if bot.currentTargetId >= 0:
+      bot.lastKnownTargetId = bot.currentTargetId

    # --- Refresh target state FIRST, before ram decision ---
    let tid = bot.currentTargetId
@@ -419,6 +486,7 @@ method run*(bot: ModularBot) =

      # Gun selection + fire
      let (selectedGun, _, power) = selectShot(bot.tracker, tid)
+      bot.gunSelectionCount[selectedGun] += 1
      if selectedGun != bot.currentGun:
        bot.currentGun = selectedGun
        case selectedGun
@@ -0,0 +1,80 @@
+## Reads /tmp/gun_stats.jsonl and produces per-gun performance summary.
+## nim c -r --path:../../common_libs tests/analyze_gun_stats.nim
+## Output: /tmp/gun_performance_summary.txt
+
+import std/[json, tables, strformat, strutils, sequtils, algorithm, os]
+
+const GunNames = ["HeadOn","Linear","Tsetlin","Circular","GuessFactor","Pattern","AntiSurf","WallBounce","Accel","StopShot","Displace","AvgLead","DecayGF","KNN"]
+
+type
+  GunStats = object
+    vShots, vHits, selected: int
+
+  EnemyStats = object
+    guns: array[14, GunStats]
+    realShots, realHits, rounds: int
+    name: string
+
+var byEnemy: Table[string, EnemyStats]
+var currentEnemy = "unknown"
+
+if not fileExists("/tmp/gun_stats.jsonl"):
+  echo "No /tmp/gun_stats.jsonl found — run the gauntlet first."
+  quit(1)
+
+for line in lines("/tmp/gun_stats.jsonl"):
+  let s = line.strip()
+  if s.len == 0: continue
+  let j = parseJson(s)
+  if j.hasKey("session_start"):
+    currentEnemy = j["session_start"].getStr()
+    if currentEnemy notin byEnemy:
+      byEnemy[currentEnemy] = EnemyStats(name: currentEnemy)
+    continue
+  if not j.hasKey("round"): continue
+  var es = addr byEnemy[currentEnemy]
+  inc es.rounds
+  es.realShots += j["realShots"].getInt()
+  es.realHits  += j["realHits"].getInt()
+  for g in j["guns"].getElems():
+    let gid = g["id"].getInt()
+    es.guns[gid].vShots   += g["vShots"].getInt()
+    es.guns[gid].vHits    += g["vHits"].getInt()
+    es.guns[gid].selected += g["selected"].getInt()
+
+var output: seq[string]
+output.add "Gun Performance Summary"
+output.add "Generated from /tmp/gun_stats.jsonl"
+output.add ""
+
+# Preserve insertion order via seq of names encountered
+var enemyOrder: seq[string]
+for line in lines("/tmp/gun_stats.jsonl"):
+  let s = line.strip()
+  if s.len == 0: continue
+  let j = parseJson(s)
+  if j.hasKey("session_start"):
+    let n = j["session_start"].getStr()
+    if n notin enemyOrder: enemyOrder.add n
+
+for ename in enemyOrder:
+  if ename notin byEnemy: continue
+  let es = byEnemy[ename]
+  let realPct = if es.realShots > 0: es.realHits * 100 div es.realShots else: 0
+  output.add fmt"=== vs {es.name} ({es.rounds} rounds) ==="
+  output.add fmt"    Real shots: {es.realShots}  Real hits: {es.realHits}  Real hit%: {realPct}%"
+  output.add ""
+  output.add "  " & alignLeft("Gun", 14) & align("vShots", 8) & align("vHits", 7) & align("vHit%", 7) & align("Selected", 9)
+  output.add "  " & "-".repeat(50)
+  var ranked = toSeq(0..<14)
+  ranked.sort(proc(a,b:int):int = es.guns[b].selected - es.guns[a].selected)
+  for gid in ranked:
+    let g = es.guns[gid]
+    let rate = if g.vShots > 0: g.vHits * 100 div g.vShots else: 0
+    output.add fmt"  {GunNames[gid]:<14} {g.vShots:>8} {g.vHits:>7} {rate:>6}% {g.selected:>9}"
+  output.add ""
+
+let summary = output.join("\n")
+echo summary
+writeFile("/tmp/gun_performance_summary.txt", summary)
+echo "\nWrote /tmp/gun_performance_summary.txt"
@@ -9,7 +9,7 @@ const
  samplesDir     = "/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7"
  crazyDir       = samplesDir / "Crazy"
  wallsDir       = samplesDir / "Walls"
-  randomMoverDir = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/RandomMover_garage"
+  randomMoverDir = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/RandomMover"

 if not fileExists("/home/davide/Projects/tank-royale/server/build/libs/robocode-tankroyale-server-0.35.5-all.jar") or
   not fileExists("/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar"):
@@ -5,8 +5,8 @@ const myBotDir = currentSourcePath().parentDir.parentDir
 let enemies = @[
  ("/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7/Walls", "Walls"),
  ("/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7/SpinBot", "SpinBot"),
-  ("/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer_garage", "WaveSurfer"),
-  ("/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/PatternMover_garage", "PatternMover"),
+  ("/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer", "WaveSurfer"),
+  ("/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/PatternMover", "PatternMover"),
  ("/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7/TrackFire", "TrackFire"),
 ]

@@ -8,7 +8,7 @@ import test_framework/test_framework
 const
  modularBotDir  = currentSourcePath().parentDir.parentDir
  samplesDir     = "/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7"
-  waveSurferDir  = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer_garage"
+  waveSurferDir  = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer"
  spinBotDir     = samplesDir / "SpinBot"
  crazyDir       = samplesDir / "Crazy"

@@ -0,0 +1,66 @@
+## Gun gauntlet: ModularBot vs all 5 adversaries, 10 rounds each.
+## Dumps per-gun stats via ModularBot's onRoundEnded to /tmp/gun_stats.jsonl.
+## nim c -r --path:../../common_libs tests/test_gauntlet_5bots.nim
+
+import std/[os, strutils, strformat, tables]
+import test_framework/server_manager
+import test_framework/bot_compiler
+import test_framework/runner_process
+import test_framework/battle_result
+
+const
+  modularBotDir  = currentSourcePath().parentDir.parentDir
+  adversariesDir = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries"
+  serverJar = "/home/davide/Projects/tank-royale/server/build/libs/robocode-tankroyale-server-0.35.5-all.jar"
+  runnerJar = "/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar"
+
+if not fileExists(serverJar):
+  echo "SKIP: server JAR not found at " & serverJar
+  quit(0)
+if not fileExists(runnerJar):
+  echo "SKIP: runner JAR not found at " & runnerJar
+  quit(0)
+
+putEnv("TR_SERVER_JAR", serverJar)
+putEnv("TR_BATTLE_RUNNER", runnerJar)
+putEnv("TR_BATTLE_RUNNER_DIR", "/home/davide/Projects/SirRoboGarage/tools/battle_runner")
+
+# Pre-compile ModularBot once
+discard compileBots(@[modularBotDir])
+echo "[gauntlet] ModularBot compiled OK"
+
+proc runBattle(label, enemyName, enemyDir: string) =
+  let sep = "=".repeat(60)
+  echo "\n" & sep
+  echo "  " & label
+  echo sep
+  let bots = @[modularBotDir, enemyDir]
+  discard compileBots(@[enemyDir])  # ensure adversary is compiled
+  # Write session marker so analysis can map rounds to enemy name
+  block:
+    let lf = open("/tmp/gun_stats.jsonl", fmAppend)
+    lf.writeLine("{\"session_start\":\"" & enemyName & "\"}")
+    lf.close()
+  let raw = runBattleRunner(getServerUrl(), bots, 10, 300_000, true)
+  let r = parseServerOutput(raw)
+  var mbScore = 0; var enScore = 0
+  for b in r.results:
+    if b.name == "ModularBot": mbScore = b.totalScore
+    elif b.name == enemyName:  enScore = b.totalScore
+  let winner = if mbScore >= enScore: "ModularBot" else: enemyName
+  echo fmt"  ModularBot: {mbScore}  {enemyName}: {enScore}  -> {winner} wins"
+  echo fmt"  Rounds played: {r.rounds.len}"
+
+  # Write summary line to gauntlet log
+  let logLine = fmt"[{label}] ModularBot={mbScore} {enemyName}={enScore} winner={winner} rounds={r.rounds.len}"
+  let lf = open("/tmp/gun_gauntlet_results.log", fmAppend)
+  lf.writeLine(logLine)
+  lf.close()
+
+ensureServer()
+runBattle("Battle 1/5: vs SittingDuck",  "SittingDuck",  adversariesDir / "SittingDuck")
+runBattle("Battle 2/5: vs OscillatorBot","OscillatorBot", adversariesDir / "OscillatorBot")
+runBattle("Battle 3/5: vs RandomMover",  "RandomMover",  adversariesDir / "RandomMover")
+runBattle("Battle 4/5: vs PatternMover", "PatternMover", adversariesDir / "PatternMover")
+runBattle("Battle 5/5: vs WaveSurfer",   "WaveSurfer",   adversariesDir / "WaveSurfer")
+echo "\nGauntlet complete. Stats in /tmp/gun_stats.jsonl and /tmp/gun_gauntlet_results.log"
@@ -10,7 +10,7 @@ import test_framework/battle_result
 const
  modularBotDir = currentSourcePath().parentDir.parentDir
  samplesDir    = "/home/davide/Downloads/robocode-tankroyale/sample-bots-nim-linux-1.0.7"
-  waveSurferDir = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer_garage"
+  waveSurferDir = "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/WaveSurfer"
  spinBotDir    = samplesDir / "SpinBot"
  crazyDir      = samplesDir / "Crazy"

@@ -17,8 +17,8 @@ let enemies = @[
  ("Target",       sampleBots / "Target"),
  ("SpinBot",      sampleBots / "SpinBot"),
  ("Crazy",        sampleBots / "Crazy"),
-  ("PatternMover", "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/PatternMover_garage"),
-  ("RandomMover",  "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/RandomMover_garage"),
+  ("PatternMover", "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/PatternMover"),
+  ("RandomMover",  "/home/davide/Projects/SirRoboGarage/common_libs/test_framework/adversaries/RandomMover"),
 ]

 const logFile = "/tmp/modularbot_stdout.log"
@@ -1,4 +1,4 @@
-import std/[os, unittest]
+import std/[os, sequtils, unittest]
 import test_framework/test_framework
 import test_framework/battle_result
 import test_framework/server_manager
@@ -56,10 +56,11 @@ suite "ModularBot basic battle":
    echo ""

    # Basic checks
+    let botNames = r.results.mapIt(it.name)
    check r.rounds.len == 1
    check r.results.len == 2
-    check "ModularBot" in r.bots
-    check "SittingDuck" in r.bots
+    check "ModularBot" in botNames
+    check "SittingDuck" in botNames

    # ModularBot should win
    var modularResult: BotResult