feat(gun_harness): offline gun range, proven equivalent to live play

Gun evaluation previously required a full end-to-end battle (Java server +
battle runner + websocket IPC to 2 bot processes, 50 rounds, ~3.4 min) and
yielded only ~300-900 REAL shots across 13 guns -- far too few to rank
guns, which is why tuning needed many repetitions.

VirtualTracker is already a pure function of (WorldState stream, gun list);
the only reason it needed Java was where WorldState came from. So the range
replays a seq[WorldState] through the SAME tracker: offline and online
scores are the same metric by construction, not an approximation.

ACCEPTANCE TEST (the point of the whole thing): record one live round, replay
it offline, compare per-gun virtual hit rates. 12/12 deterministic guns match
EXACTLY, reproduced twice. Tsetlin is compared separately because tmLearnOne
calls rand(). Getting to 12/12 exposed two real ordering quirks in the live
loop: run() calls go() before the aim/fire block, so tickBullets resolves
against the NEXT tick's scan while the prediction used the previous one; and
if the target dies during that go() the final tick's spawn+resolution is
skipped entirely. The recorder emits an end marker for the second case.
The 5th (selected-gun) predict call was verified to be a no-op.

Measured cost: 8 fixtures (1770 ticks, ~92k virtual bullets, 13 guns) replay
in 2.9 s, ~32k virtual bullets/s -- roughly 70x faster and 100x more samples
than a live gauntlet.

Also adds a per-tick WorldState recorder behind const RecordWorldState
(default off, mirrors the ShotLog idiom) which records the state the bot
ACTUALLY builds, staleness included, rather than true positions -- recording
the latter would hand the guns perfect information and produce flattering
scores.

9 new guard checks (33 total, all passing), including fixture round-trip,
replay determinism, stationary->HeadOn 100%, constant-velocity->Linear>HeadOn,
and the energy-threshold turner crossing at t=41.
This commit is contained in:
2026-09-20 23:44:42 +02:00
parent 3c90a5941d
commit 974528d5cf
16 changed files with 2698 additions and 2 deletions
@@ -0,0 +1,115 @@
## Task 3: acceptance test — prove the offline range reproduces the live
## virtual-bullet metric, or the range is worthless.
##
## Steps:
## 1. run ONE live ModularBot vs OscillatorBot round with the ModularBot
## recorder ON (compiled in via `const RecordWorldState = true`),
## 2. read the online per-gun virtual fitness from /tmp/gun_stats.jsonl,
## 3. replay the recorded WorldState fixture offline through the same guns,
## 4. compare.
##
## Tsetlin is stochastic (tmLearnOne calls rand(); its constructor calls
## randomize()), so byte-identical replay is impossible for it. The 12
## deterministic guns must match EXACTLY; Tsetlin is reported separately.
##
## Run with:
## nim c -r common_libs/tests/acceptance_offline_vs_online.nim
##
## Requires TR_SERVER_JAR / TR_BATTLE_RUNNER (or the default dev paths below).
import std/[json, os, strformat, strutils, math]
import test_framework/test_framework
import gun_harness/offline_range
import range_guns
const
repoRoot = currentSourcePath().parentDir.parentDir.parentDir
modularBotDir = repoRoot / "ModularBot_garage"
adversaryDir = repoRoot / "common_libs" / "test_framework" / "adversaries" / "OscillatorBot"
statsPath = "/tmp/gun_stats.jsonl"
recordPath = "/tmp/worldstate_record.jsonl"
serverJar = "/home/davide/Projects/tank-royale/server/build/libs/robocode-tankroyale-server-0.35.5-all.jar"
runnerJar = "/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar"
const TsetlinId = 2
proc lastOnlineRound(path: string): JsonNode =
result = nil
for line in lines(path):
let s = line.strip()
if s.len == 0: continue
let node = parseJson(s)
if node.hasKey("guns"): result = node
proc main() =
if not fileExists(serverJar) or not fileExists(runnerJar):
echo "Skipping: TR JARs not found (server=", serverJar, ", runner=", runnerJar, ")"
quit(0)
if not fileExists(modularBotDir / "src" / "ModularBot.nim"):
echo "Skipping: ModularBot source not found"
quit(0)
for p in [statsPath, recordPath]:
if fileExists(p): removeFile(p)
echo "=== live battle: ModularBot vs OscillatorBot, 1 round, max speed ==="
let battle = runBattle(@[modularBotDir, adversaryDir], rounds = 1,
timeout = 240000, maxSpeed = true)
for res in battle.results:
echo fmt" {res.name:<14} rank={res.rank} score={res.totalScore}"
if not fileExists(recordPath):
echo "FAIL: recorder produced no fixture (is RecordWorldState true?)"
quit(1)
if not fileExists(statsPath):
echo "FAIL: no /tmp/gun_stats.jsonl"
quit(1)
let online = lastOnlineRound(statsPath)
let fx = loadFixture(recordPath)
let reports = replayFixture(fx, buildAllGunDrivers(), liveActual = true)
# Map online stats by gun id.
var onShots: array[13, int]
var onHits: array[13, int]
var onNames: array[13, string]
for g in online["guns"]:
let id = g["id"].getInt()
if id >= 0 and id < 13:
onShots[id] = g["vShots"].getInt()
onHits[id] = g["vHits"].getInt()
onNames[id] = g["name"].getStr()
echo ""
echo fmt"fixture: {recordPath} ticks={fx.states.len} enemyId={fx.enemyId} enemyDied={fx.enemyDied}"
echo "online stats: /tmp/gun_stats.jsonl round ", online["round"].getInt()
echo ""
echo "gun online(vHits/vShots) offline(hits/shots) verdict"
echo "-----------------------------------------------------------------------"
var matches = 0
var deterministic = 0
for id in 0..<13:
let r = reports[id]
let match = r.hits == onHits[id] and r.shots == onShots[id]
var verdict: string
if id == TsetlinId:
verdict = if match: "MATCH (stochastic)" else: "differs (stochastic, expected)"
else:
inc deterministic
if match:
inc matches
verdict = "OK"
else:
verdict = "MISMATCH"
echo fmt"{r.name:<12} {onHits[id]:>5}/{onShots[id]:<5} {r.hits:>5}/{r.shots:<5} {verdict}"
echo ""
echo fmt"deterministic guns matching exactly: {matches}/{deterministic}"
if matches != deterministic:
echo "VERDICT: FAIL — offline range does NOT reproduce the live metric."
quit(1)
echo "VERDICT: PASS — offline == online for all 12 deterministic guns."
echo "(Tsetlin is stochastic and is allowed to differ.)"
when isMainModule:
main()
@@ -0,0 +1,21 @@
## Task 2: emit the synthetic trajectory fixtures to tools/fixtures/ as JSONL.
## Run with:
## nim c -r common_libs/tests/gen_synthetic_fixtures.nim
## Deterministic: the random-walk fixture is seeded.
import std/[os, strformat]
import gun_harness/offline_range
const outDir = currentSourcePath().parentDir.parentDir.parentDir / "tools" / "fixtures"
proc main() =
createDir(outDir)
for name in SyntheticFixtureNames:
let fx = synthesizeByName(name)
let path = outDir / (name & ".jsonl")
saveFixture(path, fx)
echo fmt"wrote {path} ({fx.states.len} ticks, source={fx.meta.source})"
echo "done."
when isMainModule:
main()
+42
View File
@@ -0,0 +1,42 @@
## Shared helper: build the 13 ModularBot guns as type-erased offline range
## drivers, in ModularBot's gun-id order, with the same readiness gate the live
## loop uses (Tsetlin only spawns once its 10-frame window is full).
import std/random
import gun_harness/offline_range
import guns/head_on
import guns/linear
import guns/circular
import guns/tsetlin
import guns/guess_factor
import guns/pattern_matcher
import guns/wall_bounce
import guns/accel_predictor
import guns/stop_shot
import guns/displacement
import guns/averaged_lead
import guns/decay_gf
import guns/knn_gun
proc buildAllGunDrivers*(seed = -1): seq[GunDriver] =
## seed >= 0 re-seeds the global RNG after constructing Tsetlin so the
## stochastic gun's learning is reproducible for offline runs. (Its
## constructor calls randomize(); we override that seed afterwards.)
var tsetlin = initTsetlinGun()
if seed >= 0:
randomize(seed)
result = @[
makeDriver("HeadOn", HeadOnGun()),
makeDriver("Linear", LinearGun()),
makeDriver("Tsetlin", tsetlin),
makeDriver("Circular", CircularGun()),
makeDriver("GuessFactor", initGFGun()),
makeDriver("Pattern", PatternMatcherGun()),
makeDriver("WallBounce", initWallBounceGun()),
makeDriver("Accel", initAccelGun()),
makeDriver("StopShot", initStopShotGun()),
makeDriver("Displace", initDisplacementGun()),
makeDriver("AvgLead", initAveragedLeadGun()),
makeDriver("DecayGF", initDecayGFGun()),
makeDriver("KNN", initKNNGun()),
]
+66
View File
@@ -0,0 +1,66 @@
## Task 4: offline gun-range runner. Replays one or more fixtures through the
## 13 ModularBot guns and prints a deterministic per-gun table (overall hit rate
## plus per-power-bin hits/shots) for each fixture.
##
## Usage:
## nim c -r common_libs/tests/run_range.nim # all tools/fixtures/*.jsonl
## nim c -r common_libs/tests/run_range.nim path/a.jsonl path/b.jsonl
## nim c -r common_libs/tests/run_range.nim --timing # add per-fixture wall time
##
## Tsetlin is stochastic; it is seeded (seed=1) here so this tool's output is
## reproducible run-to-run. Its numbers are NOT expected to match a live battle.
import std/[os, algorithm, strformat, times]
import gun_harness/offline_range
import range_guns
const fixturesDir = currentSourcePath().parentDir.parentDir.parentDir / "tools" / "fixtures"
proc fixtureFiles(args: seq[string]): seq[string] =
for a in args:
if a == "--timing": continue
if dirExists(a):
for f in walkFiles(a / "*.jsonl"): result.add f
elif fileExists(a):
result.add a
else:
echo "warning: not found: ", a
if result.len == 0:
for f in walkFiles(fixturesDir / "*.jsonl"): result.add f
result.sort()
proc main() =
var args: seq[string]
for i in 1..paramCount():
args.add paramStr(i)
let showTiming = "--timing" in args
let files = fixtureFiles(args)
if files.len == 0:
echo "no fixtures found under ", fixturesDir
echo "run: nim c -r common_libs/tests/gen_synthetic_fixtures.nim"
quit(1)
var grandShots, grandHits = 0
for path in files:
let fx = loadFixture(path)
var t0 = 0.0
if showTiming: t0 = epochTime()
let reports = replayFixture(fx, buildAllGunDrivers(seed = 1),
liveActual = (fx.meta.source == "live"))
let elapsed = if showTiming: epochTime() - t0 else: 0.0
stdout.write formatReports(fx, reports)
if showTiming:
echo fmt" replay: {elapsed*1000.0:.1f} ms"
var totShots, totHits = 0
for r in reports:
totShots += r.shots
totHits += r.hits
grandShots += totShots
grandHits += totHits
echo ""
if files.len > 1:
echo fmt"TOTAL across {files.len} fixtures: {grandHits}/{grandShots} ({grandHits.float/grandShots.float*100.0:.1f}%)"
echo fmt"(replayed {files.len} fixture(s))"
when isMainModule:
main()
+87 -1
View File
@@ -10,14 +10,17 @@
## and fast. The spawn/tick pipeline itself is exercised by the droppedBullets
## test below and by the full gauntlet.
import std/[math, random, tables]
import std/[math, random, tables, os]
import gun_harness/gun_interface
import gun_harness/virtual_bullets
import gun_harness/selector
import gun_harness/offline_range
import guns/stop_shot
import guns/displacement
import guns/averaged_lead
import guns/pattern_matcher
import guns/head_on
import guns/linear
var failures = 0
@@ -265,6 +268,83 @@ proc testDegenerateDistance() =
check "range gate: NaN distance also falls back to the ceiling",
aimToleranceDeg(NaN) == MaxAimThresholdDeg
# ── offline range (Task 5) ────────────────────────────────────────────────────
proc testFixtureRoundTrip() =
let fx = synthesizeConstantVelocity(ticks = 40)
let path = getTempDir() / "gun_range_roundtrip.jsonl"
saveFixture(path, fx)
let back = loadFixture(path)
check "range: fixture round-trip preserves tick count",
back.states.len == fx.states.len
var same = back.states.len == fx.states.len
for i in 0..<min(back.states.len, fx.states.len):
let a = fx.states[i]
let b = back.states[i]
if a.tick != b.tick or abs(a.enemyX - b.enemyX) > 1e-9 or
abs(a.enemyY - b.enemyY) > 1e-9 or
abs(a.enemyHeading - b.enemyHeading) > 1e-9 or
abs(a.enemySpeed - b.enemySpeed) > 1e-9 or
abs(a.selfX - b.selfX) > 1e-9:
same = false
check "range: fixture round-trip preserves per-tick fields", same
check "range: fixture round-trip defaults arena to 800x600",
back.meta.arenaW == 800.0 and back.meta.arenaH == 600.0
check "range: fixture round-trip preserves enemy id", back.enemyId == fx.enemyId
removeFile(path)
proc testFixtureEndMarkerRoundTrip() =
let fx = synthesizeStationary(ticks = 10)
var withEnd = fx
withEnd.enemyDied = true
let path = getTempDir() / "gun_range_end.jsonl"
saveFixture(path, withEnd)
let back = loadFixture(path)
check "range: fixture round-trip preserves the enemyDied end marker", back.enemyDied
removeFile(path)
proc testReplayDeterminism() =
let fx = synthesizeCircular(ticks = 80)
let r1 = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
makeDriver("Linear", LinearGun())])
let r2 = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
makeDriver("Linear", LinearGun())])
var same = r1.len == r2.len
for i in 0..<r1.len:
if r1[i].name != r2[i].name or r1[i].shots != r2[i].shots or
r1[i].hits != r2[i].hits:
same = false
for b in 0..<len(PowerBins):
if r1[i].bins[b].shots != r2[i].bins[b].shots or
r1[i].bins[b].hits != r2[i].bins[b].hits:
same = false
check "range: replaying the same fixture twice is byte-identical (deterministic guns)",
same
proc testRangeGroundTruthStationary() =
let fx = synthesizeStationary(ticks = 120)
let r = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun())])
# The last bullets are still in flight when the fixture ends, so shots < 480;
# every resolved shot must still be a hit.
check "range: stationary enemy -> HeadOn scores 100%",
r[0].shots > 300 and r[0].hits == r[0].shots
proc testRangeConstantVelocityLinearWins() =
let fx = synthesizeConstantVelocity(ticks = 120)
let r = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
makeDriver("Linear", LinearGun())])
check "range: constant velocity -> Linear beats HeadOn",
r[1].hitRate() > r[0].hitRate()
proc testEnergyThresholdFixtureRule() =
# RULE e(t) = max(5, 50 - 0.5*t); e drops below 30 at t = 41.
let fx = synthesizeEnergyThresholdTurner(ticks = 100, e0 = 50.0,
decay = 0.5, threshold = 30.0)
let idx = 41
check "range: energy-threshold fixture crosses the rule threshold at t=41",
fx.states[idx].enemyEnergy < 30.0 and
fx.states[idx-1].enemyEnergy >= 30.0
# ── driver ───────────────────────────────────────────────────────────────────
randomize()
@@ -286,6 +366,12 @@ testToleranceFormula()
testPerfectAlignmentAlwaysFires()
testGrossMisalignmentFailsLongRange()
testDegenerateDistance()
testFixtureRoundTrip()
testFixtureEndMarkerRoundTrip()
testReplayDeterminism()
testRangeGroundTruthStationary()
testRangeConstantVelocityLinearWins()
testEnergyThresholdFixtureRule()
if failures > 0:
echo "\n", failures, " check(s) FAILED"