feat(gun_harness): offline gun range, proven equivalent to live play
Gun evaluation previously required a full end-to-end battle (Java server + battle runner + websocket IPC to 2 bot processes, 50 rounds, ~3.4 min) and yielded only ~300-900 REAL shots across 13 guns -- far too few to rank guns, which is why tuning needed many repetitions. VirtualTracker is already a pure function of (WorldState stream, gun list); the only reason it needed Java was where WorldState came from. So the range replays a seq[WorldState] through the SAME tracker: offline and online scores are the same metric by construction, not an approximation. ACCEPTANCE TEST (the point of the whole thing): record one live round, replay it offline, compare per-gun virtual hit rates. 12/12 deterministic guns match EXACTLY, reproduced twice. Tsetlin is compared separately because tmLearnOne calls rand(). Getting to 12/12 exposed two real ordering quirks in the live loop: run() calls go() before the aim/fire block, so tickBullets resolves against the NEXT tick's scan while the prediction used the previous one; and if the target dies during that go() the final tick's spawn+resolution is skipped entirely. The recorder emits an end marker for the second case. The 5th (selected-gun) predict call was verified to be a no-op. Measured cost: 8 fixtures (1770 ticks, ~92k virtual bullets, 13 guns) replay in 2.9 s, ~32k virtual bullets/s -- roughly 70x faster and 100x more samples than a live gauntlet. Also adds a per-tick WorldState recorder behind const RecordWorldState (default off, mirrors the ShotLog idiom) which records the state the bot ACTUALLY builds, staleness included, rather than true positions -- recording the latter would hand the guns perfect information and produce flattering scores. 9 new guard checks (33 total, all passing), including fixture round-trip, replay determinism, stationary->HeadOn 100%, constant-velocity->Linear>HeadOn, and the energy-threshold turner crossing at t=41.
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
## Task 3: acceptance test — prove the offline range reproduces the live
|
||||
## virtual-bullet metric, or the range is worthless.
|
||||
##
|
||||
## Steps:
|
||||
## 1. run ONE live ModularBot vs OscillatorBot round with the ModularBot
|
||||
## recorder ON (compiled in via `const RecordWorldState = true`),
|
||||
## 2. read the online per-gun virtual fitness from /tmp/gun_stats.jsonl,
|
||||
## 3. replay the recorded WorldState fixture offline through the same guns,
|
||||
## 4. compare.
|
||||
##
|
||||
## Tsetlin is stochastic (tmLearnOne calls rand(); its constructor calls
|
||||
## randomize()), so byte-identical replay is impossible for it. The 12
|
||||
## deterministic guns must match EXACTLY; Tsetlin is reported separately.
|
||||
##
|
||||
## Run with:
|
||||
## nim c -r common_libs/tests/acceptance_offline_vs_online.nim
|
||||
##
|
||||
## Requires TR_SERVER_JAR / TR_BATTLE_RUNNER (or the default dev paths below).
|
||||
|
||||
import std/[json, os, strformat, strutils, math]
|
||||
import test_framework/test_framework
|
||||
import gun_harness/offline_range
|
||||
import range_guns
|
||||
|
||||
const
|
||||
repoRoot = currentSourcePath().parentDir.parentDir.parentDir
|
||||
modularBotDir = repoRoot / "ModularBot_garage"
|
||||
adversaryDir = repoRoot / "common_libs" / "test_framework" / "adversaries" / "OscillatorBot"
|
||||
statsPath = "/tmp/gun_stats.jsonl"
|
||||
recordPath = "/tmp/worldstate_record.jsonl"
|
||||
serverJar = "/home/davide/Projects/tank-royale/server/build/libs/robocode-tankroyale-server-0.35.5-all.jar"
|
||||
runnerJar = "/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar"
|
||||
|
||||
const TsetlinId = 2
|
||||
|
||||
proc lastOnlineRound(path: string): JsonNode =
|
||||
result = nil
|
||||
for line in lines(path):
|
||||
let s = line.strip()
|
||||
if s.len == 0: continue
|
||||
let node = parseJson(s)
|
||||
if node.hasKey("guns"): result = node
|
||||
|
||||
proc main() =
|
||||
if not fileExists(serverJar) or not fileExists(runnerJar):
|
||||
echo "Skipping: TR JARs not found (server=", serverJar, ", runner=", runnerJar, ")"
|
||||
quit(0)
|
||||
if not fileExists(modularBotDir / "src" / "ModularBot.nim"):
|
||||
echo "Skipping: ModularBot source not found"
|
||||
quit(0)
|
||||
|
||||
for p in [statsPath, recordPath]:
|
||||
if fileExists(p): removeFile(p)
|
||||
|
||||
echo "=== live battle: ModularBot vs OscillatorBot, 1 round, max speed ==="
|
||||
let battle = runBattle(@[modularBotDir, adversaryDir], rounds = 1,
|
||||
timeout = 240000, maxSpeed = true)
|
||||
for res in battle.results:
|
||||
echo fmt" {res.name:<14} rank={res.rank} score={res.totalScore}"
|
||||
|
||||
if not fileExists(recordPath):
|
||||
echo "FAIL: recorder produced no fixture (is RecordWorldState true?)"
|
||||
quit(1)
|
||||
if not fileExists(statsPath):
|
||||
echo "FAIL: no /tmp/gun_stats.jsonl"
|
||||
quit(1)
|
||||
|
||||
let online = lastOnlineRound(statsPath)
|
||||
let fx = loadFixture(recordPath)
|
||||
let reports = replayFixture(fx, buildAllGunDrivers(), liveActual = true)
|
||||
|
||||
# Map online stats by gun id.
|
||||
var onShots: array[13, int]
|
||||
var onHits: array[13, int]
|
||||
var onNames: array[13, string]
|
||||
for g in online["guns"]:
|
||||
let id = g["id"].getInt()
|
||||
if id >= 0 and id < 13:
|
||||
onShots[id] = g["vShots"].getInt()
|
||||
onHits[id] = g["vHits"].getInt()
|
||||
onNames[id] = g["name"].getStr()
|
||||
|
||||
echo ""
|
||||
echo fmt"fixture: {recordPath} ticks={fx.states.len} enemyId={fx.enemyId} enemyDied={fx.enemyDied}"
|
||||
echo "online stats: /tmp/gun_stats.jsonl round ", online["round"].getInt()
|
||||
echo ""
|
||||
echo "gun online(vHits/vShots) offline(hits/shots) verdict"
|
||||
echo "-----------------------------------------------------------------------"
|
||||
var matches = 0
|
||||
var deterministic = 0
|
||||
for id in 0..<13:
|
||||
let r = reports[id]
|
||||
let match = r.hits == onHits[id] and r.shots == onShots[id]
|
||||
var verdict: string
|
||||
if id == TsetlinId:
|
||||
verdict = if match: "MATCH (stochastic)" else: "differs (stochastic, expected)"
|
||||
else:
|
||||
inc deterministic
|
||||
if match:
|
||||
inc matches
|
||||
verdict = "OK"
|
||||
else:
|
||||
verdict = "MISMATCH"
|
||||
echo fmt"{r.name:<12} {onHits[id]:>5}/{onShots[id]:<5} {r.hits:>5}/{r.shots:<5} {verdict}"
|
||||
|
||||
echo ""
|
||||
echo fmt"deterministic guns matching exactly: {matches}/{deterministic}"
|
||||
if matches != deterministic:
|
||||
echo "VERDICT: FAIL — offline range does NOT reproduce the live metric."
|
||||
quit(1)
|
||||
echo "VERDICT: PASS — offline == online for all 12 deterministic guns."
|
||||
echo "(Tsetlin is stochastic and is allowed to differ.)"
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
@@ -0,0 +1,21 @@
|
||||
## Task 2: emit the synthetic trajectory fixtures to tools/fixtures/ as JSONL.
|
||||
## Run with:
|
||||
## nim c -r common_libs/tests/gen_synthetic_fixtures.nim
|
||||
## Deterministic: the random-walk fixture is seeded.
|
||||
|
||||
import std/[os, strformat]
|
||||
import gun_harness/offline_range
|
||||
|
||||
const outDir = currentSourcePath().parentDir.parentDir.parentDir / "tools" / "fixtures"
|
||||
|
||||
proc main() =
|
||||
createDir(outDir)
|
||||
for name in SyntheticFixtureNames:
|
||||
let fx = synthesizeByName(name)
|
||||
let path = outDir / (name & ".jsonl")
|
||||
saveFixture(path, fx)
|
||||
echo fmt"wrote {path} ({fx.states.len} ticks, source={fx.meta.source})"
|
||||
echo "done."
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
## Shared helper: build the 13 ModularBot guns as type-erased offline range
|
||||
## drivers, in ModularBot's gun-id order, with the same readiness gate the live
|
||||
## loop uses (Tsetlin only spawns once its 10-frame window is full).
|
||||
|
||||
import std/random
|
||||
import gun_harness/offline_range
|
||||
import guns/head_on
|
||||
import guns/linear
|
||||
import guns/circular
|
||||
import guns/tsetlin
|
||||
import guns/guess_factor
|
||||
import guns/pattern_matcher
|
||||
import guns/wall_bounce
|
||||
import guns/accel_predictor
|
||||
import guns/stop_shot
|
||||
import guns/displacement
|
||||
import guns/averaged_lead
|
||||
import guns/decay_gf
|
||||
import guns/knn_gun
|
||||
|
||||
proc buildAllGunDrivers*(seed = -1): seq[GunDriver] =
|
||||
## seed >= 0 re-seeds the global RNG after constructing Tsetlin so the
|
||||
## stochastic gun's learning is reproducible for offline runs. (Its
|
||||
## constructor calls randomize(); we override that seed afterwards.)
|
||||
var tsetlin = initTsetlinGun()
|
||||
if seed >= 0:
|
||||
randomize(seed)
|
||||
result = @[
|
||||
makeDriver("HeadOn", HeadOnGun()),
|
||||
makeDriver("Linear", LinearGun()),
|
||||
makeDriver("Tsetlin", tsetlin),
|
||||
makeDriver("Circular", CircularGun()),
|
||||
makeDriver("GuessFactor", initGFGun()),
|
||||
makeDriver("Pattern", PatternMatcherGun()),
|
||||
makeDriver("WallBounce", initWallBounceGun()),
|
||||
makeDriver("Accel", initAccelGun()),
|
||||
makeDriver("StopShot", initStopShotGun()),
|
||||
makeDriver("Displace", initDisplacementGun()),
|
||||
makeDriver("AvgLead", initAveragedLeadGun()),
|
||||
makeDriver("DecayGF", initDecayGFGun()),
|
||||
makeDriver("KNN", initKNNGun()),
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
## Task 4: offline gun-range runner. Replays one or more fixtures through the
|
||||
## 13 ModularBot guns and prints a deterministic per-gun table (overall hit rate
|
||||
## plus per-power-bin hits/shots) for each fixture.
|
||||
##
|
||||
## Usage:
|
||||
## nim c -r common_libs/tests/run_range.nim # all tools/fixtures/*.jsonl
|
||||
## nim c -r common_libs/tests/run_range.nim path/a.jsonl path/b.jsonl
|
||||
## nim c -r common_libs/tests/run_range.nim --timing # add per-fixture wall time
|
||||
##
|
||||
## Tsetlin is stochastic; it is seeded (seed=1) here so this tool's output is
|
||||
## reproducible run-to-run. Its numbers are NOT expected to match a live battle.
|
||||
|
||||
import std/[os, algorithm, strformat, times]
|
||||
import gun_harness/offline_range
|
||||
import range_guns
|
||||
|
||||
const fixturesDir = currentSourcePath().parentDir.parentDir.parentDir / "tools" / "fixtures"
|
||||
|
||||
proc fixtureFiles(args: seq[string]): seq[string] =
|
||||
for a in args:
|
||||
if a == "--timing": continue
|
||||
if dirExists(a):
|
||||
for f in walkFiles(a / "*.jsonl"): result.add f
|
||||
elif fileExists(a):
|
||||
result.add a
|
||||
else:
|
||||
echo "warning: not found: ", a
|
||||
if result.len == 0:
|
||||
for f in walkFiles(fixturesDir / "*.jsonl"): result.add f
|
||||
result.sort()
|
||||
|
||||
proc main() =
|
||||
var args: seq[string]
|
||||
for i in 1..paramCount():
|
||||
args.add paramStr(i)
|
||||
let showTiming = "--timing" in args
|
||||
let files = fixtureFiles(args)
|
||||
if files.len == 0:
|
||||
echo "no fixtures found under ", fixturesDir
|
||||
echo "run: nim c -r common_libs/tests/gen_synthetic_fixtures.nim"
|
||||
quit(1)
|
||||
|
||||
var grandShots, grandHits = 0
|
||||
for path in files:
|
||||
let fx = loadFixture(path)
|
||||
var t0 = 0.0
|
||||
if showTiming: t0 = epochTime()
|
||||
let reports = replayFixture(fx, buildAllGunDrivers(seed = 1),
|
||||
liveActual = (fx.meta.source == "live"))
|
||||
let elapsed = if showTiming: epochTime() - t0 else: 0.0
|
||||
stdout.write formatReports(fx, reports)
|
||||
if showTiming:
|
||||
echo fmt" replay: {elapsed*1000.0:.1f} ms"
|
||||
var totShots, totHits = 0
|
||||
for r in reports:
|
||||
totShots += r.shots
|
||||
totHits += r.hits
|
||||
grandShots += totShots
|
||||
grandHits += totHits
|
||||
echo ""
|
||||
if files.len > 1:
|
||||
echo fmt"TOTAL across {files.len} fixtures: {grandHits}/{grandShots} ({grandHits.float/grandShots.float*100.0:.1f}%)"
|
||||
echo fmt"(replayed {files.len} fixture(s))"
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
@@ -10,14 +10,17 @@
|
||||
## and fast. The spawn/tick pipeline itself is exercised by the droppedBullets
|
||||
## test below and by the full gauntlet.
|
||||
|
||||
import std/[math, random, tables]
|
||||
import std/[math, random, tables, os]
|
||||
import gun_harness/gun_interface
|
||||
import gun_harness/virtual_bullets
|
||||
import gun_harness/selector
|
||||
import gun_harness/offline_range
|
||||
import guns/stop_shot
|
||||
import guns/displacement
|
||||
import guns/averaged_lead
|
||||
import guns/pattern_matcher
|
||||
import guns/head_on
|
||||
import guns/linear
|
||||
|
||||
var failures = 0
|
||||
|
||||
@@ -265,6 +268,83 @@ proc testDegenerateDistance() =
|
||||
check "range gate: NaN distance also falls back to the ceiling",
|
||||
aimToleranceDeg(NaN) == MaxAimThresholdDeg
|
||||
|
||||
# ── offline range (Task 5) ────────────────────────────────────────────────────
|
||||
|
||||
proc testFixtureRoundTrip() =
|
||||
let fx = synthesizeConstantVelocity(ticks = 40)
|
||||
let path = getTempDir() / "gun_range_roundtrip.jsonl"
|
||||
saveFixture(path, fx)
|
||||
let back = loadFixture(path)
|
||||
check "range: fixture round-trip preserves tick count",
|
||||
back.states.len == fx.states.len
|
||||
var same = back.states.len == fx.states.len
|
||||
for i in 0..<min(back.states.len, fx.states.len):
|
||||
let a = fx.states[i]
|
||||
let b = back.states[i]
|
||||
if a.tick != b.tick or abs(a.enemyX - b.enemyX) > 1e-9 or
|
||||
abs(a.enemyY - b.enemyY) > 1e-9 or
|
||||
abs(a.enemyHeading - b.enemyHeading) > 1e-9 or
|
||||
abs(a.enemySpeed - b.enemySpeed) > 1e-9 or
|
||||
abs(a.selfX - b.selfX) > 1e-9:
|
||||
same = false
|
||||
check "range: fixture round-trip preserves per-tick fields", same
|
||||
check "range: fixture round-trip defaults arena to 800x600",
|
||||
back.meta.arenaW == 800.0 and back.meta.arenaH == 600.0
|
||||
check "range: fixture round-trip preserves enemy id", back.enemyId == fx.enemyId
|
||||
removeFile(path)
|
||||
|
||||
proc testFixtureEndMarkerRoundTrip() =
|
||||
let fx = synthesizeStationary(ticks = 10)
|
||||
var withEnd = fx
|
||||
withEnd.enemyDied = true
|
||||
let path = getTempDir() / "gun_range_end.jsonl"
|
||||
saveFixture(path, withEnd)
|
||||
let back = loadFixture(path)
|
||||
check "range: fixture round-trip preserves the enemyDied end marker", back.enemyDied
|
||||
removeFile(path)
|
||||
|
||||
proc testReplayDeterminism() =
|
||||
let fx = synthesizeCircular(ticks = 80)
|
||||
let r1 = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
|
||||
makeDriver("Linear", LinearGun())])
|
||||
let r2 = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
|
||||
makeDriver("Linear", LinearGun())])
|
||||
var same = r1.len == r2.len
|
||||
for i in 0..<r1.len:
|
||||
if r1[i].name != r2[i].name or r1[i].shots != r2[i].shots or
|
||||
r1[i].hits != r2[i].hits:
|
||||
same = false
|
||||
for b in 0..<len(PowerBins):
|
||||
if r1[i].bins[b].shots != r2[i].bins[b].shots or
|
||||
r1[i].bins[b].hits != r2[i].bins[b].hits:
|
||||
same = false
|
||||
check "range: replaying the same fixture twice is byte-identical (deterministic guns)",
|
||||
same
|
||||
|
||||
proc testRangeGroundTruthStationary() =
|
||||
let fx = synthesizeStationary(ticks = 120)
|
||||
let r = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun())])
|
||||
# The last bullets are still in flight when the fixture ends, so shots < 480;
|
||||
# every resolved shot must still be a hit.
|
||||
check "range: stationary enemy -> HeadOn scores 100%",
|
||||
r[0].shots > 300 and r[0].hits == r[0].shots
|
||||
|
||||
proc testRangeConstantVelocityLinearWins() =
|
||||
let fx = synthesizeConstantVelocity(ticks = 120)
|
||||
let r = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()),
|
||||
makeDriver("Linear", LinearGun())])
|
||||
check "range: constant velocity -> Linear beats HeadOn",
|
||||
r[1].hitRate() > r[0].hitRate()
|
||||
|
||||
proc testEnergyThresholdFixtureRule() =
|
||||
# RULE e(t) = max(5, 50 - 0.5*t); e drops below 30 at t = 41.
|
||||
let fx = synthesizeEnergyThresholdTurner(ticks = 100, e0 = 50.0,
|
||||
decay = 0.5, threshold = 30.0)
|
||||
let idx = 41
|
||||
check "range: energy-threshold fixture crosses the rule threshold at t=41",
|
||||
fx.states[idx].enemyEnergy < 30.0 and
|
||||
fx.states[idx-1].enemyEnergy >= 30.0
|
||||
|
||||
# ── driver ───────────────────────────────────────────────────────────────────
|
||||
|
||||
randomize()
|
||||
@@ -286,6 +366,12 @@ testToleranceFormula()
|
||||
testPerfectAlignmentAlwaysFires()
|
||||
testGrossMisalignmentFailsLongRange()
|
||||
testDegenerateDistance()
|
||||
testFixtureRoundTrip()
|
||||
testFixtureEndMarkerRoundTrip()
|
||||
testReplayDeterminism()
|
||||
testRangeGroundTruthStationary()
|
||||
testRangeConstantVelocityLinearWins()
|
||||
testEnergyThresholdFixtureRule()
|
||||
|
||||
if failures > 0:
|
||||
echo "\n", failures, " check(s) FAILED"
|
||||
|
||||
Reference in New Issue
Block a user