SirStone
974528d5cf
Gun evaluation previously required a full end-to-end battle (Java server + battle runner + websocket IPC to 2 bot processes, 50 rounds, ~3.4 min) and yielded only ~300-900 REAL shots across 13 guns -- far too few to rank guns, which is why tuning needed many repetitions. VirtualTracker is already a pure function of (WorldState stream, gun list); the only reason it needed Java was where WorldState came from. So the range replays a seq[WorldState] through the SAME tracker: offline and online scores are the same metric by construction, not an approximation. ACCEPTANCE TEST (the point of the whole thing): record one live round, replay it offline, compare per-gun virtual hit rates. 12/12 deterministic guns match EXACTLY, reproduced twice. Tsetlin is compared separately because tmLearnOne calls rand(). Getting to 12/12 exposed two real ordering quirks in the live loop: run() calls go() before the aim/fire block, so tickBullets resolves against the NEXT tick's scan while the prediction used the previous one; and if the target dies during that go() the final tick's spawn+resolution is skipped entirely. The recorder emits an end marker for the second case. The 5th (selected-gun) predict call was verified to be a no-op. Measured cost: 8 fixtures (1770 ticks, ~92k virtual bullets, 13 guns) replay in 2.9 s, ~32k virtual bullets/s -- roughly 70x faster and 100x more samples than a live gauntlet. Also adds a per-tick WorldState recorder behind const RecordWorldState (default off, mirrors the ShotLog idiom) which records the state the bot ACTUALLY builds, staleness included, rather than true positions -- recording the latter would hand the guns perfect information and produce flattering scores. 9 new guard checks (33 total, all passing), including fixture round-trip, replay determinism, stationary->HeadOn 100%, constant-velocity->Linear>HeadOn, and the energy-threshold turner crossing at t=41.
116 lines
4.3 KiB
Nim
116 lines
4.3 KiB
Nim
## Task 3: acceptance test — prove the offline range reproduces the live
|
|
## virtual-bullet metric, or the range is worthless.
|
|
##
|
|
## Steps:
|
|
## 1. run ONE live ModularBot vs OscillatorBot round with the ModularBot
|
|
## recorder ON (compiled in via `const RecordWorldState = true`),
|
|
## 2. read the online per-gun virtual fitness from /tmp/gun_stats.jsonl,
|
|
## 3. replay the recorded WorldState fixture offline through the same guns,
|
|
## 4. compare.
|
|
##
|
|
## Tsetlin is stochastic (tmLearnOne calls rand(); its constructor calls
|
|
## randomize()), so byte-identical replay is impossible for it. The 12
|
|
## deterministic guns must match EXACTLY; Tsetlin is reported separately.
|
|
##
|
|
## Run with:
|
|
## nim c -r common_libs/tests/acceptance_offline_vs_online.nim
|
|
##
|
|
## Requires TR_SERVER_JAR / TR_BATTLE_RUNNER (or the default dev paths below).
|
|
|
|
import std/[json, os, strformat, strutils, math]
|
|
import test_framework/test_framework
|
|
import gun_harness/offline_range
|
|
import range_guns
|
|
|
|
const
|
|
repoRoot = currentSourcePath().parentDir.parentDir.parentDir
|
|
modularBotDir = repoRoot / "ModularBot_garage"
|
|
adversaryDir = repoRoot / "common_libs" / "test_framework" / "adversaries" / "OscillatorBot"
|
|
statsPath = "/tmp/gun_stats.jsonl"
|
|
recordPath = "/tmp/worldstate_record.jsonl"
|
|
serverJar = "/home/davide/Projects/tank-royale/server/build/libs/robocode-tankroyale-server-0.35.5-all.jar"
|
|
runnerJar = "/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar"
|
|
|
|
const TsetlinId = 2
|
|
|
|
proc lastOnlineRound(path: string): JsonNode =
|
|
result = nil
|
|
for line in lines(path):
|
|
let s = line.strip()
|
|
if s.len == 0: continue
|
|
let node = parseJson(s)
|
|
if node.hasKey("guns"): result = node
|
|
|
|
proc main() =
|
|
if not fileExists(serverJar) or not fileExists(runnerJar):
|
|
echo "Skipping: TR JARs not found (server=", serverJar, ", runner=", runnerJar, ")"
|
|
quit(0)
|
|
if not fileExists(modularBotDir / "src" / "ModularBot.nim"):
|
|
echo "Skipping: ModularBot source not found"
|
|
quit(0)
|
|
|
|
for p in [statsPath, recordPath]:
|
|
if fileExists(p): removeFile(p)
|
|
|
|
echo "=== live battle: ModularBot vs OscillatorBot, 1 round, max speed ==="
|
|
let battle = runBattle(@[modularBotDir, adversaryDir], rounds = 1,
|
|
timeout = 240000, maxSpeed = true)
|
|
for res in battle.results:
|
|
echo fmt" {res.name:<14} rank={res.rank} score={res.totalScore}"
|
|
|
|
if not fileExists(recordPath):
|
|
echo "FAIL: recorder produced no fixture (is RecordWorldState true?)"
|
|
quit(1)
|
|
if not fileExists(statsPath):
|
|
echo "FAIL: no /tmp/gun_stats.jsonl"
|
|
quit(1)
|
|
|
|
let online = lastOnlineRound(statsPath)
|
|
let fx = loadFixture(recordPath)
|
|
let reports = replayFixture(fx, buildAllGunDrivers(), liveActual = true)
|
|
|
|
# Map online stats by gun id.
|
|
var onShots: array[13, int]
|
|
var onHits: array[13, int]
|
|
var onNames: array[13, string]
|
|
for g in online["guns"]:
|
|
let id = g["id"].getInt()
|
|
if id >= 0 and id < 13:
|
|
onShots[id] = g["vShots"].getInt()
|
|
onHits[id] = g["vHits"].getInt()
|
|
onNames[id] = g["name"].getStr()
|
|
|
|
echo ""
|
|
echo fmt"fixture: {recordPath} ticks={fx.states.len} enemyId={fx.enemyId} enemyDied={fx.enemyDied}"
|
|
echo "online stats: /tmp/gun_stats.jsonl round ", online["round"].getInt()
|
|
echo ""
|
|
echo "gun online(vHits/vShots) offline(hits/shots) verdict"
|
|
echo "-----------------------------------------------------------------------"
|
|
var matches = 0
|
|
var deterministic = 0
|
|
for id in 0..<13:
|
|
let r = reports[id]
|
|
let match = r.hits == onHits[id] and r.shots == onShots[id]
|
|
var verdict: string
|
|
if id == TsetlinId:
|
|
verdict = if match: "MATCH (stochastic)" else: "differs (stochastic, expected)"
|
|
else:
|
|
inc deterministic
|
|
if match:
|
|
inc matches
|
|
verdict = "OK"
|
|
else:
|
|
verdict = "MISMATCH"
|
|
echo fmt"{r.name:<12} {onHits[id]:>5}/{onShots[id]:<5} {r.hits:>5}/{r.shots:<5} {verdict}"
|
|
|
|
echo ""
|
|
echo fmt"deterministic guns matching exactly: {matches}/{deterministic}"
|
|
if matches != deterministic:
|
|
echo "VERDICT: FAIL — offline range does NOT reproduce the live metric."
|
|
quit(1)
|
|
echo "VERDICT: PASS — offline == online for all 12 deterministic guns."
|
|
echo "(Tsetlin is stochastic and is allowed to differ.)"
|
|
|
|
when isMainModule:
|
|
main()
|