a82c864c60
New harness (common_libs/gun_harness/prediction_quality.nim + common_libs/tests/run_prediction_quality.nim): per-gun single-tick aim error in degrees against the true continuous interception point on the recorded live-vs-real-DrussGT corpus (/tmp/tfil_ab2/out, 70 runs, 899607 ticks), per range band, with the hit-probability proxy mean(|err|<=atan(18/range)). Validated: recorded hits separate from misses 13.34x px (reference 11.59x), perfect-oracle max |err| = 0, correct ordering on synthetic ground truth, two full runs byte-identical. Fixed a wrap180 bug (Nim float mod keeps the dividend sign) that inflated the negative error tail. Bar (mean|err| deg [hitProxy] at 450+): Pattern 16.19 [0.077], naive-linear 22.86 [0.054], TMHorizon 16.20 [0.076], BitBrain 16.20 [0.077], static HeadOn 12.33 [0.098], oracle 0 [1.0]. Lead-gain sweep on Pattern is a dead end (1.0 wins every band). Naive-linear applies ~1.8x Pattern's lead but carries no more lead information (corr 0.178 vs 0.165) and is strictly worse. Ledger: docs/bitbrain_campaign.md. All verdicts remain live-only.
606 lines
24 KiB
Nim
606 lines
24 KiB
Nim
## Offline per-gun PREDICTION-QUALITY scorer — the campaign ruler.
|
||
##
|
||
## WHAT THIS MEASURES (and only this)
|
||
## ----------------------------------
|
||
## `docs/offline_harness_trust.md` established that the offline harness is
|
||
## trustworthy for ONE question: the per-gun, single-tick PREDICTION QUALITY of a
|
||
## gun on a FIXED enemy trajectory. It is never trustworthy for closed-loop
|
||
## questions (wins, damage, survival, movement, range, adaptation, selection).
|
||
## This module builds exactly that one trustworthy thing: given a recorded live
|
||
## battle, "if this gun had aimed at every tick it was asked to, how good was its
|
||
## aim?" — nothing more.
|
||
##
|
||
## THE METRIC
|
||
## ----------
|
||
## At each recorded tick `i` the shooter sits at `O = (selfX, selfY)`. For a
|
||
## bullet of speed `v` the AIM-INDEPENDENT interception point is the first future
|
||
## tick `t* = i + k` (k >= 1) with `|E(t*) - O| <= v*k` — where the enemy's
|
||
## ACTUAL recorded track crosses the bullet's path. This is the same solve as
|
||
## `common_libs/tests/analyze_lead_capture_by_range.py` (commit f91e121): it
|
||
## depends only on the recorded truth and the bullet speed, never on our aim, so
|
||
## it is a fixed target every gun is scored against.
|
||
##
|
||
## Angular error is `wrap180(bearing(O -> pred) - bearing(O -> E(t*)))` in
|
||
## DEGREES. Degrees are the physically meaningful unit (the arena spans 800 px
|
||
## but the tolerance shrinks with range), so the ruler never reports pixels except
|
||
## in the sanity-validation path. Per RANGE BAND (0-100/100-200/200-300/300-450/
|
||
## 450+) we report:
|
||
## mean |err| (deg), RMSE (deg), mean signed err (deg),
|
||
## the hit-probability proxy `mean(|err| < atan(18/range))`, and n.
|
||
##
|
||
## The "perfect oracle" arm aims at `E(t*)` itself, so it must score ~0 error —
|
||
## that is the plumbing check. The real correctness check is `validateShots`,
|
||
## which scores OUR ACTUAL recorded fired bearings (from the event sidecar)
|
||
## against the same interception solve: recorded HITS must cluster near zero and
|
||
## MISSES far away. If that separation collapses, the ruler is wrong.
|
||
##
|
||
## EVALUATION GRANULARITY
|
||
## ----------------------
|
||
## The offline replay fires no real bullets, so a gun is asked for a prediction
|
||
## per POWER BIN (`PowerBins = 1.0/1.5/2.0/3.0`) each tick; each bin's prediction
|
||
## is scored against the interception at that bin's bullet speed. The phrase "the
|
||
## power that was really fired" applies to the separate `validateShots` check,
|
||
## which uses the server-recorded fired power. Aggregating over the four bins is
|
||
## a common horizon set shared by every arm, so the comparison is fair.
|
||
##
|
||
## WHY NOT `replayFixture`
|
||
## -----------------------
|
||
## `common_libs/gun_harness/offline_range.nim` replays a fixture through a
|
||
## `VirtualTracker` so that feedback-adaptive guns (Tsetlin, KNN, DecayGF) learn
|
||
## from resolved virtual bullets. Every arm measured here — Pattern, TMHorizon,
|
||
## BitBrain, HeadOn, and the naive-linear control — has a no-op `onResult`: its
|
||
## prediction is a pure function of the observed `WorldState` stream, so the
|
||
## tracker changes nothing while costing an O(MaxBullets) resolution scan per
|
||
## tick (~8192 slots), which would dominate runtime. This module therefore keeps
|
||
## the recorded-state semantics (states replayed in order, one gun's history is
|
||
## the whole stream) but drives the guns directly, and it needs per-tick
|
||
## predictions — which `replayFixture`'s aggregate report does not expose.
|
||
## The corpus representation is the recorded-run loader below (round boundaries
|
||
## from `*.rounds.json`, event sidecar for validation, binary cache for speed),
|
||
## i.e. the same recorded-state contract with the battle metadata the analysis
|
||
## needs.
|
||
##
|
||
## COST / CACHING
|
||
## --------------
|
||
## The corpus is ~900k recorded ticks over 70 battles (149 MB of JSONL). Parsing
|
||
## that with `std/json` on every sweep would dominate runtime, so the loader
|
||
## converts each run once into a compact binary `.qcache` (float32, 10 fields per
|
||
## tick) keyed on the source mtime+size. All measurements are taken from the
|
||
## cache, never from a mixture of cache and fresh parse, so two runs are
|
||
## byte-identical.
|
||
|
||
import std/[math, os, strformat, strutils, json, tables, times, algorithm]
|
||
|
||
const
|
||
NFields* = 10
|
||
NBands* = 5
|
||
BandLo* = [0.0, 100.0, 200.0, 300.0, 450.0]
|
||
BandHi* = [100.0, 200.0, 300.0, 450.0, 1.0e18]
|
||
BandLabels* = ["0-100", "100-200", "200-300", "300-450", "450+"]
|
||
MaxFlight* = 220
|
||
## max ticks a bullet is followed when solving for the interception tick
|
||
## (matches analyze_lead_capture_by_range.py).
|
||
BbRadius* = 18.0 ## hit-detection radius in px (atan(18/range) tolerance)
|
||
|
||
const CacheMagic = 0x31434242'i32 # "BBQ1"
|
||
const CacheVersion = 2'i32
|
||
|
||
proc wb(f: File, p: pointer, n: int) {.inline.} =
|
||
if n > 0: discard f.writeBuffer(p, n)
|
||
|
||
proc rb(f: File, p: pointer, n: int) {.inline.} =
|
||
if n > 0: discard f.readBuffer(p, n)
|
||
|
||
type
|
||
Corpus* = ref object
|
||
path*: string
|
||
arenaW*, arenaH*: float
|
||
n*: int
|
||
st*: seq[float32] ## NFields floats per tick
|
||
tick*: seq[int32]
|
||
rStart*: seq[int32] ## per-round first tick value
|
||
rCount*: seq[int32]
|
||
contiguous*: bool
|
||
base*: int
|
||
byTick*: Table[int32, int]
|
||
|
||
BandStat* = object
|
||
n*: int
|
||
sumAbs*: float64
|
||
sumSq*: float64
|
||
sumSigned*: float64
|
||
hits*: int
|
||
maxAbs*: float64
|
||
sumPred*: float64 ## sum of the arm's own lead over LOS (deg)
|
||
sumReq*: float64 ## sum of the true required lead over LOS (deg)
|
||
sumAbsReq*: float64
|
||
sumPredReq*: float64
|
||
sumPred2*: float64
|
||
sumReq2*: float64
|
||
|
||
ArmAcc* = object
|
||
name*: string
|
||
bands*: array[NBands, BandStat]
|
||
skipped*: int ## ticks×bins with no valid interception (no evidence)
|
||
evaluated*: int ## ticks×bins actually scored
|
||
|
||
TrEvent* = object
|
||
round*, tick*, owner*, bullet*: int
|
||
kind*: string
|
||
power*, x*, y*, dir*: float
|
||
|
||
ShotStat* = object
|
||
hits*, misses*: int
|
||
hitSumDeg*, missSumDeg*: float64
|
||
hitSumPx*, missSumPx*: float64
|
||
|
||
template ex*(c: Corpus, i: int): float = c.st[i * NFields + 0].float
|
||
template ey*(c: Corpus, i: int): float = c.st[i * NFields + 1].float
|
||
template eh*(c: Corpus, i: int): float = c.st[i * NFields + 2].float
|
||
template es*(c: Corpus, i: int): float = c.st[i * NFields + 3].float
|
||
template ee*(c: Corpus, i: int): float = c.st[i * NFields + 4].float
|
||
template sx*(c: Corpus, i: int): float = c.st[i * NFields + 5].float
|
||
template sy*(c: Corpus, i: int): float = c.st[i * NFields + 6].float
|
||
template sh*(c: Corpus, i: int): float = c.st[i * NFields + 7].float
|
||
template ss*(c: Corpus, i: int): float = c.st[i * NFields + 8].float
|
||
template se*(c: Corpus, i: int): float = c.st[i * NFields + 9].float
|
||
|
||
# ── geometry ─────────────────────────────────────────────────────────────────
|
||
|
||
proc wrap180*(x: float): float {.inline.} =
|
||
## Signed angular difference in (-180, 180]. NOTE: Nim's float `mod` keeps the
|
||
## sign of the dividend (C fmod), so `(x + 180) mod 360 - 180` is WRONG for
|
||
## x < -180 — it returns x - 360 instead of the wrapped equivalent. Normalise
|
||
## explicitly (this bug inflated the negative tail of every error before fix).
|
||
result = x mod 360.0
|
||
if result > 180.0: result -= 360.0
|
||
elif result <= -180.0: result += 360.0
|
||
|
||
proc bearingDeg*(ox, oy, px, py: float): float {.inline.} =
|
||
radToDeg(arctan2(py - oy, px - ox))
|
||
|
||
proc bandOf*(r: float): int {.inline.} =
|
||
for b in 0..<NBands:
|
||
if r >= BandLo[b] and r < BandHi[b]: return b
|
||
NBands - 1
|
||
|
||
proc tolDeg*(r: float): float {.inline.} =
|
||
## Angular half-width of the target disc at range `r`: atan(18/range).
|
||
radToDeg(arctan2(BbRadius, max(r, 1e-9)))
|
||
|
||
proc interceptTick(c: Corpus, i0, iEnd: int, ox, oy, speed: float): int =
|
||
## First integer k >= 1 with |E(i0+k) - O| <= speed*k and i0+k < iEnd; -1 if
|
||
## none. This is the integer-tick solve used by
|
||
## analyze_lead_capture_by_range.py.
|
||
let v = speed
|
||
var k = 1
|
||
while k <= MaxFlight:
|
||
let j = i0 + k
|
||
if j >= iEnd: break
|
||
let dx = c.ex(j) - ox
|
||
let dy = c.ey(j) - oy
|
||
if sqrt(dx * dx + dy * dy) <= v * float(k):
|
||
return k
|
||
inc k
|
||
-1
|
||
|
||
proc enemyPosAt(c: Corpus, i0, iEnd: int, t: float): tuple[x, y: float] =
|
||
## Linearly interpolated enemy position at fractional time `t` after i0.
|
||
let base = float(int(t))
|
||
let frac = t - base
|
||
let jm = min(i0 + int(base), iEnd - 1)
|
||
let jm2 = min(jm + 1, iEnd - 1)
|
||
(c.ex(jm) + (c.ex(jm2) - c.ex(jm)) * frac,
|
||
c.ey(jm) + (c.ey(jm2) - c.ey(jm)) * frac)
|
||
|
||
proc interceptBearingQuant*(c: Corpus, i0, iEnd: int, ox, oy,
|
||
speed: float): tuple[ok: bool, bearing, range: float] =
|
||
## The analyze_lead_capture_by_range.py solve: bearing to E(i0+k) at the first
|
||
## integer tick k the enemy is within the bullet's reach.
|
||
let rng = hypot(c.ex(i0) - ox, c.ey(i0) - oy)
|
||
let k = interceptTick(c, i0, iEnd, ox, oy, speed)
|
||
if k < 0: return (false, 0.0, rng)
|
||
(true, bearingDeg(ox, oy, c.ex(i0 + k), c.ey(i0 + k)), rng)
|
||
|
||
proc interceptBearingCont*(c: Corpus, i0, iEnd: int, ox, oy,
|
||
speed: float): tuple[ok: bool, bearing, range: float] =
|
||
## THE PHYSICALLY EXACT TRUE INTERCEPTION POINT: the first fractional time
|
||
## t > 0 at which the enemy's recorded track reaches distance speed*t from the
|
||
## origin, found by bisecting the first integer tick where it comes within
|
||
## reach. A bullet fired along the bearing to E(t) coincides with the enemy at
|
||
## t, so aiming at this point is a true hit (the integer-tick solve overshoots:
|
||
## it aims at E(k) but the bullet and enemy meet at E(t) with t <= k).
|
||
let rng = hypot(c.ex(i0) - ox, c.ey(i0) - oy)
|
||
let v = speed
|
||
var k = 1
|
||
while k <= MaxFlight:
|
||
let j = i0 + k
|
||
if j >= iEnd: break
|
||
let f = hypot(c.ex(j) - ox, c.ey(j) - oy) - v * float(k)
|
||
if f <= 0.0:
|
||
var lo = float(k - 1)
|
||
var hi = float(k)
|
||
for _ in 0 ..< 24:
|
||
let mid = 0.5 * (lo + hi)
|
||
let p = enemyPosAt(c, i0, iEnd, mid)
|
||
if hypot(p.x - ox, p.y - oy) - v * mid > 0.0: lo = mid
|
||
else: hi = mid
|
||
let ts = 0.5 * (lo + hi)
|
||
let p = enemyPosAt(c, i0, iEnd, ts)
|
||
return (true, bearingDeg(ox, oy, p.x, p.y), rng)
|
||
inc k
|
||
(false, 0.0, rng)
|
||
|
||
proc interceptBearing*(c: Corpus, i0, iEnd: int, ox, oy, speed: float,
|
||
cont: bool): tuple[ok: bool, bearing, range: float] =
|
||
if cont: interceptBearingCont(c, i0, iEnd, ox, oy, speed)
|
||
else: interceptBearingQuant(c, i0, iEnd, ox, oy, speed)
|
||
|
||
# ── accumulators ─────────────────────────────────────────────────────────────
|
||
|
||
proc addErr*(s: var BandStat, errDeg, range: float) =
|
||
inc s.n
|
||
let a = abs(errDeg)
|
||
s.sumAbs += a
|
||
s.sumSq += float64(errDeg) * float64(errDeg)
|
||
s.sumSigned += errDeg
|
||
if a > s.maxAbs: s.maxAbs = a
|
||
if a <= tolDeg(range): inc s.hits
|
||
|
||
proc record*(a: var ArmAcc, range: float, errDeg, predLead, reqLead: float) =
|
||
let b = bandOf(range)
|
||
addErr(a.bands[b], errDeg, range)
|
||
var s = addr a.bands[b]
|
||
s.sumPred += predLead
|
||
s.sumReq += reqLead
|
||
s.sumAbsReq += abs(reqLead)
|
||
s.sumPredReq += predLead * reqLead
|
||
s.sumPred2 += predLead * predLead
|
||
s.sumReq2 += reqLead * reqLead
|
||
inc a.evaluated
|
||
|
||
proc captureSlope*(s: BandStat): float =
|
||
## Regression of the arm's lead on the true required lead (job-95's "capture"
|
||
## statistic): 1.0 = perfect proportional response, 0.0 = no response.
|
||
if s.sumReq2 <= 1e-12: NaN else: s.sumPredReq / s.sumReq2
|
||
|
||
proc leadCorr*(s: BandStat): float =
|
||
## Pearson correlation between the arm's lead and the required lead. THIS is
|
||
## the honest measure of "is the lead informative"; a large capture slope on
|
||
## an uncorrelated lead is just amplification of noise.
|
||
let d = s.sumPred2 * s.sumReq2
|
||
if d <= 1e-12: NaN else: s.sumPredReq / sqrt(d)
|
||
|
||
proc meanAbsReq*(s: BandStat): float =
|
||
if s.n == 0: NaN else: s.sumAbsReq / float(s.n)
|
||
|
||
proc skip*(a: var ArmAcc) = inc a.skipped
|
||
|
||
proc meanAbs*(s: BandStat): float =
|
||
if s.n == 0: NaN else: s.sumAbs / float(s.n)
|
||
|
||
proc rmse*(s: BandStat): float =
|
||
if s.n == 0: NaN else: sqrt(s.sumSq / float(s.n))
|
||
|
||
proc meanSigned*(s: BandStat): float =
|
||
if s.n == 0: NaN else: s.sumSigned / float(s.n)
|
||
|
||
proc hitProxy*(s: BandStat): float =
|
||
if s.n == 0: NaN else: s.hits.float / float(s.n)
|
||
|
||
# ── corpus loading + binary cache ────────────────────────────────────────────
|
||
|
||
proc cachePathFor*(src: string): string = src & ".qcache"
|
||
|
||
proc eventsPathFor*(runPath: string): string =
|
||
## `run10.jsonl` -> `run10.events.jsonl` (note: NOT run10.jsonl.events.jsonl).
|
||
if runPath.endsWith(".jsonl"):
|
||
runPath[0 ..< runPath.len - 6] & ".events.jsonl"
|
||
else:
|
||
runPath & ".events.jsonl"
|
||
|
||
proc mtimeOf(p: string): float =
|
||
if fileExists(p): getFileInfo(p).lastWriteTime.toUnixFloat else: 0.0
|
||
|
||
proc buildCache(src, roundsPath, cachePath: string) =
|
||
## JSONL -> compact binary. Only runs when the cache is absent/stale.
|
||
var st: seq[float32]
|
||
var ticks: seq[int32]
|
||
var arenaW = 800.0
|
||
var arenaH = 600.0
|
||
for line in lines(src):
|
||
let s = line.strip()
|
||
if s.len == 0: continue
|
||
let node = parseJson(s)
|
||
if node.hasKey("meta"):
|
||
if node["meta"].hasKey("arena"):
|
||
let a = node["meta"]["arena"]
|
||
if a.hasKey("w"): arenaW = a["w"].getFloat()
|
||
if a.hasKey("h"): arenaH = a["h"].getFloat()
|
||
continue
|
||
if node.hasKey("end"): continue
|
||
st.add node["ex"].getFloat().float32
|
||
st.add node["ey"].getFloat().float32
|
||
st.add node["eh"].getFloat().float32
|
||
st.add node["es"].getFloat().float32
|
||
st.add node["ee"].getFloat().float32
|
||
st.add node["sx"].getFloat().float32
|
||
st.add node["sy"].getFloat().float32
|
||
st.add node["sh"].getFloat().float32
|
||
st.add node["ss"].getFloat().float32
|
||
st.add node["se"].getFloat().float32
|
||
ticks.add node["tick"].getInt().int32
|
||
|
||
var rStart, rCount: seq[int32]
|
||
if roundsPath.len > 0 and fileExists(roundsPath):
|
||
try:
|
||
let rj = parseFile(roundsPath)
|
||
for r in rj["rounds"]:
|
||
rStart.add r["startTick"].getInt().int32
|
||
rCount.add r["count"].getInt().int32
|
||
except CatchableError:
|
||
discard
|
||
if rStart.len == 0:
|
||
rStart.add 0'i32
|
||
rCount.add ticks.len.int32
|
||
|
||
var n32 = ticks.len.int32
|
||
var nr32 = rStart.len.int32
|
||
var magic = CacheMagic
|
||
var version = CacheVersion
|
||
var sm = mtimeOf(src)
|
||
var ss = getFileInfo(src).size.int64
|
||
var rm = mtimeOf(roundsPath)
|
||
let f = open(cachePath, fmWrite)
|
||
defer: f.close()
|
||
wb(f, addr magic, sizeof(magic))
|
||
wb(f, addr version, sizeof(version))
|
||
wb(f, addr n32, sizeof(n32))
|
||
wb(f, addr nr32, sizeof(nr32))
|
||
wb(f, addr arenaW, sizeof(arenaW))
|
||
wb(f, addr arenaH, sizeof(arenaH))
|
||
wb(f, addr sm, sizeof(sm))
|
||
wb(f, addr ss, sizeof(ss))
|
||
wb(f, addr rm, sizeof(rm))
|
||
if rStart.len > 0:
|
||
wb(f, addr rStart[0], rStart.len * sizeof(int32))
|
||
wb(f, addr rCount[0], rCount.len * sizeof(int32))
|
||
if ticks.len > 0:
|
||
wb(f, addr ticks[0], ticks.len * sizeof(int32))
|
||
wb(f, addr st[0], st.len * sizeof(float32))
|
||
|
||
proc readCache(path: string): Corpus =
|
||
let f = open(path, fmRead)
|
||
defer: f.close()
|
||
var magic, version: int32
|
||
rb(f, addr magic, sizeof(magic))
|
||
rb(f, addr version, sizeof(version))
|
||
if magic != CacheMagic or version != CacheVersion:
|
||
raise newException(IOError, "bad qcache header: " & path)
|
||
var n32, nr32: int32
|
||
rb(f, addr n32, sizeof(n32))
|
||
rb(f, addr nr32, sizeof(nr32))
|
||
result = Corpus(path: path, n: int(n32))
|
||
rb(f, addr result.arenaW, sizeof(result.arenaW))
|
||
rb(f, addr result.arenaH, sizeof(result.arenaH))
|
||
var sm: float
|
||
var ss: int64
|
||
var rm: float
|
||
rb(f, addr sm, sizeof(sm))
|
||
rb(f, addr ss, sizeof(ss))
|
||
rb(f, addr rm, sizeof(rm))
|
||
result.rStart.setLen(int(nr32))
|
||
result.rCount.setLen(int(nr32))
|
||
if nr32 > 0:
|
||
rb(f, addr result.rStart[0], int(nr32) * sizeof(int32))
|
||
rb(f, addr result.rCount[0], int(nr32) * sizeof(int32))
|
||
result.tick.setLen(result.n)
|
||
result.st.setLen(result.n * NFields)
|
||
if result.n > 0:
|
||
rb(f, addr result.tick[0], result.n * sizeof(int32))
|
||
rb(f, addr result.st[0], result.n * NFields * sizeof(float32))
|
||
|
||
proc indexMapping(c: var Corpus) =
|
||
c.contiguous = c.n > 0
|
||
c.base = if c.n > 0: int(c.tick[0]) else: 0
|
||
if c.contiguous:
|
||
for i in 0..<c.n:
|
||
if int(c.tick[i]) != c.base + i:
|
||
c.contiguous = false
|
||
break
|
||
if not c.contiguous:
|
||
c.byTick = initTable[int32, int](nextPowerOfTwo(max(16, c.n)))
|
||
for i in 0..<c.n: c.byTick[c.tick[i]] = i
|
||
|
||
proc idxOfTick*(c: Corpus, t: int32, found: var bool): int =
|
||
if c.contiguous:
|
||
let i = int(t) - c.base
|
||
if i >= 0 and i < c.n and c.tick[i] == t:
|
||
found = true
|
||
return i
|
||
found = false
|
||
return -1
|
||
if c.byTick.hasKey(t):
|
||
found = true
|
||
return c.byTick[t]
|
||
found = false
|
||
-1
|
||
|
||
proc cacheFresh(cache, src, roundsPath: string): bool =
|
||
if not fileExists(cache): return false
|
||
try:
|
||
let f = open(cache, fmRead)
|
||
var magic, version: int32
|
||
rb(f, addr magic, sizeof(magic))
|
||
rb(f, addr version, sizeof(version))
|
||
var n32, nr32: int32
|
||
rb(f, addr n32, sizeof(n32))
|
||
rb(f, addr nr32, sizeof(nr32))
|
||
var aw, ah, sm, rm: float
|
||
var ss: int64
|
||
rb(f, addr aw, sizeof(aw))
|
||
rb(f, addr ah, sizeof(ah))
|
||
rb(f, addr sm, sizeof(sm))
|
||
rb(f, addr ss, sizeof(ss))
|
||
rb(f, addr rm, sizeof(rm))
|
||
f.close()
|
||
return magic == CacheMagic and version == CacheVersion and
|
||
sm == mtimeOf(src) and ss == getFileInfo(src).size.int64 and
|
||
rm == mtimeOf(roundsPath)
|
||
except CatchableError:
|
||
false
|
||
|
||
proc loadCorpus*(src: string): Corpus =
|
||
## Load a recorded run (`runN.jsonl`); the per-round index is read from the
|
||
## sibling `runN.jsonl.rounds.json`. Builds the binary cache when stale/absent,
|
||
## then ALWAYS measures from the cache so repeated runs are byte-identical.
|
||
let cache = cachePathFor(src)
|
||
let roundsPath = src & ".rounds.json"
|
||
if not cacheFresh(cache, src, roundsPath):
|
||
buildCache(src, roundsPath, cache)
|
||
result = readCache(cache)
|
||
result.path = src
|
||
result.indexMapping()
|
||
|
||
proc discoverRuns*(root: string): seq[string] =
|
||
## All `<arm>/runN.jsonl` under `root` that have the events + rounds sidecars.
|
||
for sub in walkDir(root, relative = false):
|
||
if sub.kind != pcDir: continue
|
||
for fn in walkFiles(sub.path / "*.jsonl"):
|
||
if fn.endsWith(".events.jsonl"): continue
|
||
if fileExists(fn & ".rounds.json") and fileExists(eventsPathFor(fn)):
|
||
result.add fn
|
||
result.sort()
|
||
|
||
# ── shot-level geometry validation ───────────────────────────────────────────
|
||
#
|
||
# Scores OUR ACTUAL server-recorded fired bearings against the interception solve
|
||
# above. Owner ids in the event sidecar are not stable across runs, so each run is
|
||
# attributed independently (mirrors analyze_lead_capture_by_range.py): a fire
|
||
# event's (x, y) is the firing tank's centre and its energy drops by exactly the
|
||
# fired power one tick later.
|
||
|
||
proc parseEvents*(path: string): seq[TrEvent] =
|
||
for line in lines(path):
|
||
let s = line.strip()
|
||
if s.len == 0: continue
|
||
let node = parseJson(s)
|
||
result.add TrEvent(
|
||
round: node["round"].getInt(),
|
||
tick: node["tick"].getInt(),
|
||
kind: node["type"].getStr(),
|
||
owner: (if node.hasKey("owner"): node["owner"].getInt() else: -1),
|
||
bullet: (if node.hasKey("bullet"): node["bullet"].getInt() else: -1),
|
||
power: (if node.hasKey("power"): node["power"].getFloat() else: 0.0),
|
||
x: (if node.hasKey("x"): node["x"].getFloat() else: 0.0),
|
||
y: (if node.hasKey("y"): node["y"].getFloat() else: 0.0),
|
||
dir: (if node.hasKey("dir"): node["dir"].getFloat() else: 0.0))
|
||
|
||
proc matchEvent(c: Corpus, t: int, side: int, ev: TrEvent): bool =
|
||
## side 0 = 'e' (enemy), 1 = 's' (self). Position match + energy drop.
|
||
if t < 0 or t + 1 >= c.n: return false
|
||
let px = if side == 0: c.ex(t) else: c.sx(t)
|
||
let py = if side == 0: c.ey(t) else: c.sy(t)
|
||
if abs(px - ev.x) > 0.02 or abs(py - ev.y) > 0.02: return false
|
||
let e0 = if side == 0: c.ee(t) else: c.se(t)
|
||
let e1 = if side == 0: c.ee(t + 1) else: c.se(t + 1)
|
||
abs((e0 - e1) - ev.power) < 0.02
|
||
|
||
proc roundStartIndexOf(c: Corpus, rnd: int): int =
|
||
## Rounds hold a GLOBAL startTick; map to an array index.
|
||
var startTick: int32 = 0
|
||
for i, r in c.rStart:
|
||
if int(i) + 1 == rnd: startTick = r
|
||
if c.contiguous: return int(startTick) - c.base
|
||
var found = false
|
||
idxOfTick(c, startTick, found)
|
||
|
||
proc validateShots*(c: Corpus, events: seq[TrEvent], cont: bool): ShotStat =
|
||
## Hits must show small angular error and misses large; the separation ratio is
|
||
## the ruler's correctness certificate.
|
||
var votes = initTable[int, array[2, int]]()
|
||
for ev in events:
|
||
if ev.kind != "fire": continue
|
||
let guess = c.roundStartIndexOf(ev.round) + ev.tick
|
||
if ev.owner notin votes: votes[ev.owner] = [0, 0]
|
||
for t in (guess - 8) .. (guess + 8):
|
||
for side in 0..1:
|
||
if matchEvent(c, t, side, ev): inc votes[ev.owner][side]
|
||
var ownerSide = initTable[int, int]()
|
||
for owner, v in votes:
|
||
ownerSide[owner] = if v[1] >= v[0]: 1 else: 0
|
||
|
||
var resolution = initTable[string, string]()
|
||
for ev in events:
|
||
if ev.kind in ["hit", "hitwall", "hitbullet"]:
|
||
resolution[$ev.round & "/" & $ev.owner & "/" & $ev.bullet] = ev.kind
|
||
|
||
for ev in events:
|
||
if ev.kind != "fire": continue
|
||
if ownerSide.getOrDefault(ev.owner, -1) != 1: continue # only OUR shots
|
||
let guess = c.roundStartIndexOf(ev.round) + ev.tick
|
||
var t0 = -1
|
||
var bestD = high(int)
|
||
for t in (guess - 8) .. (guess + 8):
|
||
if matchEvent(c, t, ownerSide[ev.owner], ev):
|
||
let d = abs(t - guess)
|
||
if d < bestD:
|
||
bestD = d
|
||
t0 = t
|
||
if t0 < 0: continue
|
||
var rIdx = -1
|
||
for i, r in c.rStart:
|
||
if int(i) + 1 == ev.round: rIdx = i
|
||
if rIdx < 0: continue
|
||
let iEnd = int(c.rStart[rIdx]) - c.base + int(c.rCount[rIdx])
|
||
let speed = 20.0 - 3.0 * ev.power
|
||
let ib = interceptBearing(c, t0, iEnd, ev.x, ev.y, speed, cont)
|
||
if not ib.ok: continue
|
||
let err = wrap180(ev.dir - ib.bearing)
|
||
let kind = resolution.getOrDefault($ev.round & "/" & $ev.owner & "/" & $ev.bullet, "")
|
||
if kind == "hit":
|
||
inc result.hits
|
||
result.hitSumDeg += abs(err)
|
||
result.hitSumPx += abs(degToRad(err)) * ib.range
|
||
elif kind in ["hitwall", "hitbullet"]:
|
||
inc result.misses
|
||
result.missSumDeg += abs(err)
|
||
result.missSumPx += abs(degToRad(err)) * ib.range
|
||
|
||
proc separationDeg*(s: ShotStat): float =
|
||
let hd = if s.hits > 0: s.hitSumDeg / float(s.hits) else: NaN
|
||
let md = if s.misses > 0: s.missSumDeg / float(s.misses) else: NaN
|
||
md / hd
|
||
|
||
proc separationPx*(s: ShotStat): float =
|
||
let hp = if s.hits > 0: s.hitSumPx / float(s.hits) else: NaN
|
||
let mp = if s.misses > 0: s.missSumPx / float(s.misses) else: NaN
|
||
mp / hp
|
||
|
||
proc meanHitDeg*(s: ShotStat): float =
|
||
if s.hits > 0: s.hitSumDeg / float(s.hits) else: NaN
|
||
|
||
proc meanMissDeg*(s: ShotStat): float =
|
||
if s.misses > 0: s.missSumDeg / float(s.misses) else: NaN
|
||
|
||
proc meanHitPx*(s: ShotStat): float =
|
||
if s.hits > 0: s.hitSumPx / float(s.hits) else: NaN
|
||
|
||
proc meanMissPx*(s: ShotStat): float =
|
||
if s.misses > 0: s.missSumPx / float(s.misses) else: NaN
|
||
|
||
# ── formatting ───────────────────────────────────────────────────────────────
|
||
|
||
proc formatArmTable*(arms: seq[ArmAcc]): string =
|
||
let hdr = "arm band n meanAbs rmse signed hitProxy maxAbs"
|
||
result = hdr & "\n" & "-".repeat(hdr.len) & "\n"
|
||
for a in arms:
|
||
for b in 0..<NBands:
|
||
let s = a.bands[b]
|
||
if s.n == 0:
|
||
result.add fmt"{a.name:<16} {BandLabels[b]:<9} {0:>8} {'-':>8} {'-':>8} {'-':>8} {'-':>9} {'-':>8}" & "\n"
|
||
else:
|
||
result.add fmt"{a.name:<16} {BandLabels[b]:<9} {s.n:>8} {meanAbs(s):>8.3f} {rmse(s):>8.3f} {meanSigned(s):>8.3f} {hitProxy(s):>9.4f} {s.maxAbs:>8.2f}" & "\n"
|
||
if a.skipped > 0:
|
||
result.add fmt" ({a.name}: {a.skipped} tick-bins had no valid interception)" & "\n"
|