diff --git a/common_libs/tests/audit_virtual_guns.nim b/common_libs/tests/audit_virtual_guns.nim new file mode 100644 index 0000000..cc1860d --- /dev/null +++ b/common_libs/tests/audit_virtual_guns.nim @@ -0,0 +1,406 @@ +## AUDIT TOOL (new file, read-only w.r.t. shipped source). +## +## Measures, over the committed fixture set: +## 1. speed usage — does each gun emit 4 DIFFERENT predictions for the 4 +## PowerBins, or does it ignore bulletSpeed? +## 2. lead / accuracy — bmPoint miss distance (arrival-time aim error) and the +## bmPath "virtual" rate, so the optimism ratio is explicit. +## 3. pairwise agreement of predictions (redundancy). +## 4. conditional point hit rate by distance / speed / wall / reversal. +## 5. feedback-integrity counters (waveStarved / traceMisses / trained). +## +## Usage: +## nim c -r --path:common_libs common_libs/tests/audit_virtual_guns.nim + +import std/[os, math, strformat, tables, algorithm, strutils, json, random] +import gun_harness/offline_range +import gun_harness/gun_interface +import gun_harness/virtual_bullets as vb +import guns/head_on +import guns/linear +import guns/tsetlin +import guns/circular +import guns/guess_factor +import guns/pattern_matcher +import guns/wall_bounce +import guns/accel_predictor +import guns/stop_shot +import guns/displacement +import guns/averaged_lead +import guns/decay_gf +import guns/knn_gun +import guns/tm_selector + +const + NGun* = 14 + GunNames*: array[NGun, string] = ["HeadOn","Linear","Tsetlin","Circular", + "GuessFactor","Pattern","WallBounce","Accel","StopShot","Displace","AvgLead", + "DecayGF","KNN","TMSelect"] + NBins = len(vb.PowerBins) + +type + Box[G] = ref object + g: G + + Drivers* = object + d*: seq[GunDriver] + headOn*: Box[HeadOnGun] + linear*: Box[LinearGun] + tsetlin*: Box[TsetlinGun] + circular*: Box[CircularGun] + gf*: Box[GFGun] + pattern*: Box[PatternMatcherGun] + wall*: Box[WallBounceGun] + accel*: Box[AccelGun] + stop*: Box[StopShotGun] + disp*: Box[DisplacementGun] + avg*: Box[AveragedLeadGun] + decay*: Box[DecayGFGun] + knn*: Box[KNNGun] + tmsel*: Box[TmSelectorGun] + +proc mk[G](name: string, g: Box[G]): GunDriver = + result.name = name + result.predictCb = proc(s: WorldState, sp: float): GunPrediction = g.g.predict(s, sp) + result.resultCb = proc(e: FeedbackEvent) = g.g.onResult(e) + when compiles(g.g.isWarmedUp()): + result.readyCb = proc(): bool = g.g.isWarmedUp() + else: + result.readyCb = nil + +proc newBox[G](v: G): Box[G] = + new(result); result.g = v + +proc initDrivers*(seed = 1): Drivers = + result.headOn = newBox(HeadOnGun()) + result.linear = newBox(LinearGun()) + result.tsetlin = newBox(initTsetlinGun()) + result.circular = newBox(CircularGun()) + result.gf = newBox(initGFGun()) + result.pattern = newBox(PatternMatcherGun()) + result.wall = newBox(initWallBounceGun()) + result.accel = newBox(initAccelGun()) + result.stop = newBox(initStopShotGun()) + result.disp = newBox(initDisplacementGun()) + result.avg = newBox(initAveragedLeadGun()) + result.decay = newBox(initDecayGFGun()) + result.knn = newBox(initKNNGun()) + result.tmsel = newBox(initTmSelectorGun()) + if seed >= 0: randomize(seed) + result.d = @[ + mk("HeadOn", result.headOn), mk("Linear", result.linear), + mk("Tsetlin", result.tsetlin), mk("Circular", result.circular), + mk("GuessFactor", result.gf), mk("Pattern", result.pattern), + mk("WallBounce", result.wall), mk("Accel", result.accel), + mk("StopShot", result.stop), mk("Displace", result.disp), + mk("AvgLead", result.avg), mk("DecayGF", result.decay), + mk("KNN", result.knn), mk("TMSelect", result.tmsel), + ] + +# ── accumulators ────────────────────────────────────────────────────────────── + +type + BinAcc = object + count*, hits*: int + sumMiss*: float + within20*, within50*: int + sumPairDist*: float + + ContextAcc = object + count*, hits*: int + + GunAcc = object + bins: array[NBins, BinAcc] + # point-metric results (arrival-time aim accuracy) + ptCount*, ptHits*: int + ptSumMiss*: float + # prediction-diversity + identicalTicks*, predTicks*: int + sumSpread*: float ## mean over ticks of max pairwise pred distance + # conditional point hit rate (pooled bins), marginal buckets + dist*: array[4, ContextAcc] + speed*: array[4, ContextAcc] + wall*: array[2, ContextAcc] + rev*: array[2, ContextAcc] + # virtual-path rate + vpCount*, vpHits*: int + + Audit* = object + guns: array[NGun, GunAcc] + pairCount: array[NGun, array[NGun, int]] + pairWithin20: array[NGun, array[NGun, int]] + pairWithin50: array[NGun, array[NGun, int]] + pairSumDist: array[NGun, array[NGun, float]] + ticks*: int + fixtures*: int + +proc ctxDist(s: WorldState): int = + let d = hypot(s.enemyX - s.selfX, s.enemyY - s.selfY) + if d < 200.0: 0 elif d < 400.0: 1 elif d < 600.0: 2 else: 3 + +proc ctxSpeed(s: WorldState): int = + let sp = abs(s.enemySpeed) + if sp < 1.0: 0 elif sp < 4.0: 1 elif sp < 7.5: 2 else: 3 + +proc ctxWall(s: WorldState): int = + let m = min(min(s.enemyX, s.arenaWidth - s.enemyX), + min(s.enemyY, s.arenaHeight - s.enemyY)) + if m < 60.0: 1 else: 0 + +proc feedPredictions(a: var Audit, gi: int, preds: array[NBins, GunPrediction]) = + ## Speed usage: are the 4 bin predictions distinct? + var maxD = 0.0 + for i in 0.. maxD: maxD = d + inc a.guns[gi].predTicks + a.guns[gi].sumSpread += maxD + if maxD < 0.01: inc a.guns[gi].identicalTicks + +proc feedPairs(a: var Audit, preds: array[NGun, array[NBins, GunPrediction]]) = + for i in 0..= NGun: return + drivers.d[gunId].resultCb(e) # feed the owning gun its own feedback + if metric == bmPoint: + let ga = addr a.guns[gunId] + inc ga.ptCount + ga.ptSumMiss += e.missDistance + if e.hit: inc ga.ptHits + # context buckets (from fire-time state) + let fsIdx = byTick.getOrDefault(e.fireTick, si) + let fs = fx.states[fsIdx] + inc ga.dist[ctxDist(fs)].count + inc ga.speed[ctxSpeed(fs)].count + inc ga.wall[ctxWall(fs)].count + let r = if fsIdx > 0: + let ps = fx.states[fsIdx-1].enemySpeed + ps != 0.0 and fs.enemySpeed != 0.0 and (ps > 0.0) != (fs.enemySpeed > 0.0) + else: false + inc ga.rev[ord(r)].count + if e.hit: + inc ga.dist[ctxDist(fs)].hits + inc ga.speed[ctxSpeed(fs)].hits + inc ga.wall[ctxWall(fs)].hits + inc ga.rev[ord(r)].hits + else: + inc a.guns[gunId].vpCount + if e.hit: inc a.guns[gunId].vpHits + discard binIdx) + +proc addGunAcc(dst: var GunAcc, src: GunAcc) = + for b in 0.. MaxTicks: + fx.states.setLen(MaxTicks) + if fx.lastSeen.len > MaxTicks: fx.lastSeen.setLen(MaxTicks) + let dPoint = initDrivers(seed = 1) + var a = new(Audit) + auditReplay(fx, dPoint, bmPoint, a, capPreds = false) + for gi in 0.. 0.0: vp / pp else: 0.0 + let ident = rate(g.identicalTicks, g.predTicks) + let spread = if g.predTicks > 0: g.sumSpread / g.predTicks.float else: 0.0 + let ptMiss = if accPoint.guns[gi].ptCount > 0: + accPoint.guns[gi].ptSumMiss / accPoint.guns[gi].ptCount.float else: 0.0 + echo fmt"{GunNames[gi]:<12} {vp:6.1f} {pp:6.1f} {ratio:5.2f} {ident:6.1f} {spread:6.1f} {ptMiss:6.1f}" + echo "(vpath% = shipped bmPath virtual rate; point% = bmPoint arrival-accuracy rate;" + echo " ident% = ticks whose 4 power-bin predictions are identical; spread = mean max" + echo " pairwise bin distance px; ptMiss = mean bmPoint miss distance px)" + + echo "" + echo "=================== TOP REDUNDANT PAIRS (pred within 20px / 50px) ===================" + type Pair = tuple[pct20, pct50, meanD: float, i, j: int] + var pairs: seq[Pair] + for i in 0..8}" + echo hdr + for i in 0..=7.5 wall open rev fwd" + for gi in 0..