From ab86c0481f877c58dfca7b0d1fa2a0a1b58cb75f Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Tue, 22 Sep 2026 00:44:19 +0200 Subject: [PATCH] gun audit: the virtual system is sound; the rack is redundant, not broken Audited all 14 guns offline over the committed DrussGT fixtures (~150k resolved bullets/gun) plus 123 rounds of live gun_stats. Prompted by a GUI observation that selected guns "fire dozens of pixels away" and a suspicion of reverse selection. MY HYPOTHESIS WAS WRONG. I expected guns to be ignoring `bulletSpeed`, which would make their 4 power bins identical and the per-bin fitness pure noise. MEASURED: only `HeadOn` is speed-blind (100% identical bins) and that is its correct definition. Every other gun emits 91-94% DISTINCT per-bin predictions (mean intra-tick bin spread 62-103px). The earlier "tick-only cache collapsed all bins onto bin 0" fix is complete across the whole rack. LEAD/SIGN/UNITS ARE CORRECT: replaying synthetic ground truth, all 14 guns score 100% on a stationary target (which also proves predictions are ABSOLUTE - a relative or angle return would score 0), ~100% on constant-velocity for every leaded gun, Circular 100% / Accel 99.8% on a 3deg/tick circle, WallBounce 98.2% on a bounce. No missing lead, no sign inversion. Resolution is right (BotRadius 18, hit credited to the owning gun). FEEDBACK IS INTACT: offline pushes=151260/starved=0, Tsetlin trained=149205/ traceMisses=0; live `vStarved=0` and `vDropped=0` across all 123 rounds. THE "43% FLAT / 4x OPTIMISTIC" EVIDENCE I CITED IS NOT REPRODUCIBLE on current code/data. Live virtual/real ratios against DrussGT are 0.8-2.0 for most guns (Linear 10.9 virt / 13.4 real; KNN 8.1/7.1; DecayGF 10.5/9.8). The 43%-flat session matches an older config or a weak opponent (SittingDuck), not DrussGT. `bmPath` IS 2.3-3.6x `bmPoint` - but by design and documented: it asks "does the ray eventually sweep the target's path", a deliberately generous relative signal. So the flat tie is a RANKING artefact: many guns share the same base forecast and, with a near-zero learned correction, collapse onto the same ray; RelTieMargin=0.20 then treats the top ~half of the rack as tied. DUTY AND OVERLAP (>=50% of ticks within 20px = redundant): Tsetlin ~ StopShot 87% -> duplicate pair DecayGF ~ GuessFactor 92% -> duplicate pair Accel ~ Circular 65% -> partial duplicate WallBounce~ Linear 58% AvgLead = the MEAN of Linear+Circular+WallBounce (constructed redundancy) Displace worst point% (6.6) AND worst real% (2.4); wins no bucket TMSelect DEAD - never spawned (EnableTmSelector=false), 0 shots in every log Pattern the ONLY gun competitive in every distance/speed bucket HeadOn/Linear/Tsetlin/StopShot are identical copies of each other on a real surfer (v<1 ~50.8%, everything else ~2%) RECOMMENDED LEAN RACK (8): HeadOn, Linear, Circular, Accel, Pattern, GuessFactor, KNN, WallBounce. DROP (6): TMSelect (dead), AvgLead (constructed mean), Displace (worst), DecayGF (92% GF), StopShot (87% Tsetlin), Tsetlin (the repo's own sweep already showed it learns nothing on DrussGT). HONEST HEADLINE: pruning is NOT expected to raise hit rate - an earlier 15-paired-run experiment found it neutral-to-negative (p=0.57/0.21). The mechanism by which it could help is a SELECTOR effect (shrinking the tied band), not a gun effect, and that is UNVERIFIED until A/B'd. The virtual system and the rack are basically sound; the lever that matters most is the selector's metric/tie-band, not deleting guns. DESIGN SMELL FOUND (INFERRED, not measured): GF/DecayGF/KNN `onResult` pops the OLDEST wave, but under bmPath bullets leave the arena in non-FIFO order, so a resolution can be paired with a neighbouring tick's wave. starved=0 does not rule this out. Candidate fix: key waves by fireTick, as Tsetlin/TMSelect do. Adds common_libs/tests/audit_virtual_guns.nim (offline, no shipped file touched). --- common_libs/tests/audit_virtual_guns.nim | 406 +++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 common_libs/tests/audit_virtual_guns.nim diff --git a/common_libs/tests/audit_virtual_guns.nim b/common_libs/tests/audit_virtual_guns.nim new file mode 100644 index 0000000..cc1860d --- /dev/null +++ b/common_libs/tests/audit_virtual_guns.nim @@ -0,0 +1,406 @@ +## AUDIT TOOL (new file, read-only w.r.t. shipped source). +## +## Measures, over the committed fixture set: +## 1. speed usage — does each gun emit 4 DIFFERENT predictions for the 4 +## PowerBins, or does it ignore bulletSpeed? +## 2. lead / accuracy — bmPoint miss distance (arrival-time aim error) and the +## bmPath "virtual" rate, so the optimism ratio is explicit. +## 3. pairwise agreement of predictions (redundancy). +## 4. conditional point hit rate by distance / speed / wall / reversal. +## 5. feedback-integrity counters (waveStarved / traceMisses / trained). +## +## Usage: +## nim c -r --path:common_libs common_libs/tests/audit_virtual_guns.nim + +import std/[os, math, strformat, tables, algorithm, strutils, json, random] +import gun_harness/offline_range +import gun_harness/gun_interface +import gun_harness/virtual_bullets as vb +import guns/head_on +import guns/linear +import guns/tsetlin +import guns/circular +import guns/guess_factor +import guns/pattern_matcher +import guns/wall_bounce +import guns/accel_predictor +import guns/stop_shot +import guns/displacement +import guns/averaged_lead +import guns/decay_gf +import guns/knn_gun +import guns/tm_selector + +const + NGun* = 14 + GunNames*: array[NGun, string] = ["HeadOn","Linear","Tsetlin","Circular", + "GuessFactor","Pattern","WallBounce","Accel","StopShot","Displace","AvgLead", + "DecayGF","KNN","TMSelect"] + NBins = len(vb.PowerBins) + +type + Box[G] = ref object + g: G + + Drivers* = object + d*: seq[GunDriver] + headOn*: Box[HeadOnGun] + linear*: Box[LinearGun] + tsetlin*: Box[TsetlinGun] + circular*: Box[CircularGun] + gf*: Box[GFGun] + pattern*: Box[PatternMatcherGun] + wall*: Box[WallBounceGun] + accel*: Box[AccelGun] + stop*: Box[StopShotGun] + disp*: Box[DisplacementGun] + avg*: Box[AveragedLeadGun] + decay*: Box[DecayGFGun] + knn*: Box[KNNGun] + tmsel*: Box[TmSelectorGun] + +proc mk[G](name: string, g: Box[G]): GunDriver = + result.name = name + result.predictCb = proc(s: WorldState, sp: float): GunPrediction = g.g.predict(s, sp) + result.resultCb = proc(e: FeedbackEvent) = g.g.onResult(e) + when compiles(g.g.isWarmedUp()): + result.readyCb = proc(): bool = g.g.isWarmedUp() + else: + result.readyCb = nil + +proc newBox[G](v: G): Box[G] = + new(result); result.g = v + +proc initDrivers*(seed = 1): Drivers = + result.headOn = newBox(HeadOnGun()) + result.linear = newBox(LinearGun()) + result.tsetlin = newBox(initTsetlinGun()) + result.circular = newBox(CircularGun()) + result.gf = newBox(initGFGun()) + result.pattern = newBox(PatternMatcherGun()) + result.wall = newBox(initWallBounceGun()) + result.accel = newBox(initAccelGun()) + result.stop = newBox(initStopShotGun()) + result.disp = newBox(initDisplacementGun()) + result.avg = newBox(initAveragedLeadGun()) + result.decay = newBox(initDecayGFGun()) + result.knn = newBox(initKNNGun()) + result.tmsel = newBox(initTmSelectorGun()) + if seed >= 0: randomize(seed) + result.d = @[ + mk("HeadOn", result.headOn), mk("Linear", result.linear), + mk("Tsetlin", result.tsetlin), mk("Circular", result.circular), + mk("GuessFactor", result.gf), mk("Pattern", result.pattern), + mk("WallBounce", result.wall), mk("Accel", result.accel), + mk("StopShot", result.stop), mk("Displace", result.disp), + mk("AvgLead", result.avg), mk("DecayGF", result.decay), + mk("KNN", result.knn), mk("TMSelect", result.tmsel), + ] + +# ── accumulators ────────────────────────────────────────────────────────────── + +type + BinAcc = object + count*, hits*: int + sumMiss*: float + within20*, within50*: int + sumPairDist*: float + + ContextAcc = object + count*, hits*: int + + GunAcc = object + bins: array[NBins, BinAcc] + # point-metric results (arrival-time aim accuracy) + ptCount*, ptHits*: int + ptSumMiss*: float + # prediction-diversity + identicalTicks*, predTicks*: int + sumSpread*: float ## mean over ticks of max pairwise pred distance + # conditional point hit rate (pooled bins), marginal buckets + dist*: array[4, ContextAcc] + speed*: array[4, ContextAcc] + wall*: array[2, ContextAcc] + rev*: array[2, ContextAcc] + # virtual-path rate + vpCount*, vpHits*: int + + Audit* = object + guns: array[NGun, GunAcc] + pairCount: array[NGun, array[NGun, int]] + pairWithin20: array[NGun, array[NGun, int]] + pairWithin50: array[NGun, array[NGun, int]] + pairSumDist: array[NGun, array[NGun, float]] + ticks*: int + fixtures*: int + +proc ctxDist(s: WorldState): int = + let d = hypot(s.enemyX - s.selfX, s.enemyY - s.selfY) + if d < 200.0: 0 elif d < 400.0: 1 elif d < 600.0: 2 else: 3 + +proc ctxSpeed(s: WorldState): int = + let sp = abs(s.enemySpeed) + if sp < 1.0: 0 elif sp < 4.0: 1 elif sp < 7.5: 2 else: 3 + +proc ctxWall(s: WorldState): int = + let m = min(min(s.enemyX, s.arenaWidth - s.enemyX), + min(s.enemyY, s.arenaHeight - s.enemyY)) + if m < 60.0: 1 else: 0 + +proc feedPredictions(a: var Audit, gi: int, preds: array[NBins, GunPrediction]) = + ## Speed usage: are the 4 bin predictions distinct? + var maxD = 0.0 + for i in 0.. maxD: maxD = d + inc a.guns[gi].predTicks + a.guns[gi].sumSpread += maxD + if maxD < 0.01: inc a.guns[gi].identicalTicks + +proc feedPairs(a: var Audit, preds: array[NGun, array[NBins, GunPrediction]]) = + for i in 0..= NGun: return + drivers.d[gunId].resultCb(e) # feed the owning gun its own feedback + if metric == bmPoint: + let ga = addr a.guns[gunId] + inc ga.ptCount + ga.ptSumMiss += e.missDistance + if e.hit: inc ga.ptHits + # context buckets (from fire-time state) + let fsIdx = byTick.getOrDefault(e.fireTick, si) + let fs = fx.states[fsIdx] + inc ga.dist[ctxDist(fs)].count + inc ga.speed[ctxSpeed(fs)].count + inc ga.wall[ctxWall(fs)].count + let r = if fsIdx > 0: + let ps = fx.states[fsIdx-1].enemySpeed + ps != 0.0 and fs.enemySpeed != 0.0 and (ps > 0.0) != (fs.enemySpeed > 0.0) + else: false + inc ga.rev[ord(r)].count + if e.hit: + inc ga.dist[ctxDist(fs)].hits + inc ga.speed[ctxSpeed(fs)].hits + inc ga.wall[ctxWall(fs)].hits + inc ga.rev[ord(r)].hits + else: + inc a.guns[gunId].vpCount + if e.hit: inc a.guns[gunId].vpHits + discard binIdx) + +proc addGunAcc(dst: var GunAcc, src: GunAcc) = + for b in 0.. MaxTicks: + fx.states.setLen(MaxTicks) + if fx.lastSeen.len > MaxTicks: fx.lastSeen.setLen(MaxTicks) + let dPoint = initDrivers(seed = 1) + var a = new(Audit) + auditReplay(fx, dPoint, bmPoint, a, capPreds = false) + for gi in 0.. 0.0: vp / pp else: 0.0 + let ident = rate(g.identicalTicks, g.predTicks) + let spread = if g.predTicks > 0: g.sumSpread / g.predTicks.float else: 0.0 + let ptMiss = if accPoint.guns[gi].ptCount > 0: + accPoint.guns[gi].ptSumMiss / accPoint.guns[gi].ptCount.float else: 0.0 + echo fmt"{GunNames[gi]:<12} {vp:6.1f} {pp:6.1f} {ratio:5.2f} {ident:6.1f} {spread:6.1f} {ptMiss:6.1f}" + echo "(vpath% = shipped bmPath virtual rate; point% = bmPoint arrival-accuracy rate;" + echo " ident% = ticks whose 4 power-bin predictions are identical; spread = mean max" + echo " pairwise bin distance px; ptMiss = mean bmPoint miss distance px)" + + echo "" + echo "=================== TOP REDUNDANT PAIRS (pred within 20px / 50px) ===================" + type Pair = tuple[pct20, pct50, meanD: float, i, j: int] + var pairs: seq[Pair] + for i in 0..8}" + echo hdr + for i in 0..=7.5 wall open rev fwd" + for gi in 0..