## Offline selector diagnostics — WHY does the selector pick what it picks? ## ## Replays real recorded DrussGT movement (tools/fixtures/drussgt_vs_*.jsonl) ## through the 13 ModularBot guns and, on every tick, runs the ACTUAL selector ## core (`chooseFromFit`) under BOTH threshold models, recording: ## * bestRate (the value the floor compares against) ## * floorRate / floorFired ## * tiedCount (guns within the tie band) ## * the selected gun (random tie-break, seeded) ## ## The replay itself is selection-independent, so one pass yields both models. ## Run under both metrics: ## GUN_SELECTOR_MODE=relative nim c -d:release -r \ ## common_libs/tests/analyze_selector.nim [fixture.jsonl ...] ## ## This is a MEASUREMENT tool, not a test; it always exits 0. import std/[os, math, strformat, random, algorithm] import gun_harness/offline_range import gun_harness/virtual_bullets import range_guns const repoRoot = currentSourcePath().parentDir.parentDir.parentDir fixturesDir = repoRoot / "tools" / "fixtures" type SelectorStats = object ticks: int floorFires: int floorSum: float bestRates: seq[float] tiedCounts: seq[int] selected: array[13, int] badInvariant: int ## floor fired but returned a non-zero gun (bug detector) nonFloor: int nonFloorTied: int nonFloorMulti: int proc median(x: seq[float]): float = if x.len == 0: return 0.0 var y = x y.sort() y[y.len div 2] proc pct(x: seq[int], p: float): int = if x.len == 0: return 0 var y = x y.sort() y[int(p * float(max(0, y.len - 1)))] proc analyze(fx: Fixture, metric: BulletMetric, mode: SelectorMode, seed: int): SelectorStats = randomize(seed) let drivers = buildAllGunDrivers(seed = seed) var s: SelectorStats discard replayFixture(fx, drivers, metric = metric, liveActual = (fx.meta.source == "live"), tickCb = proc(t: ptr VirtualTracker) = var d: SelectorDiag let g = chooseFromFit(t[].fitnessFor(fx.enemyId), addr d, mode = mode, referenceRate = t[].peakRateRef) inc s.ticks s.bestRates.add d.bestRate s.floorSum += max(0.0, d.floorRate) s.tiedCounts.add d.tiedCount if d.floorFired: inc s.floorFires if g != 0: inc s.badInvariant else: inc s.nonFloor s.nonFloorTied += d.tiedCount if d.tiedCount > 1: inc s.nonFloorMulti if g >= 0 and g < 13: inc s.selected[g] ) s proc report(label: string, s: SelectorStats) = echo fmt" [{label}]" if s.ticks == 0: echo " (no ticks)"; return echo fmt" ticks : {s.ticks}" echo fmt" floor fired : {s.floorFires} ({100.0*float(s.floorFires)/float(s.ticks):.1f}%) mean floorRate {100.0*s.floorSum/float(s.ticks):.2f}%" echo fmt" floor-but-not-gun0 : {s.badInvariant}" let lo = if s.bestRates.len > 0: min(s.bestRates)*100.0 else: 0.0 let hi = if s.bestRates.len > 0: max(s.bestRates)*100.0 else: 0.0 echo fmt" bestRate min/med/max : {lo:.2f}% / {median(s.bestRates)*100.0:.2f}% / {hi:.2f}%" var tiedMean = 0.0 for c in s.tiedCounts: tiedMean += c.float tiedMean /= float(s.ticks) echo fmt" tiedCount mean/med/p90 : {tiedMean:.2f} / {pct(s.tiedCounts, 0.50)} / {pct(s.tiedCounts, 0.90)}" if s.nonFloor > 0: echo fmt" non-floor tie mean/multi%: {float(s.nonFloorTied)/float(s.nonFloor):.2f} / {100.0*float(s.nonFloorMulti)/float(s.nonFloor):.1f}%" echo fmt" selected HeadOn share : {100.0*float(s.selected[0])/float(s.ticks):.1f}%" var order: seq[int] for i in 0..<13: order.add i order.sort(proc(a, b: int): int = cmp(s.selected[b], s.selected[a])) var top = "" for k in 0..<6: let i = order[k] if s.selected[i] == 0: continue top.add fmt"{i} {100.0*float(s.selected[i])/float(s.ticks):.1f}% " echo fmt" top guns : {top}" proc main() = var paths: seq[string] for i in 1..paramCount(): paths.add paramStr(i) if paths.len == 0: for f in walkFiles(fixturesDir / "drussgt_vs_*.jsonl"): paths.add f paths.sort() if paths.len == 0: echo "no fixtures found"; quit(0) for p in paths: let fx = loadFixture(p) echo "================================================================" echo fmt"fixture: {extractFilename(p)} ticks={fx.states.len} source={fx.meta.source}" echo "================================================================" for metric in [bmPoint, bmPath]: let absStats = analyze(fx, metric, smAbsolute, seed = 12345) let relStats = analyze(fx, metric, smRelative, seed = 12345) echo fmt"-- {metric} --" report("absolute (legacy)", absStats) report("relative (scale-aware)", relStats) echo "" when isMainModule: main()