diff --git a/common_libs/gun_harness/offline_range.nim b/common_libs/gun_harness/offline_range.nim index eda37f0..3367295 100644 --- a/common_libs/gun_harness/offline_range.nim +++ b/common_libs/gun_harness/offline_range.nim @@ -219,7 +219,8 @@ proc reportFor(tracker: VirtualTracker, drivers: seq[GunDriver], proc replayFixture*(fx: Fixture, drivers: seq[GunDriver], targetId = -1, liveActual = false, - metric = ActiveMetric): seq[GunReport] = + metric = ActiveMetric, + tickCb: proc(t: ptr VirtualTracker) {.closure.} = nil): seq[GunReport] = ## Drive a fresh `VirtualTracker` over the whole fixture, one tick at a time, ## in the same order the live loop uses: ## 1. predict(state, bulletSpeed(PowerBins[i])) for i = 0..3, per gun @@ -271,6 +272,8 @@ proc replayFixture*(fx: Fixture, drivers: seq[GunDriver], tracker.tickBullets(state, enemyPositions, proc(gunId: GunId, binIdx: int, e: FeedbackEvent) = dref[gunId].resultCb(e)) + if tickCb != nil: + tickCb(addr tracker) result = reportFor(tracker, drivers, tid) diff --git a/common_libs/gun_harness/virtual_bullets.nim b/common_libs/gun_harness/virtual_bullets.nim index d734e04..14015d0 100644 --- a/common_libs/gun_harness/virtual_bullets.nim +++ b/common_libs/gun_harness/virtual_bullets.nim @@ -20,8 +20,13 @@ const ## is ~120 bytes, so this array costs ~960 KiB. MinHitRate* = 0.40 ## 40% threshold for acceptable power selection MinObsBeforeCompete* = 50 ## min observations before a gun×bin enters competition - TieMargin* = 0.02 ## guns within this hit-rate margin of best are tied - MinHitRateFloor* = 0.10 ## if best gun < this, fall back to gun 0 (HeadOn) + TieMargin* = 0.02 ## ABSOLUTE mode: guns within this hit-rate margin of best are tied + MinHitRateFloor* = 0.10 ## ABSOLUTE mode: if best gun < this, fall back to gun 0 (HeadOn) + RelTieMargin* = 0.20 ## RELATIVE mode: tied if rate >= bestRate*(1-this). Dimensionless + ## fraction of the best rate, so it scales with the metric. + FloorPeakFrac* = 0.25 ## RELATIVE mode: floor fires if bestRate < this*peakRateRef. + ## Dimensionless: only if the field collapsed vs its own recent best. + SelectorWindow* = 256 ## ticks of per-tick bestRate kept for the RELATIVE floor reference MetricEnvVar* = "GUN_VBULLET_METRIC" ## RUNTIME switch selecting how a virtual bullet is scored. Read once per @@ -33,34 +38,62 @@ type GunId* = int ## index into the guns seq BulletMetric* = enum - bmPoint ## DEFAULT. A bullet is scored at the single point it reaches at - ## the fire-time aim distance. HIT iff that point is within - ## BotRadius of the target on that tick. Measures prediction - ## accuracy (does the bullet arrive at the predicted point at the - ## right time). - bmPath ## The bullet flies along its straight ray until it leaves the - ## arena. Each tick the swept segment (previous -> new position) + bmPoint ## A bullet is scored at the single point it reaches at the + ## fire-time aim distance. HIT iff that point is within BotRadius of + ## the target on that tick. Measures prediction accuracy (does the + ## bullet arrive at the predicted point at the right time). + bmPath ## DEFAULT. The bullet flies along its straight ray until it leaves + ## the arena. Each tick the swept segment (previous -> new position) ## is tested against the target's radius; HIT iff ANY segment came ## within BotRadius. Measures hypothetical hit chance against the - ## target's real path. + ## target's real path. Chosen by the DrussGT A/B: 7.2-7.6% real hit + ## rate vs 3.8% for point (p<0.0001). + +const DefaultMetric* = bmPath + ## Shipped virtual-bullet scoring model. `GUN_VBULLET_METRIC` overrides it at + ## runtime; an unset OR empty value means this default. proc parseMetric*(value: string): BulletMetric = ## Parse a `GUN_VBULLET_METRIC` value. Empty / unknown values fall back to - ## the shipped `point` model and emit a one-line warning on stderr, so a + ## the shipped `DefaultMetric` and emit a one-line warning on stderr, so a ## typo can never silently change the metric and a bad value can never take ## the bot down. case value.strip().toLowerAscii() - of "", "point", "points", "bmpoint": bmPoint + of "", "default": DefaultMetric + of "point", "points", "bmpoint": bmPoint of "path", "paths", "bmpath": bmPath else: stderr.writeLine("[gun_harness] unknown " & MetricEnvVar & "='" & value & - "'; falling back to 'point' (valid: point|path)") - bmPoint + "'; falling back to '" & $DefaultMetric & "' (valid: point|path)") + DefaultMetric -let ActiveMetric* = parseMetric(getEnv(MetricEnvVar, "point")) +let ActiveMetric* = parseMetric(getEnv(MetricEnvVar, "")) ## The metric every tracker uses unless a caller overrides it explicitly in ## `initTracker`. Frozen at process start from the environment. +const SelectorModeEnvVar* = "GUN_SELECTOR_MODE" + ## RUNTIME switch selecting the selection-threshold model. Read once per + ## process, so one binary can A/B both (§ virtual_bullets). + +type + SelectorMode* = enum + smAbsolute ## legacy: fixed 2pp tie band + 10% absolute floor. Correct only + ## if the virtual hit-rate scale happens to land near 10%. + smRelative ## scale-aware: tie band is a fraction of the best rate; the floor + ## fires only when the field has collapsed vs its own recent peak. + +proc parseSelectorMode*(value: string): SelectorMode = + ## Empty / unknown values fall back to the shipped `relative` model and warn. + case value.strip().toLowerAscii() + of "", "relative", "rel": smRelative + of "absolute", "abs", "legacy": smAbsolute + else: + stderr.writeLine("[gun_harness] unknown " & SelectorModeEnvVar & "='" & value & + "'; falling back to 'relative' (valid: absolute|relative)") + smRelative + +let ActiveSelectorMode* = parseSelectorMode(getEnv(SelectorModeEnvVar, "relative")) + type VirtualBullet* = object gunId*: GunId @@ -89,6 +122,15 @@ type GunFitness* = object bins*: array[len(PowerBins), FitnessWindow] + SelectorDiag* = object + ## Optional observability for `chooseFromFit`/`bestGun`. Never needed by the + ## bot; lets the offline range report WHY a gun was selected (floor vs tie). + bestRate*: float ## max hit rate over eligible guns (the floor comparison value) + floorFired*: bool ## bestRate below the active floor -> returned gun 0 + tiedCount*: int ## eligible guns within the tie band of bestRate (0 if floor fired) + anyQualifies*: bool ## at least one gun reached MinObsBeforeCompete + floorRate*: float ## the floor actually applied this tick + VirtualTracker* = object bullets*: array[MaxBullets, VirtualBullet] head*: int ## ring buffer head @@ -96,6 +138,11 @@ type metric*: BulletMetric ## scoring model (defaults to ActiveMetric) fitness*: Table[int, seq[GunFitness]] ## keyed by enemy bot ID, indexed by GunId droppedBullets*: int ## unresolved bullets clobbered by the ring buffer (should stay 0) + # RELATIVE-mode floor reference: per-tick bestRate history + its running max. + rateHist*: array[SelectorWindow, float] + rateHistHead*: int + rateHistCount*: int + peakRateRef*: float ## max bestRate in rateHist; 0.0 = not enough history yet proc initTracker*(numGuns: int, metric = ActiveMetric): VirtualTracker = ## `metric` defaults to the process-wide `GUN_VBULLET_METRIC` switch; pass it @@ -167,6 +214,57 @@ proc distPointToSegment*(px, py, ax, ay, bx, by: float): float = s = clamp(((px - ax)*abx + (py - ay)*aby) / abLen2, 0.0, 1.0) hypot(px - (ax + s*abx), py - (ay + s*aby)) +# ── rate helpers (shared by the selector and the reference tracker) ────────── + +proc gunEligible*(fit: GunFitness, requireMin: bool): bool = + ## True if a gun has >= MinObsBeforeCompete samples in at least one bin (or + ## when `requireMin` is false, every gun is eligible). + if not requireMin: return true + for binIdx in 0..= MinObsBeforeCompete: return true + false + +proc gunRate*(fit: GunFitness, pooled: bool): float = + ## A gun's hit rate. `pooled` sums hits/shots across all power bins (more + ## samples, immune to one lucky bin); otherwise the max single-bin rate. + if pooled: + var h, n = 0 + for binIdx in 0.. 0: h.float / n.float else: 0.0 + else: + result = 0.0 + for binIdx in 0..= MinObsBeforeCompete samples count: under-sampled bins + ## produce 100%/-looking spikes that would inflate the floor reference and + ## force HeadOn for the whole window. Zero when nothing is warmed up yet. + result = 0.0 + for _, perEnemy in t.fitness: + for gunId in 0.. peak: peak = t.rateHist[i] + t.peakRateRef = peak + proc tickBullets*(t: var VirtualTracker, state: WorldState, enemies: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]], onResolved: proc(gunId: GunId, binIdx: int, e: FeedbackEvent)) = @@ -291,6 +389,9 @@ proc tickBullets*(t: var VirtualTracker, state: WorldState, onResolved(b.gunId, b.powerBin, fe) b.active = false + if ActiveSelectorMode == smRelative: + noteBestRate(t) + proc fitnessFor*(t: VirtualTracker, targetId: int): seq[GunFitness] = ## Returns fitness seq for targetId, or merges all enemies as fallback. ## @@ -339,39 +440,24 @@ proc bestPower*(t: VirtualTracker, gunId: GunId, targetId: int = -1): (int, floa if rate >= MinHitRate or fit[gunId].bins[binIdx].count == 0: return (binIdx, PowerBins[binIdx]) -proc bestGun*(t: VirtualTracker, targetId: int = -1): GunId = - ## Pick gun with highest hit rate across all power bins. - ## Guns with fewer than MinObsBeforeCompete observations are skipped - ## unless every gun is below threshold (then fall back to best of all). - ## Uses per-enemy fitness when targetId >= 0 and data exists; else aggregate. - ## Ties (within TieMargin) are broken randomly to avoid index-0 bias. - ## ponytail: O(n*bins), fine for small gun counts - let fit = t.fitnessFor(targetId) - proc bestAmong(fit: seq[GunFitness], requireMin: bool): GunId = - var bestRate = -1.0 - for gunId in 0.. bestRate: bestRate = r - # Floor check: if nothing hits well enough, HeadOn is the safe default - if bestRate < MinHitRateFloor: return 0 - var tied: seq[GunId] - for gunId in 0..= bestRate - TieMargin: - tied.add(gunId) - break - if tied.len == 0: return 0 - return tied[rand(tied.len - 1)] +proc chooseFromFit*(fit: seq[GunFitness], diag: ptr SelectorDiag = nil, + mode: SelectorMode = smAbsolute, + referenceRate = -1.0): GunId = + ## Core gun ranking over an already-resolved fitness seq. Split out from + ## `bestGun` so the offline range can rank without copying a VirtualTracker, + ## and so callers can request `diag` for the selection internals. + ## + ## Guns with fewer than MinObsBeforeCompete observations are skipped unless + ## every gun is below threshold (then fall back to best of all). + ## + ## `mode` chooses the threshold model: + ## smAbsolute — legacy fixed TieMargin / MinHitRateFloor. + ## smRelative — tie band = bestRate*RelTieMargin; floor = FloorPeakFrac + ## * `referenceRate` (the recent field-best rate). Pooled over + ## power bins, since one lucky bin is a poor ranker. + ## `referenceRate` <= 0 disables the RELATIVE floor (no history yet). + ## Ties (within the band) are broken randomly to avoid index-0 bias. + let pooled = mode == smRelative var anyQualifies = false for gunId in 0.. 0.0: FloorPeakFrac * referenceRate + else: 0.0 + if diag != nil: diag[].floorRate = floorRate + + # No hit at all, or the field collapsed below its own recent peak: HeadOn. + if bestRate <= 0.0 or bestRate < floorRate: + if diag != nil: diag[].floorFired = true + return 0 + + let tieBand = + if mode == smAbsolute: TieMargin + else: bestRate * RelTieMargin + var tied: seq[GunId] + for gunId in 0..= bestRate - tieBand: + tied.add(gunId) + if diag != nil: diag[].tiedCount = tied.len + if tied.len == 0: return 0 + result = tied[rand(tied.len - 1)] + +proc bestGun*(t: VirtualTracker, targetId: int = -1, + diag: ptr SelectorDiag = nil): GunId = + ## Pick gun with highest hit rate across all power bins. + ## Uses per-enemy fitness when targetId >= 0 and data exists; else aggregate. + ## `diag`, when non-nil, receives the selection internals (bestRate, floor, + ## tie count) exactly as used by the decision. + result = chooseFromFit(t.fitnessFor(targetId), diag, + mode = ActiveSelectorMode, + referenceRate = t.peakRateRef) diff --git a/common_libs/tests/analyze_selector.nim b/common_libs/tests/analyze_selector.nim new file mode 100644 index 0000000..8539943 --- /dev/null +++ b/common_libs/tests/analyze_selector.nim @@ -0,0 +1,128 @@ +## Offline selector diagnostics — WHY does the selector pick what it picks? +## +## Replays real recorded DrussGT movement (tools/fixtures/drussgt_vs_*.jsonl) +## through the 13 ModularBot guns and, on every tick, runs the ACTUAL selector +## core (`chooseFromFit`) under BOTH threshold models, recording: +## * bestRate (the value the floor compares against) +## * floorRate / floorFired +## * tiedCount (guns within the tie band) +## * the selected gun (random tie-break, seeded) +## +## The replay itself is selection-independent, so one pass yields both models. +## Run under both metrics: +## GUN_SELECTOR_MODE=relative nim c -d:release -r \ +## common_libs/tests/analyze_selector.nim [fixture.jsonl ...] +## +## This is a MEASUREMENT tool, not a test; it always exits 0. + +import std/[os, math, strformat, random, algorithm] +import gun_harness/offline_range +import gun_harness/virtual_bullets +import range_guns + +const + repoRoot = currentSourcePath().parentDir.parentDir.parentDir + fixturesDir = repoRoot / "tools" / "fixtures" + +type SelectorStats = object + ticks: int + floorFires: int + floorSum: float + bestRates: seq[float] + tiedCounts: seq[int] + selected: array[13, int] + badInvariant: int ## floor fired but returned a non-zero gun (bug detector) + nonFloor: int + nonFloorTied: int + nonFloorMulti: int + +proc median(x: seq[float]): float = + if x.len == 0: return 0.0 + var y = x + y.sort() + y[y.len div 2] + +proc pct(x: seq[int], p: float): int = + if x.len == 0: return 0 + var y = x + y.sort() + y[int(p * float(max(0, y.len - 1)))] + +proc analyze(fx: Fixture, metric: BulletMetric, mode: SelectorMode, + seed: int): SelectorStats = + randomize(seed) + let drivers = buildAllGunDrivers(seed = seed) + var s: SelectorStats + discard replayFixture(fx, drivers, metric = metric, + liveActual = (fx.meta.source == "live"), + tickCb = proc(t: ptr VirtualTracker) = + var d: SelectorDiag + let g = chooseFromFit(t[].fitnessFor(fx.enemyId), addr d, + mode = mode, referenceRate = t[].peakRateRef) + inc s.ticks + s.bestRates.add d.bestRate + s.floorSum += max(0.0, d.floorRate) + s.tiedCounts.add d.tiedCount + if d.floorFired: + inc s.floorFires + if g != 0: inc s.badInvariant + else: + inc s.nonFloor + s.nonFloorTied += d.tiedCount + if d.tiedCount > 1: inc s.nonFloorMulti + if g >= 0 and g < 13: inc s.selected[g] + ) + s + +proc report(label: string, s: SelectorStats) = + echo fmt" [{label}]" + if s.ticks == 0: + echo " (no ticks)"; return + echo fmt" ticks : {s.ticks}" + echo fmt" floor fired : {s.floorFires} ({100.0*float(s.floorFires)/float(s.ticks):.1f}%) mean floorRate {100.0*s.floorSum/float(s.ticks):.2f}%" + echo fmt" floor-but-not-gun0 : {s.badInvariant}" + let lo = if s.bestRates.len > 0: min(s.bestRates)*100.0 else: 0.0 + let hi = if s.bestRates.len > 0: max(s.bestRates)*100.0 else: 0.0 + echo fmt" bestRate min/med/max : {lo:.2f}% / {median(s.bestRates)*100.0:.2f}% / {hi:.2f}%" + var tiedMean = 0.0 + for c in s.tiedCounts: tiedMean += c.float + tiedMean /= float(s.ticks) + echo fmt" tiedCount mean/med/p90 : {tiedMean:.2f} / {pct(s.tiedCounts, 0.50)} / {pct(s.tiedCounts, 0.90)}" + if s.nonFloor > 0: + echo fmt" non-floor tie mean/multi%: {float(s.nonFloorTied)/float(s.nonFloor):.2f} / {100.0*float(s.nonFloorMulti)/float(s.nonFloor):.1f}%" + echo fmt" selected HeadOn share : {100.0*float(s.selected[0])/float(s.ticks):.1f}%" + var order: seq[int] + for i in 0..<13: order.add i + order.sort(proc(a, b: int): int = cmp(s.selected[b], s.selected[a])) + var top = "" + for k in 0..<6: + let i = order[k] + if s.selected[i] == 0: continue + top.add fmt"{i} {100.0*float(s.selected[i])/float(s.ticks):.1f}% " + echo fmt" top guns : {top}" + +proc main() = + var paths: seq[string] + for i in 1..paramCount(): + paths.add paramStr(i) + if paths.len == 0: + for f in walkFiles(fixturesDir / "drussgt_vs_*.jsonl"): paths.add f + paths.sort() + if paths.len == 0: + echo "no fixtures found"; quit(0) + + for p in paths: + let fx = loadFixture(p) + echo "================================================================" + echo fmt"fixture: {extractFilename(p)} ticks={fx.states.len} source={fx.meta.source}" + echo "================================================================" + for metric in [bmPoint, bmPath]: + let absStats = analyze(fx, metric, smAbsolute, seed = 12345) + let relStats = analyze(fx, metric, smRelative, seed = 12345) + echo fmt"-- {metric} --" + report("absolute (legacy)", absStats) + report("relative (scale-aware)", relStats) + echo "" + +when isMainModule: + main() diff --git a/common_libs/tests/test_gun_harness.nim b/common_libs/tests/test_gun_harness.nim index 94a46f2..17f8a46 100644 --- a/common_libs/tests/test_gun_harness.nim +++ b/common_libs/tests/test_gun_harness.nim @@ -330,10 +330,15 @@ proc testRangeGroundTruthStationary() = r[0].shots > 300 and r[0].hits == r[0].shots proc testRangeConstantVelocityLinearWins() = + # Model-specific property: under the point model HeadOn aims at the current + # position and misses a moving target, while Linear leads it. (Under the + # shipped path model every gun scores 100% here, so the check pins the + # metric it was calibrated against.) let fx = synthesizeConstantVelocity(ticks = 120) let r = replayFixture(fx, @[makeDriver("HeadOn", HeadOnGun()), - makeDriver("Linear", LinearGun())]) - check "range: constant velocity -> Linear beats HeadOn", + makeDriver("Linear", LinearGun())], + metric = bmPoint) + check "range: constant velocity -> Linear beats HeadOn (point model)", r[1].hitRate() > r[0].hitRate() proc testEnergyThresholdFixtureRule() = diff --git a/common_libs/tests/test_tsetlin_gun.nim b/common_libs/tests/test_tsetlin_gun.nim index c026a86..e3e9f5d 100644 --- a/common_libs/tests/test_tsetlin_gun.nim +++ b/common_libs/tests/test_tsetlin_gun.nim @@ -29,7 +29,7 @@ proc main() = block: let fx = synthesizeEnergyThresholdTurner() let (drv, gun) = makeTsetlinDriver(seed = 1) - let reps = replayFixture(fx, @[drv]) + let reps = replayFixture(fx, @[drv], metric = bmPoint) let st = gun[].tmClauseStats() echo &"L1 energy-threshold-turner (ticks={fx.states.len}):" echo &" clauses: mean={st.meanIncluded:.1f} max={st.maxIncluded} " & @@ -52,7 +52,8 @@ proc main() = for name in SyntheticFixtureNames: let fx = synthesizeByName(name) let (drv, gun) = makeTsetlinDriver(seed = 1) - let reps = replayFixture(fx, @[drv, makeDriver("Linear", LinearGun())]) + let reps = replayFixture(fx, @[drv, makeDriver("Linear", LinearGun())], + metric = bmPoint) let t = reps[0] let l = reps[1] echo &" {name:<26} Tsetlin {t.hits:>4}/{t.shots:<4} Linear {l.hits:>4}/{l.shots:<4} " & diff --git a/common_libs/tests/test_vbullet_metric.nim b/common_libs/tests/test_vbullet_metric.nim index b96e98c..8e7b756 100644 --- a/common_libs/tests/test_vbullet_metric.nim +++ b/common_libs/tests/test_vbullet_metric.nim @@ -27,16 +27,16 @@ proc check(name: string, ok: bool) = # ── parsing / default ───────────────────────────────────────────────────────── proc testParsing() = - check "metric parse: empty -> point (default)", - parseMetric("") == bmPoint + check "metric parse: empty -> shipped default", + parseMetric("") == DefaultMetric check "metric parse: 'point' -> bmPoint", parseMetric("point") == bmPoint check "metric parse: 'path' -> bmPath", parseMetric("path") == bmPath check "metric parse: case/space insensitive", parseMetric(" PaTh ") == bmPath - check "metric parse: unknown -> point (safe fallback, warns)", - parseMetric("definitely-not-a-metric") == bmPoint + check "metric parse: unknown -> shipped default (safe fallback, warns)", + parseMetric("definitely-not-a-metric") == DefaultMetric # ── geometry that distinguishes the models ────────────────────────────────────