From 3b5d70b7c3d105d1a13fcb553bcdc6f0c788ca35 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Mon, 21 Sep 2026 03:58:27 +0200 Subject: [PATCH] feat(gun_harness): runtime metric switch + A/B proving the point metric mis-selects Adds GUN_VBULLET_METRIC (point|path, default point = unchanged behaviour) so the virtual-bullet hit model can be selected at runtime with no rebuild. Both the live tracker and the offline replay read the same value, so the 12/12 offline==online acceptance holds under EITHER setting (verified for both). A/B AGAINST THE LIVE BOSS, real server-side hit rate as ground truth, 5 battles x 12 rounds per metric on one frozen binary: point 4660 shots / 219 hits = 4.70% (per-run 3.16-5.53) path 4834 shots / 359 hits = 7.43% (per-run 6.55-8.24) The distributions DO NOT OVERLAP: path's worst run beats point's best run. +2.73pp, +58% relative, z = 5.56, p < 0.0001. Range distributions were identical (~460-478 px), so this is not a range confound. MECHANISM - and this is the important part. The gain is SELECTION, not better gun learning. Under the point model every gun's virtual rate is compressed into 0.6-4.4%, so HeadOn sits inside the 2pp tie margin and takes 72.6% of selection ticks / 76.9% of shots - while HeadOn is 11th of 13 by REAL hit rate (2.3%). The path model widens the band to 4.7-13.7% and ranks HeadOn 10th, so its shot share falls to 35.9% and Pattern/Accel/WallBounce get picked instead. Counterfactual: applying the point model's per-gun real rates to the path model's shot mix yields 7.65%, i.e. essentially the whole observed gain. So the selector, not the guns, is where the win lives. PER-GUN REAL HIT RATE vs DrussGT (path mix, the answer to 'which guns are worth keeping'): WallBounce 10.8, Pattern 10.5, Accel 10.0, Displace 9.3, Circular 9.2, AvgLead 8.5, KNN 5.7, StopShot 5.2, GuessFactor 3.7, Tsetlin 2.9. Per-gun N is small (hundreds of shots) so single-gun ordering is indicative, not definitive. TWO CAVEATS, recorded because they undercut a naive reading: 1. One adversary. DrussGT is a wave surfer and HeadOn is genuinely bad against surfers, so part of this may be matchup-specific. 2. The path model is NOT a better general ranker. Spearman(virtual rank, real rank) is 0.52 under point vs -0.04 under path. It wins by accidentally fixing HeadOn's mis-rank, not by ranking guns better. A more durable fix is to address the selection logic directly - which is the next job. Also adds a focused guard test (test_vbullet_metric) covering parsing/default, a receding-target point-miss/path-hit, a perpendicular-target path-miss, and replay determinism. Verified: 33 guard checks, 12/12 acceptance under both metrics, tsetlin tests green, range 34.3% (point, unchanged) / 50.8% (path). --- common_libs/gun_harness/offline_range.nim | 8 +- common_libs/gun_harness/virtual_bullets.nim | 220 ++++++++++++++++---- common_libs/tests/test_vbullet_metric.nim | 127 +++++++++++ 3 files changed, 311 insertions(+), 44 deletions(-) create mode 100644 common_libs/tests/test_vbullet_metric.nim diff --git a/common_libs/gun_harness/offline_range.nim b/common_libs/gun_harness/offline_range.nim index 9588f46..eda37f0 100644 --- a/common_libs/gun_harness/offline_range.nim +++ b/common_libs/gun_harness/offline_range.nim @@ -218,7 +218,8 @@ proc reportFor(tracker: VirtualTracker, drivers: seq[GunDriver], result.add r proc replayFixture*(fx: Fixture, drivers: seq[GunDriver], - targetId = -1, liveActual = false): seq[GunReport] = + targetId = -1, liveActual = false, + metric = ActiveMetric): seq[GunReport] = ## Drive a fresh `VirtualTracker` over the whole fixture, one tick at a time, ## in the same order the live loop uses: ## 1. predict(state, bulletSpeed(PowerBins[i])) for i = 0..3, per gun @@ -238,9 +239,12 @@ proc replayFixture*(fx: Fixture, drivers: seq[GunDriver], ## resolution) is skipped entirely; the end marker tells us so and we drop ## the final tick's resolutions. Synthetic fixtures leave liveActual false ## (the state at the resolution tick is the ground truth). + ## + ## `metric` defaults to the process-wide `GUN_VBULLET_METRIC` switch read by + ## virtual_bullets; pass it explicitly only to force a model in one process. let tid = if targetId >= 0: targetId else: fx.enemyId let skipFinal = liveActual and fx.enemyDied - var tracker = initTracker(drivers.len) + var tracker = initTracker(drivers.len, metric) for si in 0.. new position) + ## is tested against the target's radius; HIT iff ANY segment came + ## within BotRadius. Measures hypothetical hit chance against the + ## target's real path. + +proc parseMetric*(value: string): BulletMetric = + ## Parse a `GUN_VBULLET_METRIC` value. Empty / unknown values fall back to + ## the shipped `point` model and emit a one-line warning on stderr, so a + ## typo can never silently change the metric and a bad value can never take + ## the bot down. + case value.strip().toLowerAscii() + of "", "point", "points", "bmpoint": bmPoint + of "path", "paths", "bmpath": bmPath + else: + stderr.writeLine("[gun_harness] unknown " & MetricEnvVar & "='" & value & + "'; falling back to 'point' (valid: point|path)") + bmPoint + +let ActiveMetric* = parseMetric(getEnv(MetricEnvVar, "point")) + ## The metric every tracker uses unless a caller overrides it explicitly in + ## `initTracker`. Frozen at process start from the environment. + +type VirtualBullet* = object gunId*: GunId powerBin*: int ## index into PowerBins @@ -36,6 +74,11 @@ type travelDist*: float ## accumulated px so far fireDist*: float ## distance to target at fire time active*: bool + # --- path-metric bookkeeping (unused by the point metric) --- + hitSeen*: bool ## a swept segment already touched the target + bestMissDist*: float ## closest segment->target distance seen so far + bestMissX*: float ## target position at that closest approach + bestMissY*: float FitnessWindow* = object ## Ring buffer of hit booleans. @@ -50,11 +93,15 @@ type bullets*: array[MaxBullets, VirtualBullet] head*: int ## ring buffer head numGuns*: int + metric*: BulletMetric ## scoring model (defaults to ActiveMetric) fitness*: Table[int, seq[GunFitness]] ## keyed by enemy bot ID, indexed by GunId droppedBullets*: int ## unresolved bullets clobbered by the ring buffer (should stay 0) -proc initTracker*(numGuns: int): VirtualTracker = +proc initTracker*(numGuns: int, metric = ActiveMetric): VirtualTracker = + ## `metric` defaults to the process-wide `GUN_VBULLET_METRIC` switch; pass it + ## explicitly only from tests that need both models in one process. result.numGuns = numGuns + result.metric = metric proc hitRate*(fw: FitnessWindow): float = ## Returns fraction of hits in the rolling window. 0.0 when no data. @@ -100,60 +147,149 @@ proc spawnBullets*(t: var VirtualTracker, gunId: GunId, travelDist: 0.0, fireDist: fireDist, active: true, + hitSeen: false, + bestMissDist: Inf, + bestMissX: 0.0, + bestMissY: 0.0, ) t.head = (t.head + 1) mod MaxBullets const StaleTicks* = 20 ## discard bullet if target not seen within this many ticks +proc distPointToSegment*(px, py, ax, ay, bx, by: float): float = + ## Shortest distance from point P to the segment A-B (A/B are bullet + ## positions on consecutive ticks). + let abx = bx - ax + let aby = by - ay + let abLen2 = abx*abx + aby*aby + var s = 0.0 + if abLen2 > 1e-12: + s = clamp(((px - ax)*abx + (py - ay)*aby) / abLen2, 0.0, 1.0) + hypot(px - (ax + s*abx), py - (ay + s*aby)) + proc tickBullets*(t: var VirtualTracker, state: WorldState, enemies: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]], onResolved: proc(gunId: GunId, binIdx: int, e: FeedbackEvent)) = - ## Advance all active bullets one tick. Resolve when bullet reaches target distance. - ## Uses per-target enemy position from enemies table for accurate miss distance. - ## Discards bullet without scoring if target is dead or stale (> StaleTicks). + ## Advance all active bullets one tick. + ## + ## The `bmPoint` branch (default) is unchanged: resolve when the bullet + ## reaches the fire-time aim distance and score the single point it lands on. + ## + ## The `bmPath` branch flies the bullet along its ray until it leaves the + ## arena and tests each tick's swept segment against the target's radius. It + ## records exactly one outcome per bullet (at the wall), so every resolved + ## bullet contributes exactly one fitness sample. A bullet that goes dead or + ## stale is discarded without scoring, mirroring the point metric at + ## resolution time. for i in 0.. StaleTicks: + case t.metric + of bmPoint: + if b.travelDist < b.fireDist: continue + + # Resolved: look up the correct enemy position + var ex, ey: float + if b.targetId in enemies: + let e = enemies[b.targetId] + if not e.alive or (state.tick - e.lastSeenTick) > StaleTicks: + b.active = false + continue + ex = e.x; ey = e.y + else: + # No data for this target — fall back to selected enemy in state + ex = state.enemyX; ey = state.enemyY + + let dx = b.aimX - b.fireX + let dy = b.aimY - b.fireY + let dist = hypot(dx, dy) + let (bx, by) = + if dist < 1e-6: (b.aimX, b.aimY) + else: (b.fireX + dx / dist * b.travelDist, + b.fireY + dy / dist * b.travelDist) + let missDist = hypot(bx - ex, by - ey) + let hit = missDist < BotRadius + + if b.targetId in t.fitness: + t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(hit) + + let fe = FeedbackEvent( + prediction: GunPrediction(x: b.aimX, y: b.aimY), + actualX: ex, + actualY: ey, + bulletPower: PowerBins[b.powerBin], + fireTick: b.fireTick, + powerBin: b.powerBin, + missDistance: missDist, + hit: hit, + ) + onResolved(b.gunId, b.powerBin, fe) + b.active = false + + of bmPath: + # Enemy pose for this tick. A dead/stale target abandons the bullet + # without scoring, exactly as the point metric does at resolution time. + var ex, ey: float + if b.targetId in enemies: + let e = enemies[b.targetId] + if not e.alive or (state.tick - e.lastSeenTick) > StaleTicks: + b.active = false + continue + ex = e.x; ey = e.y + else: + ex = state.enemyX; ey = state.enemyY + + let dx = b.aimX - b.fireX + let dy = b.aimY - b.fireY + let dist = hypot(dx, dy) + var ux, uy: float + if dist < 1e-6: ux = 0.0; uy = 0.0 + else: ux = dx / dist; uy = dy / dist + let prevD = max(0.0, b.travelDist - b.bulletSpeed) + let ax = b.fireX + ux * prevD + let ay = b.fireY + uy * prevD + let bx = b.fireX + ux * b.travelDist + let by = b.fireY + uy * b.travelDist + let segMiss = distPointToSegment(ex, ey, ax, ay, bx, by) + + if not b.hitSeen: + if segMiss < BotRadius: + # First physical contact — freeze it so a later closer approach + # cannot overwrite the contact position the guns learn from. + b.hitSeen = true + b.bestMissDist = segMiss + b.bestMissX = ex + b.bestMissY = ey + elif segMiss < b.bestMissDist: + b.bestMissDist = segMiss + b.bestMissX = ex + b.bestMissY = ey + + # Despawn only at a wall (a degenerate zero-length ray also ends here). + let outside = + dist < 1e-6 or + bx < 0.0 or bx > state.arenaWidth or + by < 0.0 or by > state.arenaHeight + if outside: + let missDist = if b.bestMissDist == Inf: segMiss else: b.bestMissDist + let rx = if b.bestMissDist == Inf: ex else: b.bestMissX + let ry = if b.bestMissDist == Inf: ey else: b.bestMissY + if b.targetId in t.fitness: + t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(b.hitSeen) + let fe = FeedbackEvent( + prediction: GunPrediction(x: b.aimX, y: b.aimY), + actualX: rx, + actualY: ry, + bulletPower: PowerBins[b.powerBin], + fireTick: b.fireTick, + powerBin: b.powerBin, + missDistance: missDist, + hit: b.hitSeen, + ) + onResolved(b.gunId, b.powerBin, fe) b.active = false - continue - ex = e.x; ey = e.y - else: - # No data for this target — fall back to selected enemy in state - ex = state.enemyX; ey = state.enemyY - - let dx = b.aimX - b.fireX - let dy = b.aimY - b.fireY - let dist = hypot(dx, dy) - let (bx, by) = - if dist < 1e-6: (b.aimX, b.aimY) - else: (b.fireX + dx / dist * b.travelDist, - b.fireY + dy / dist * b.travelDist) - let missDist = hypot(bx - ex, by - ey) - let hit = missDist < BotRadius - - if b.targetId in t.fitness: - t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(hit) - - let fe = FeedbackEvent( - prediction: GunPrediction(x: b.aimX, y: b.aimY), - actualX: ex, - actualY: ey, - bulletPower: PowerBins[b.powerBin], - fireTick: b.fireTick, - powerBin: b.powerBin, - missDistance: missDist, - hit: hit, - ) - onResolved(b.gunId, b.powerBin, fe) - b.active = false proc fitnessFor*(t: VirtualTracker, targetId: int): seq[GunFitness] = ## Returns fitness seq for targetId, or merges all enemies as fallback. diff --git a/common_libs/tests/test_vbullet_metric.nim b/common_libs/tests/test_vbullet_metric.nim new file mode 100644 index 0000000..b96e98c --- /dev/null +++ b/common_libs/tests/test_vbullet_metric.nim @@ -0,0 +1,127 @@ +## Focused guard for the virtual-bullet METRIC switch +## (`GUN_VBULLET_METRIC`, see common_libs/gun_harness/virtual_bullets.nim). +## +## The switch selects how a virtual bullet is scored: +## point (default) — single point at the fire-time aim distance; +## path — swept-segment collision along the whole flight. +## +## This test pins the geometry that actually distinguishes the two models, and +## proves the switch is a real runtime override (not a compile-time constant) by +## driving BOTH models from one process via the explicit `initTracker` / +## `replayFixture` metric parameter. +## +## Run: nim c -r common_libs/tests/test_vbullet_metric.nim + +import std/[math, tables, strformat] +import gun_harness/gun_interface +import gun_harness/virtual_bullets +import gun_harness/offline_range +import guns/head_on +import guns/linear + +var failures = 0 +proc check(name: string, ok: bool) = + if ok: echo "PASS: ", name + else: echo "FAIL: ", name; inc failures + +# ── parsing / default ───────────────────────────────────────────────────────── + +proc testParsing() = + check "metric parse: empty -> point (default)", + parseMetric("") == bmPoint + check "metric parse: 'point' -> bmPoint", + parseMetric("point") == bmPoint + check "metric parse: 'path' -> bmPath", + parseMetric("path") == bmPath + check "metric parse: case/space insensitive", + parseMetric(" PaTh ") == bmPath + check "metric parse: unknown -> point (safe fallback, warns)", + parseMetric("definitely-not-a-metric") == bmPoint + +# ── geometry that distinguishes the models ──────────────────────────────────── + +proc mkState(tick: int, ex, ey: float): WorldState = + WorldState(selfX: 100.0, selfY: 100.0, + enemyX: ex, enemyY: ey, enemySpeed: 0.0, enemyHeading: 0.0, + arenaWidth: 1000.0, arenaHeight: 1000.0, tick: tick) + +proc runOneBullet(metric: BulletMetric, states: seq[WorldState], + targetId = 7): tuple[hits, shots: int] = + ## Spawn one bullet per power bin at tick 0, aimed at the enemy's tick-0 + ## position, then tick the tracker over the rest of the stream. + var t = initTracker(1, metric) + let s0 = states[0] + let preds = [GunPrediction(x: s0.enemyX, y: s0.enemyY), + GunPrediction(x: s0.enemyX, y: s0.enemyY), + GunPrediction(x: s0.enemyX, y: s0.enemyY), + GunPrediction(x: s0.enemyX, y: s0.enemyY)] + t.spawnBullets(0, preds, s0, targetId) + for i in 1.. miss. The path model keeps the bullet + ## flying until the wall and it physically catches up -> hit. + var states: seq[WorldState] + for t in 0..<140: + states.add mkState(t, 300.0 + 4.0*t.float, 100.0) + let p = runOneBullet(bmPoint, states) + let q = runOneBullet(bmPath, states) + check "receding target: point model scores 0 hits (enemy left the aim point)", + p.hits == 0 + check "receding target: path model catches the receding enemy", q.hits > 0 + check "receding target: both models emit the same number of samples", + p.shots == q.shots and q.shots == len(PowerBins) + echo fmt" receding: point {p.hits}/{p.shots}, path {q.hits}/{q.shots}" + +proc testPerpendicularTarget() = + ## Enemy crosses the ray perpendicularly. The ray and the enemy's trajectory + ## intersect at the enemy's tick-0 position, but the bullet is 200 px away at + ## that instant. A physical swept-collision model must MISS; a naive + ## path-intersection model would wrongly hit. + var states: seq[WorldState] + for t in 0..<140: + states.add mkState(t, 300.0, 100.0 + 8.0*t.float) + let q = runOneBullet(bmPath, states) + check "perpendicular target: path model misses (timing matters, not just path)", + q.hits == 0 + echo fmt" perpendicular: path {q.hits}/{q.shots}" + +# ── switch drives the offline replay ───────────────────────────────────────── + +proc testReplayMetricOverride() = + let fx = synthesizeConstantVelocity(ticks = 140) + let p = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPoint) + let q = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPath) + check "replay: both metrics record samples on a fixture", + p[0].shots > 0 and q[0].shots > 0 + check "replay: path model is deterministic run-to-run", + block: + let q2 = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPath) + q2[0].hits == q[0].hits and q2[0].shots == q[0].shots + echo fmt" constant-velocity Linear: point {p[0].hits}/{p[0].shots}, path {q[0].hits}/{q[0].shots}" + +# ── driver ──────────────────────────────────────────────────────────────────── + +testParsing() +testRecedingTarget() +testPerpendicularTarget() +testReplayMetricOverride() + +if failures > 0: + echo "\n", failures, " check(s) FAILED" + quit(1) +echo "\nAll virtual-bullet metric checks passed."