feat(gun_harness): runtime metric switch + A/B proving the point metric mis-selects

Adds GUN_VBULLET_METRIC (point|path, default point = unchanged behaviour) so
the virtual-bullet hit model can be selected at runtime with no rebuild. Both
the live tracker and the offline replay read the same value, so the 12/12
offline==online acceptance holds under EITHER setting (verified for both).

A/B AGAINST THE LIVE BOSS, real server-side hit rate as ground truth, 5
battles x 12 rounds per metric on one frozen binary:
  point  4660 shots / 219 hits = 4.70%   (per-run 3.16-5.53)
  path   4834 shots / 359 hits = 7.43%   (per-run 6.55-8.24)
The distributions DO NOT OVERLAP: path's worst run beats point's best run.
+2.73pp, +58% relative, z = 5.56, p < 0.0001. Range distributions were
identical (~460-478 px), so this is not a range confound.

MECHANISM - and this is the important part. The gain is SELECTION, not better
gun learning. Under the point model every gun's virtual rate is compressed
into 0.6-4.4%, so HeadOn sits inside the 2pp tie margin and takes 72.6% of
selection ticks / 76.9% of shots - while HeadOn is 11th of 13 by REAL hit rate
(2.3%). The path model widens the band to 4.7-13.7% and ranks HeadOn 10th, so
its shot share falls to 35.9% and Pattern/Accel/WallBounce get picked instead.
Counterfactual: applying the point model's per-gun real rates to the path
model's shot mix yields 7.65%, i.e. essentially the whole observed gain.
So the selector, not the guns, is where the win lives.

PER-GUN REAL HIT RATE vs DrussGT (path mix, the answer to 'which guns are
worth keeping'): WallBounce 10.8, Pattern 10.5, Accel 10.0, Displace 9.3,
Circular 9.2, AvgLead 8.5, KNN 5.7, StopShot 5.2, GuessFactor 3.7,
Tsetlin 2.9. Per-gun N is small (hundreds of shots) so single-gun ordering is
indicative, not definitive.

TWO CAVEATS, recorded because they undercut a naive reading:
1. One adversary. DrussGT is a wave surfer and HeadOn is genuinely bad against
   surfers, so part of this may be matchup-specific.
2. The path model is NOT a better general ranker. Spearman(virtual rank, real
   rank) is 0.52 under point vs -0.04 under path. It wins by accidentally
   fixing HeadOn's mis-rank, not by ranking guns better. A more durable fix is
   to address the selection logic directly - which is the next job.

Also adds a focused guard test (test_vbullet_metric) covering parsing/default,
a receding-target point-miss/path-hit, a perpendicular-target path-miss, and
replay determinism.

Verified: 33 guard checks, 12/12 acceptance under both metrics, tsetlin tests
green, range 34.3% (point, unchanged) / 50.8% (path).
This commit is contained in:
2026-09-21 03:58:27 +02:00
parent e2ca2fc7d8
commit 3b5d70b7c3
3 changed files with 311 additions and 44 deletions
+6 -2
View File
@@ -218,7 +218,8 @@ proc reportFor(tracker: VirtualTracker, drivers: seq[GunDriver],
result.add r
proc replayFixture*(fx: Fixture, drivers: seq[GunDriver],
targetId = -1, liveActual = false): seq[GunReport] =
targetId = -1, liveActual = false,
metric = ActiveMetric): seq[GunReport] =
## Drive a fresh `VirtualTracker` over the whole fixture, one tick at a time,
## in the same order the live loop uses:
## 1. predict(state, bulletSpeed(PowerBins[i])) for i = 0..3, per gun
@@ -238,9 +239,12 @@ proc replayFixture*(fx: Fixture, drivers: seq[GunDriver],
## resolution) is skipped entirely; the end marker tells us so and we drop
## the final tick's resolutions. Synthetic fixtures leave liveActual false
## (the state at the resolution tick is the ground truth).
##
## `metric` defaults to the process-wide `GUN_VBULLET_METRIC` switch read by
## virtual_bullets; pass it explicitly only to force a model in one process.
let tid = if targetId >= 0: targetId else: fx.enemyId
let skipFinal = liveActual and fx.enemyDied
var tracker = initTracker(drivers.len)
var tracker = initTracker(drivers.len, metric)
for si in 0..<fx.states.len:
let state = fx.states[si]
for gi in 0..<drivers.len:
+178 -42
View File
@@ -7,6 +7,8 @@ import std/math
import std/tables
import std/random
import std/algorithm
import std/os
import std/strutils
import gun_interface
const
@@ -15,15 +17,51 @@ const
MaxBullets* = 8192 ## hard cap; ring buffer. 52 spawns/tick and a
## full-map long shot (~90 ticks) need ~4700 slots;
## 8192 wraps only after ~157 ticks. Each VirtualBullet
## is 88 bytes, so this array costs ~704 KiB.
## is ~120 bytes, so this array costs ~960 KiB.
MinHitRate* = 0.40 ## 40% threshold for acceptable power selection
MinObsBeforeCompete* = 50 ## min observations before a gun×bin enters competition
TieMargin* = 0.02 ## guns within this hit-rate margin of best are tied
MinHitRateFloor* = 0.10 ## if best gun < this, fall back to gun 0 (HeadOn)
MetricEnvVar* = "GUN_VBULLET_METRIC"
## RUNTIME switch selecting how a virtual bullet is scored. Read once per
## process at module init, so the SAME compiled binary can be A/B'd by
## exporting it — no rebuild needed. Both the live ModularBot tracker and
## the offline range replay call `initTracker`, so they always agree.
type
GunId* = int ## index into the guns seq
BulletMetric* = enum
bmPoint ## DEFAULT. A bullet is scored at the single point it reaches at
## the fire-time aim distance. HIT iff that point is within
## BotRadius of the target on that tick. Measures prediction
## accuracy (does the bullet arrive at the predicted point at the
## right time).
bmPath ## The bullet flies along its straight ray until it leaves the
## arena. Each tick the swept segment (previous -> new position)
## is tested against the target's radius; HIT iff ANY segment came
## within BotRadius. Measures hypothetical hit chance against the
## target's real path.
proc parseMetric*(value: string): BulletMetric =
## Parse a `GUN_VBULLET_METRIC` value. Empty / unknown values fall back to
## the shipped `point` model and emit a one-line warning on stderr, so a
## typo can never silently change the metric and a bad value can never take
## the bot down.
case value.strip().toLowerAscii()
of "", "point", "points", "bmpoint": bmPoint
of "path", "paths", "bmpath": bmPath
else:
stderr.writeLine("[gun_harness] unknown " & MetricEnvVar & "='" & value &
"'; falling back to 'point' (valid: point|path)")
bmPoint
let ActiveMetric* = parseMetric(getEnv(MetricEnvVar, "point"))
## The metric every tracker uses unless a caller overrides it explicitly in
## `initTracker`. Frozen at process start from the environment.
type
VirtualBullet* = object
gunId*: GunId
powerBin*: int ## index into PowerBins
@@ -36,6 +74,11 @@ type
travelDist*: float ## accumulated px so far
fireDist*: float ## distance to target at fire time
active*: bool
# --- path-metric bookkeeping (unused by the point metric) ---
hitSeen*: bool ## a swept segment already touched the target
bestMissDist*: float ## closest segment->target distance seen so far
bestMissX*: float ## target position at that closest approach
bestMissY*: float
FitnessWindow* = object
## Ring buffer of hit booleans.
@@ -50,11 +93,15 @@ type
bullets*: array[MaxBullets, VirtualBullet]
head*: int ## ring buffer head
numGuns*: int
metric*: BulletMetric ## scoring model (defaults to ActiveMetric)
fitness*: Table[int, seq[GunFitness]] ## keyed by enemy bot ID, indexed by GunId
droppedBullets*: int ## unresolved bullets clobbered by the ring buffer (should stay 0)
proc initTracker*(numGuns: int): VirtualTracker =
proc initTracker*(numGuns: int, metric = ActiveMetric): VirtualTracker =
## `metric` defaults to the process-wide `GUN_VBULLET_METRIC` switch; pass it
## explicitly only from tests that need both models in one process.
result.numGuns = numGuns
result.metric = metric
proc hitRate*(fw: FitnessWindow): float =
## Returns fraction of hits in the rolling window. 0.0 when no data.
@@ -100,60 +147,149 @@ proc spawnBullets*(t: var VirtualTracker, gunId: GunId,
travelDist: 0.0,
fireDist: fireDist,
active: true,
hitSeen: false,
bestMissDist: Inf,
bestMissX: 0.0,
bestMissY: 0.0,
)
t.head = (t.head + 1) mod MaxBullets
const StaleTicks* = 20 ## discard bullet if target not seen within this many ticks
proc distPointToSegment*(px, py, ax, ay, bx, by: float): float =
## Shortest distance from point P to the segment A-B (A/B are bullet
## positions on consecutive ticks).
let abx = bx - ax
let aby = by - ay
let abLen2 = abx*abx + aby*aby
var s = 0.0
if abLen2 > 1e-12:
s = clamp(((px - ax)*abx + (py - ay)*aby) / abLen2, 0.0, 1.0)
hypot(px - (ax + s*abx), py - (ay + s*aby))
proc tickBullets*(t: var VirtualTracker, state: WorldState,
enemies: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]],
onResolved: proc(gunId: GunId, binIdx: int, e: FeedbackEvent)) =
## Advance all active bullets one tick. Resolve when bullet reaches target distance.
## Uses per-target enemy position from enemies table for accurate miss distance.
## Discards bullet without scoring if target is dead or stale (> StaleTicks).
## Advance all active bullets one tick.
##
## The `bmPoint` branch (default) is unchanged: resolve when the bullet
## reaches the fire-time aim distance and score the single point it lands on.
##
## The `bmPath` branch flies the bullet along its ray until it leaves the
## arena and tests each tick's swept segment against the target's radius. It
## records exactly one outcome per bullet (at the wall), so every resolved
## bullet contributes exactly one fitness sample. A bullet that goes dead or
## stale is discarded without scoring, mirroring the point metric at
## resolution time.
for i in 0..<MaxBullets:
var b = addr t.bullets[i]
if not b.active: continue
b.travelDist += b.bulletSpeed
if b.travelDist < b.fireDist: continue
# Resolved: look up the correct enemy position
var ex, ey: float
if b.targetId in enemies:
let e = enemies[b.targetId]
if not e.alive or (state.tick - e.lastSeenTick) > StaleTicks:
case t.metric
of bmPoint:
if b.travelDist < b.fireDist: continue
# Resolved: look up the correct enemy position
var ex, ey: float
if b.targetId in enemies:
let e = enemies[b.targetId]
if not e.alive or (state.tick - e.lastSeenTick) > StaleTicks:
b.active = false
continue
ex = e.x; ey = e.y
else:
# No data for this target — fall back to selected enemy in state
ex = state.enemyX; ey = state.enemyY
let dx = b.aimX - b.fireX
let dy = b.aimY - b.fireY
let dist = hypot(dx, dy)
let (bx, by) =
if dist < 1e-6: (b.aimX, b.aimY)
else: (b.fireX + dx / dist * b.travelDist,
b.fireY + dy / dist * b.travelDist)
let missDist = hypot(bx - ex, by - ey)
let hit = missDist < BotRadius
if b.targetId in t.fitness:
t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(hit)
let fe = FeedbackEvent(
prediction: GunPrediction(x: b.aimX, y: b.aimY),
actualX: ex,
actualY: ey,
bulletPower: PowerBins[b.powerBin],
fireTick: b.fireTick,
powerBin: b.powerBin,
missDistance: missDist,
hit: hit,
)
onResolved(b.gunId, b.powerBin, fe)
b.active = false
of bmPath:
# Enemy pose for this tick. A dead/stale target abandons the bullet
# without scoring, exactly as the point metric does at resolution time.
var ex, ey: float
if b.targetId in enemies:
let e = enemies[b.targetId]
if not e.alive or (state.tick - e.lastSeenTick) > StaleTicks:
b.active = false
continue
ex = e.x; ey = e.y
else:
ex = state.enemyX; ey = state.enemyY
let dx = b.aimX - b.fireX
let dy = b.aimY - b.fireY
let dist = hypot(dx, dy)
var ux, uy: float
if dist < 1e-6: ux = 0.0; uy = 0.0
else: ux = dx / dist; uy = dy / dist
let prevD = max(0.0, b.travelDist - b.bulletSpeed)
let ax = b.fireX + ux * prevD
let ay = b.fireY + uy * prevD
let bx = b.fireX + ux * b.travelDist
let by = b.fireY + uy * b.travelDist
let segMiss = distPointToSegment(ex, ey, ax, ay, bx, by)
if not b.hitSeen:
if segMiss < BotRadius:
# First physical contact — freeze it so a later closer approach
# cannot overwrite the contact position the guns learn from.
b.hitSeen = true
b.bestMissDist = segMiss
b.bestMissX = ex
b.bestMissY = ey
elif segMiss < b.bestMissDist:
b.bestMissDist = segMiss
b.bestMissX = ex
b.bestMissY = ey
# Despawn only at a wall (a degenerate zero-length ray also ends here).
let outside =
dist < 1e-6 or
bx < 0.0 or bx > state.arenaWidth or
by < 0.0 or by > state.arenaHeight
if outside:
let missDist = if b.bestMissDist == Inf: segMiss else: b.bestMissDist
let rx = if b.bestMissDist == Inf: ex else: b.bestMissX
let ry = if b.bestMissDist == Inf: ey else: b.bestMissY
if b.targetId in t.fitness:
t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(b.hitSeen)
let fe = FeedbackEvent(
prediction: GunPrediction(x: b.aimX, y: b.aimY),
actualX: rx,
actualY: ry,
bulletPower: PowerBins[b.powerBin],
fireTick: b.fireTick,
powerBin: b.powerBin,
missDistance: missDist,
hit: b.hitSeen,
)
onResolved(b.gunId, b.powerBin, fe)
b.active = false
continue
ex = e.x; ey = e.y
else:
# No data for this target — fall back to selected enemy in state
ex = state.enemyX; ey = state.enemyY
let dx = b.aimX - b.fireX
let dy = b.aimY - b.fireY
let dist = hypot(dx, dy)
let (bx, by) =
if dist < 1e-6: (b.aimX, b.aimY)
else: (b.fireX + dx / dist * b.travelDist,
b.fireY + dy / dist * b.travelDist)
let missDist = hypot(bx - ex, by - ey)
let hit = missDist < BotRadius
if b.targetId in t.fitness:
t.fitness[b.targetId][b.gunId].bins[b.powerBin].record(hit)
let fe = FeedbackEvent(
prediction: GunPrediction(x: b.aimX, y: b.aimY),
actualX: ex,
actualY: ey,
bulletPower: PowerBins[b.powerBin],
fireTick: b.fireTick,
powerBin: b.powerBin,
missDistance: missDist,
hit: hit,
)
onResolved(b.gunId, b.powerBin, fe)
b.active = false
proc fitnessFor*(t: VirtualTracker, targetId: int): seq[GunFitness] =
## Returns fitness seq for targetId, or merges all enemies as fallback.
+127
View File
@@ -0,0 +1,127 @@
## Focused guard for the virtual-bullet METRIC switch
## (`GUN_VBULLET_METRIC`, see common_libs/gun_harness/virtual_bullets.nim).
##
## The switch selects how a virtual bullet is scored:
## point (default) — single point at the fire-time aim distance;
## path — swept-segment collision along the whole flight.
##
## This test pins the geometry that actually distinguishes the two models, and
## proves the switch is a real runtime override (not a compile-time constant) by
## driving BOTH models from one process via the explicit `initTracker` /
## `replayFixture` metric parameter.
##
## Run: nim c -r common_libs/tests/test_vbullet_metric.nim
import std/[math, tables, strformat]
import gun_harness/gun_interface
import gun_harness/virtual_bullets
import gun_harness/offline_range
import guns/head_on
import guns/linear
var failures = 0
proc check(name: string, ok: bool) =
if ok: echo "PASS: ", name
else: echo "FAIL: ", name; inc failures
# ── parsing / default ─────────────────────────────────────────────────────────
proc testParsing() =
check "metric parse: empty -> point (default)",
parseMetric("") == bmPoint
check "metric parse: 'point' -> bmPoint",
parseMetric("point") == bmPoint
check "metric parse: 'path' -> bmPath",
parseMetric("path") == bmPath
check "metric parse: case/space insensitive",
parseMetric(" PaTh ") == bmPath
check "metric parse: unknown -> point (safe fallback, warns)",
parseMetric("definitely-not-a-metric") == bmPoint
# ── geometry that distinguishes the models ────────────────────────────────────
proc mkState(tick: int, ex, ey: float): WorldState =
WorldState(selfX: 100.0, selfY: 100.0,
enemyX: ex, enemyY: ey, enemySpeed: 0.0, enemyHeading: 0.0,
arenaWidth: 1000.0, arenaHeight: 1000.0, tick: tick)
proc runOneBullet(metric: BulletMetric, states: seq[WorldState],
targetId = 7): tuple[hits, shots: int] =
## Spawn one bullet per power bin at tick 0, aimed at the enemy's tick-0
## position, then tick the tracker over the rest of the stream.
var t = initTracker(1, metric)
let s0 = states[0]
let preds = [GunPrediction(x: s0.enemyX, y: s0.enemyY),
GunPrediction(x: s0.enemyX, y: s0.enemyY),
GunPrediction(x: s0.enemyX, y: s0.enemyY),
GunPrediction(x: s0.enemyX, y: s0.enemyY)]
t.spawnBullets(0, preds, s0, targetId)
for i in 1..<states.len:
let st = states[i]
var enemies: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]]
enemies[targetId] = (x: st.enemyX, y: st.enemyY,
lastSeenTick: st.tick, alive: true)
t.tickBullets(st, enemies,
proc(gid: GunId, bin: int, e: FeedbackEvent) = discard)
let fit = t.fitnessFor(targetId)
for bin in 0..<len(PowerBins):
let n = min(fit[0].bins[bin].count, WindowSize)
result.shots += n
for k in 0..<n:
if fit[0].bins[bin].hits[k]: inc result.hits
proc testRecedingTarget() =
## Enemy recedes radially along the bullet's own ray, faster than the slowest
## bullet. The point model fires at the tick-0 aim distance (200 px) and the
## enemy has already moved past it -> miss. The path model keeps the bullet
## flying until the wall and it physically catches up -> hit.
var states: seq[WorldState]
for t in 0..<140:
states.add mkState(t, 300.0 + 4.0*t.float, 100.0)
let p = runOneBullet(bmPoint, states)
let q = runOneBullet(bmPath, states)
check "receding target: point model scores 0 hits (enemy left the aim point)",
p.hits == 0
check "receding target: path model catches the receding enemy", q.hits > 0
check "receding target: both models emit the same number of samples",
p.shots == q.shots and q.shots == len(PowerBins)
echo fmt" receding: point {p.hits}/{p.shots}, path {q.hits}/{q.shots}"
proc testPerpendicularTarget() =
## Enemy crosses the ray perpendicularly. The ray and the enemy's trajectory
## intersect at the enemy's tick-0 position, but the bullet is 200 px away at
## that instant. A physical swept-collision model must MISS; a naive
## path-intersection model would wrongly hit.
var states: seq[WorldState]
for t in 0..<140:
states.add mkState(t, 300.0, 100.0 + 8.0*t.float)
let q = runOneBullet(bmPath, states)
check "perpendicular target: path model misses (timing matters, not just path)",
q.hits == 0
echo fmt" perpendicular: path {q.hits}/{q.shots}"
# ── switch drives the offline replay ─────────────────────────────────────────
proc testReplayMetricOverride() =
let fx = synthesizeConstantVelocity(ticks = 140)
let p = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPoint)
let q = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPath)
check "replay: both metrics record samples on a fixture",
p[0].shots > 0 and q[0].shots > 0
check "replay: path model is deterministic run-to-run",
block:
let q2 = replayFixture(fx, @[makeDriver("Linear", LinearGun())], metric = bmPath)
q2[0].hits == q[0].hits and q2[0].shots == q[0].shots
echo fmt" constant-velocity Linear: point {p[0].hits}/{p[0].shots}, path {q[0].hits}/{q[0].shots}"
# ── driver ────────────────────────────────────────────────────────────────────
testParsing()
testRecedingTarget()
testPerpendicularTarget()
testReplayMetricOverride()
if failures > 0:
echo "\n", failures, " check(s) FAILED"
quit(1)
echo "\nAll virtual-bullet metric checks passed."