wave pairing: 36-58% of GF/DecayGF/KNN learning samples were MISLABELLED

The audit inferred (from code) that GF/DecayGF/KNN pop the OLDEST wave on
resolution, while under bmPath bullets leave the arena in NON-FIFO order - so an
outcome could be attached to the wrong wave. It also noted that `starved=0` does
NOT rule this out. Both halves are now MEASURED.

MISPAIRING RATE (10 DrussGT fixtures, real VirtualTracker, 344k resolutions/gun):
  gun         bmPath mispair   label err      bmPoint mispair   label err
  GuessFactor     36.48%         19.39%           18.24%          7.62%
  DecayGF         36.85%         19.52%           20.57%          8.64%
  KNN             57.91%         27.63%           29.75%         11.58%
  (starved = 0 everywhere, exactly as the audit predicted)
So ~1 in 5 GF/DecayGF learning samples and ~1 in 4 KNN samples carried a WRONG
guess-factor bin. This is a material corruption of the learning signal.

FIX: the same fireTick-keyed ring scheme `tsetlin.nim`/`tm_selector.nim` already
use - `slot = (fireTick*4 + bin) mod 1024` (period 256 ticks, longer than the
~91-tick max flight), looked up by exact key. Public interfaces unchanged; added
`waveResolved`/`waveMispaired` integrity counters. AFTER: mispaired = 0 and
starved = 0, both metrics, all three guns.

EFFECT ON HIT RATE: SMALL AND NOT SIGNIFICANT. bmPath 4000 samples/gun:
  GuessFactor 23.20% -> 23.02% (-0.18pp, per-run sign-flip p=0.750)
  DecayGF     23.80% -> 24.25% (+0.45pp, p=0.625)
  KNN         18.27% -> 18.80% (+0.53pp, p=0.547)
bmPoint: +0.05 / +0.33 / -0.15pp, p = 1.00 / 0.50 / 0.50. Per-run ranges overlap
almost completely. A bullet-level z-test is anti-conservative (bullets within a
fixture share a trajectory) and its KNN p=1.9e-16 cannot be trusted given ~10
effective independent runs.
PLAIN READING: this is a CORRECTNESS fix, not a measurable hit-rate win. It
removes a 36-58% mislabelling of the learning signal; the point estimates move by
at most ~0.5pp, within run-to-run noise. Stated plainly rather than oversold.

A REGRESSION IT CAUGHT IN ITSELF (and this explains the SIGSEGV another job saw
and correctly attributed to a concurrent knn_gun.nim rewrite): the first
implementation put an inline `array[1024, KNNWave]` (~100KB) inside each gun,
which overflowed the default 8MB stack and made `test_power_selection` SIGSEGV.
Causation was proven by stashing only the three gun files (test passed), then
fixed by making the rings heap-backed `seq`. Verified: `test_power_selection`
3 PASS on the default stack, and zero inline `array[1024]` remain.

Guards: test_wave_pairing 17 (new, pure), test_gun_harness 39,
test_vbullet_metric 11, test_power_selection 3, test_adaptive_radar 41,
test_tfil_ring_weights 24, test_power_policy 26, test_ram_decision 28.
ModularBot compiles. Adds audit_wave_pairing.nim and compare_pairing.nim.
This commit is contained in:
2026-09-22 01:33:31 +02:00
parent 1ea72c7f14
commit 4657fe715e
6 changed files with 586 additions and 65 deletions
+33 -20
View File
@@ -1,6 +1,7 @@
## Recency-weighted GF gun: exponential decay on histogram bins.
## decay=0.998/tick gives ~350-tick half-life — adapts to mid-battle strategy shifts.
## Everything else identical to guess_factor.nim, including per-power-bin wave queues.
## Everything else identical to guess_factor.nim, including exact (fireTick,
## powerBin)-keyed wave pairing (see guess_factor.nim for the measured FIFO defect).
import std/math
import gun_harness/gun_interface
@@ -11,29 +12,35 @@ const
GFBins = 31
GFPrior = 0.1
DecayRate = 0.998 # ponytail: single global decay, tune if adaptation too slow/fast
DecayWaveCompactAt = 64
DecayWaveRingSlots = 1024 # (fireTick, powerBin) ring; see guess_factor.nim
type
DWave = object
fireX, fireY: float
fireBearing: float
fireTick: int
bin: int
alive: bool
DecayGFGun* = object
bins: array[GFBins, float]
# One wave queue per power bin; a resolved bullet only learns from a wave
# queued for its own bin (matched on bulletSpeed / bulletPower).
waves: array[len(vb.PowerBins), seq[DWave]]
waveHead: array[len(vb.PowerBins), int] # O(1) pop cursor
# Exact (fireTick, powerBin)-keyed ring; a resolved bullet is matched to the
# wave it actually fired, no matter how many other shots resolved first.
# Heap-backed (seq): see guess_factor.nim.
waves: seq[DWave]
waveStoredTick: array[len(vb.PowerBins), int] # last tick a wave was queued for this bin
vt: VelocityTracker # enemy velocity history (base selection)
cachedTick: int # last tick bins were decayed
wavePushes*: int
waveStarved*: int
waveResolved*: int
waveMispaired*: int # ring-slot collision (impossible by design)
debugGraphics*: bool
proc initDecayGFGun*(): DecayGFGun =
result.cachedTick = -1
result.debugGraphics = false
result.waves = newSeq[DWave](DecayWaveRingSlots)
for b in 0..<len(vb.PowerBins):
result.waveStoredTick[b] = -1
let center = (GFBins - 1) div 2
@@ -47,7 +54,7 @@ proc gfToIndex(gf: float): int {.inline.} =
proc indexToGF(idx: int): float {.inline.} =
float(idx) / float(GFBins - 1) * 2.0 - 1.0
proc peakBin(g: DecayGFGun): int =
proc peakBin*(g: DecayGFGun): int =
var best = 0
for i in 1..<GFBins:
if g.bins[i] > g.bins[best]:
@@ -66,15 +73,8 @@ proc binForPower(power: float): int {.inline.} =
return i
-1
proc takeOldestWave(g: var DecayGFGun, binIdx: int): (bool, DWave) =
if binIdx < 0 or g.waveHead[binIdx] >= g.waves[binIdx].len:
return (false, DWave())
result = (true, g.waves[binIdx][g.waveHead[binIdx]])
inc g.waveHead[binIdx]
if g.waveHead[binIdx] >= DecayWaveCompactAt and
g.waveHead[binIdx] * 2 >= g.waves[binIdx].len:
g.waves[binIdx] = g.waves[binIdx][g.waveHead[binIdx] .. g.waves[binIdx].high]
g.waveHead[binIdx] = 0
proc waveSlot(fireTick, binIdx: int): int {.inline.} =
((fireTick * len(vb.PowerBins)) + binIdx) mod DecayWaveRingSlots
proc predict*(g: var DecayGFGun, state: WorldState, bulletSpeed: float): GunPrediction =
if bulletSpeed <= 0.0:
@@ -97,7 +97,10 @@ proc predict*(g: var DecayGFGun, state: WorldState, bulletSpeed: float): GunPred
# call for the selected bin lands on the same tick and reuses the queued wave.
let binIdx = binForSpeed(bulletSpeed)
if binIdx >= 0 and g.waveStoredTick[binIdx] != state.tick:
g.waves[binIdx].add DWave(fireX: state.selfX, fireY: state.selfY, fireBearing: f.bearing)
let slot = waveSlot(state.tick, binIdx)
g.waves[slot] = DWave(fireX: state.selfX, fireY: state.selfY,
fireBearing: f.bearing, fireTick: state.tick,
bin: binIdx, alive: true)
g.waveStoredTick[binIdx] = state.tick
inc g.wavePushes
@@ -113,13 +116,21 @@ proc predict*(g: var DecayGFGun, state: WorldState, bulletSpeed: float): GunPred
)
proc onResult*(g: var DecayGFGun, e: FeedbackEvent) =
let binIdx = binForPower(e.bulletPower)
let binIdx =
if e.powerBin >= 0 and e.powerBin < len(vb.PowerBins): e.powerBin
else: binForPower(e.bulletPower)
if binIdx < 0: return
let (found, w) = g.takeOldestWave(binIdx)
if not found:
let slot = waveSlot(e.fireTick, binIdx)
var w = addr g.waves[slot]
if not w.alive:
inc g.waveStarved
return
if w.fireTick != e.fireTick:
inc g.waveMispaired
inc g.waveStarved
return
inc g.waveResolved
let speed = bulletSpeed(e.bulletPower)
let mea = arcsin(clamp(8.0 / speed, -1.0, 1.0))
@@ -136,3 +147,5 @@ proc onResult*(g: var DecayGFGun, e: FeedbackEvent) =
for i in 0..<GFBins:
let d = abs(i - centerIdx)
g.bins[i] += 1.0 / float(1 + d)
w.alive = false
+56 -27
View File
@@ -1,6 +1,16 @@
## Guess-factor gun: statistical targeting via GF histogram.
## Bins: 31, ranging GF -1 (max CW escape) to +1 (max CCW escape).
## Learns from virtual bullet outcomes; queues one wave per (tick, power bin).
##
## Wave pairing is EXACT: a wave is stored in a ring slot keyed by
## (fireTick, powerBin) and `onResult` looks up the wave with the resolution
## event's `fireTick`, NOT the oldest queued wave. Under the shipped bmPath
## metric bullets leave the arena in non-FIFO order (the aim direction changes
## every tick), so FIFO pairing attached outcomes to the wrong wave. Measured
## over the committed DrussGT fixtures, FIFO mispaired 36.5% of resolutions
## (19.4% of which changed the recorded GF bin). See
## common_libs/tests/audit_wave_pairing.nim. The exact key is the same pattern
## guns/tsetlin.nim and guns/tm_selector.nim use.
import std/[math, strformat]
import gun_harness/gun_interface
@@ -11,31 +21,43 @@ const
GFBins = 31
GFPrior = 0.1
DebugGF* = false
WaveCompactAt = 64 ## compact a bin's wave seq once this many entries are consumed
WaveRingSlots = 1024
## (fireTick, powerBin) -> ring slot. Period = WaveRingSlots/bins = 256 ticks.
## A bmPath bullet leaves an 800x600 (Tank Royale max 1000x1000) arena within
## ~91 (128) ticks, so a live wave is never overwritten by a newer one.
## Identical sizing to tsetlin.nim's TM_TRACE_SLOTS.
type
Wave = object
fireX, fireY: float
fireBearing: float # atan2(enemyY-selfY, enemyX-selfX) at fire tick (rad)
fireTick: int # key part: tick the bullet was fired
bin: int # key part: power bin the bullet belonged to
alive: bool
# mea not stored — recomputed from FeedbackEvent.bulletPower at resolution time
GFGun* = object
bins: array[GFBins, float]
# One wave queue per power bin. The owning bin is fixed at push time (from the
# bulletSpeed argument) and at pop time (from FeedbackEvent.bulletPower), so a
# resolved bullet is always paired with a wave from its own bin.
waves: array[len(vb.PowerBins), seq[Wave]]
waveHead: array[len(vb.PowerBins), int] # O(1) pop cursor into waves[bin]
# Exact (fireTick, powerBin)-keyed ring. A resolved bullet is matched to the
# wave it actually fired, no matter how many other shots resolved first.
# Heap-backed (seq) so the gun value stays small on the stack — a 1024-slot
# inline array overflowed the default 8 MB stack in test_power_selection.
waves: seq[Wave]
waveStoredTick: array[len(vb.PowerBins), int] # last tick a wave was queued for this bin
vt: VelocityTracker # enemy velocity history (base selection)
cachedTick: int # last tick the velocity tracker was advanced
wavePushes*: int # total waves enqueued (== one per (tick, bin))
waveStarved*: int # onResult found an empty queue for its own bin
waveStarved*: int # onResult found no live wave for its (fireTick, bin)
# ── pairing integrity ───────────────────────────────────────────────────
waveResolved*: int # onResult calls that found their exact wave
waveMispaired*: int # ring-slot collision (impossible by design): the
# slot held a different fireTick
debugGraphics*: bool
proc initGFGun*(): GFGun =
result.debugGraphics = false
result.cachedTick = -1
result.waves = newSeq[Wave](WaveRingSlots)
for b in 0..<len(vb.PowerBins):
result.waveStoredTick[b] = -1
# Seed with a head-on prior: triangular bump at bin 15 (GF=0).
@@ -51,7 +73,7 @@ proc gfToIndex(gf: float): int {.inline.} =
proc indexToGF(idx: int): float {.inline.} =
float(idx) / float(GFBins - 1) * 2.0 - 1.0
proc peakBin(g: GFGun): int =
proc peakBin*(g: GFGun): int =
var best = 0
for i in 1..<GFBins:
if g.bins[i] > g.bins[best]:
@@ -73,18 +95,9 @@ proc binForPower(power: float): int {.inline.} =
return i
-1
proc takeOldestWave(g: var GFGun, binIdx: int): (bool, Wave) =
## Pop the oldest unresolved wave for this bin (O(1) amortized via waveHead).
## Returns (false, default) when the bin's queue is empty.
if binIdx < 0 or g.waveHead[binIdx] >= g.waves[binIdx].len:
return (false, Wave())
result = (true, g.waves[binIdx][g.waveHead[binIdx]])
inc g.waveHead[binIdx]
# Amortized O(1): drop the consumed prefix once it dominates the queue.
if g.waveHead[binIdx] >= WaveCompactAt and
g.waveHead[binIdx] * 2 >= g.waves[binIdx].len:
g.waves[binIdx] = g.waves[binIdx][g.waveHead[binIdx] .. g.waves[binIdx].high]
g.waveHead[binIdx] = 0
proc waveSlot(fireTick, binIdx: int): int {.inline.} =
## Exact (fireTick, powerBin) key -> ring slot (same scheme as tsetlin.nim).
((fireTick * len(vb.PowerBins)) + binIdx) mod WaveRingSlots
proc predict*(g: var GFGun, state: WorldState, bulletSpeed: float): GunPrediction =
if bulletSpeed <= 0.0:
@@ -104,10 +117,14 @@ proc predict*(g: var GFGun, state: WorldState, bulletSpeed: float): GunPredictio
# call for the selected bin lands on the same tick and reuses the queued wave.
let binIdx = binForSpeed(bulletSpeed)
if binIdx >= 0 and g.waveStoredTick[binIdx] != state.tick:
g.waves[binIdx].add Wave(
let slot = waveSlot(state.tick, binIdx)
g.waves[slot] = Wave(
fireX: state.selfX,
fireY: state.selfY,
fireBearing: f.bearing,
fireTick: state.tick,
bin: binIdx,
alive: true,
)
g.waveStoredTick[binIdx] = state.tick
inc g.wavePushes
@@ -119,7 +136,7 @@ proc predict*(g: var GFGun, state: WorldState, bulletSpeed: float): GunPredictio
let py = state.selfY + sin(gfAngle) * f.dist
when DebugGF:
echo fmt"[gf-dbg] predict: peakGF={peakGF:.2f} peakBin={peak} mea={radToDeg(mea):.1f}° aimAngle={radToDeg(gfAngle):.1f}° waves={g.waves[binIdx].len}"
echo fmt"[gf-dbg] predict: peakGF={peakGF:.2f} peakBin={peak} mea={radToDeg(mea):.1f}° aimAngle={radToDeg(gfAngle):.1f}° tick={state.tick}"
GunPrediction(
x: clamp(px, BotRadius, state.arenaWidth - BotRadius),
@@ -127,15 +144,25 @@ proc predict*(g: var GFGun, state: WorldState, bulletSpeed: float): GunPredictio
)
proc onResult*(g: var GFGun, e: FeedbackEvent) =
## Called when a virtual bullet resolves. Pop the OLDEST unresolved wave from
## this bullet's own power-bin queue, compute the actual GF, and smooth-add it.
let binIdx = binForPower(e.bulletPower)
## Called when a virtual bullet resolves. Look up the wave with this event's
## exact (fireTick, powerBin), compute the actual GF, and smooth-add it. A
## missing key is counted, never silently mislabelled.
let binIdx =
if e.powerBin >= 0 and e.powerBin < len(vb.PowerBins): e.powerBin
else: binForPower(e.bulletPower)
if binIdx < 0: return
let (found, w) = g.takeOldestWave(binIdx)
if not found:
let slot = waveSlot(e.fireTick, binIdx)
var w = addr g.waves[slot]
if not w.alive:
inc g.waveStarved
return
if w.fireTick != e.fireTick:
# Ring slot collision: impossible while the ring period exceeds max flight.
inc g.waveMispaired
inc g.waveStarved
return
inc g.waveResolved
# Recompute mea from the actual bullet power (correct per-bin, not the cached first-bin mea)
let speed = bulletSpeed(e.bulletPower)
@@ -160,3 +187,5 @@ proc onResult*(g: var GFGun, e: FeedbackEvent) =
for i in 0..<GFBins:
let dist = abs(i - centerIdx)
g.bins[i] += 1.0 / float(1 + dist)
w.alive = false
+32 -18
View File
@@ -14,6 +14,7 @@ const
KernelW = 0.3 # Gaussian kernel width multiplier
DensityBins = 60 # scan resolution for peak-GF search
NFeat = 8 # feature vector length (see buildFeatures)
KNNWaveRingSlots = 1024 # exact (fireTick, powerBin) ring; see guess_factor.nim
type
Obs = object
@@ -23,15 +24,18 @@ type
KNNWave = object
fireX, fireY: float
fireBearing: float
fireTick: int
bin: int
alive: bool
feat: array[NFeat, float]
KNNGun* = object
obs: seq[Obs]
obsHead: int # ring-buffer write index
# One wave queue per power bin; matched on bulletSpeed / bulletPower so a
# resolved bullet only ever learns from a wave fired with the same power.
waves: array[len(vb.PowerBins), seq[KNNWave]]
waveHead: array[len(vb.PowerBins), int] # O(1) pop cursor
# Exact (fireTick, powerBin)-keyed ring; a resolved bullet is matched to the
# wave it actually fired, no matter how many other shots resolved first.
# Heap-backed (seq): see guess_factor.nim.
waves: seq[KNNWave]
waveStoredTick: array[len(vb.PowerBins), int] # last tick a wave was queued for this bin
# per-tick cache
cachedTick: int
@@ -46,12 +50,15 @@ type
timeSinceDirChange: int
wavePushes*: int
waveStarved*: int
waveResolved*: int
waveMispaired*: int # ring-slot collision (impossible by design)
debugGraphics*: bool
proc initKNNGun*(): KNNGun =
result.cachedTick = -1
result.lastDirection = 1.0
result.debugGraphics = false
result.waves = newSeq[KNNWave](KNNWaveRingSlots)
for b in 0..<len(vb.PowerBins):
result.waveStoredTick[b] = -1
for i in 0..<NFeat:
@@ -141,16 +148,9 @@ proc binForPower(power: float): int {.inline.} =
return i
-1
proc takeOldestWave(g: var KNNGun, binIdx: int): (bool, KNNWave) =
## Pop the oldest unresolved wave for this bin (O(1) amortized via waveHead).
if binIdx < 0 or g.waveHead[binIdx] >= g.waves[binIdx].len:
return (false, KNNWave())
result = (true, g.waves[binIdx][g.waveHead[binIdx]])
inc g.waveHead[binIdx]
if g.waveHead[binIdx] >= 64 and
g.waveHead[binIdx] * 2 >= g.waves[binIdx].len:
g.waves[binIdx] = g.waves[binIdx][g.waveHead[binIdx] .. g.waves[binIdx].high]
g.waveHead[binIdx] = 0
proc waveSlot(fireTick, binIdx: int): int {.inline.} =
## Exact (fireTick, powerBin) key -> ring slot (same scheme as tsetlin.nim).
((fireTick * len(vb.PowerBins)) + binIdx) mod KNNWaveRingSlots
# ── Gun interface ─────────────────────────────────────────────────────────────
@@ -186,6 +186,7 @@ proc predict*(g: var KNNGun, state: WorldState, bulletSpd: float): GunPrediction
fireX: state.selfX,
fireY: state.selfY,
fireBearing: bearing,
fireTick: state.tick,
feat: feat,
)
g.lastSpeed = state.enemySpeed
@@ -202,7 +203,10 @@ proc predict*(g: var KNNGun, state: WorldState, bulletSpd: float): GunPrediction
# differs per bin); override the shared per-tick template here.
var w = g.tickWave
w.fireBearing = f.bearing
g.waves[binIdx].add w
w.fireTick = state.tick
w.bin = binIdx
w.alive = true
g.waves[waveSlot(state.tick, binIdx)] = w
g.waveStoredTick[binIdx] = state.tick
inc g.wavePushes
@@ -298,12 +302,20 @@ proc predict*(g: var KNNGun, state: WorldState, bulletSpd: float): GunPrediction
)
proc onResult*(g: var KNNGun, e: FeedbackEvent) =
let binIdx = binForPower(e.bulletPower)
let binIdx =
if e.powerBin >= 0 and e.powerBin < len(vb.PowerBins): e.powerBin
else: binForPower(e.bulletPower)
if binIdx < 0: return
let (found, w) = g.takeOldestWave(binIdx)
if not found:
let slot = waveSlot(e.fireTick, binIdx)
var w = addr g.waves[slot]
if not w.alive:
inc g.waveStarved
return
if w.fireTick != e.fireTick:
inc g.waveMispaired
inc g.waveStarved
return
inc g.waveResolved
let speed = bulletSpeed(e.bulletPower)
let mea = arcsin(clamp(8.0 / speed, -1.0, 1.0))
@@ -323,3 +335,5 @@ proc onResult*(g: var KNNGun, e: FeedbackEvent) =
# ring buffer
g.obs[g.obsHead] = Obs(feat: w.feat, gf: gf)
g.obsHead = (g.obsHead + 1) mod MaxObs
w.alive = false
+149
View File
@@ -0,0 +1,149 @@
## Task 1 + Task 3 measurement: wave-pairing audit and before/after hit rates for
## the three learned GF guns (GuessFactor / DecayGF / KNN).
##
## Replays the committed DrussGT fixtures through the REAL VirtualTracker, exactly
## as common_libs/gun_harness/offline_range.replayFixture does, but keeps handles
## to the concrete guns so it can read their pairing-audit counters and dump the
## raw per-bullet hit booleans for a later bullet-level permutation test.
##
## Run:
## nim c -r --path:common_libs common_libs/tests/audit_wave_pairing.nim <tag> [metric]
## tag = label written into /tmp/wavepair_<tag>_<metric>.txt
## metric = path (default, shipped) | point | both
import std/[os, math, strformat, tables, algorithm, random]
import gun_harness/gun_interface
import gun_harness/virtual_bullets as vb
import gun_harness/offline_range
import guns/guess_factor
import guns/decay_gf
import guns/knn_gun
const fixturesDir = currentSourcePath().parentDir.parentDir.parentDir / "tools" / "fixtures"
const GunNames = ["GuessFactor", "DecayGF", "KNN"]
type
Ref[G] = ref object
g: G
proc mkRef[G](v: G): Ref[G] = Ref[G](g: v)
proc driver[G](name: string, r: Ref[G]): GunDriver =
result.name = name
result.predictCb = proc(s: WorldState, sp: float): GunPrediction = r.g.predict(s, sp)
result.resultCb = proc(e: FeedbackEvent) = r.g.onResult(e)
result.readyCb = nil
proc collectOne(fx: Fixture, drivers: seq[GunDriver], metric: BulletMetric,
perGunHits: ref seq[seq[bool]]): seq[GunReport] =
let tid = fx.enemyId
var tracker = vb.initTracker(drivers.len, metric)
for si in 0..<fx.states.len:
let state = fx.states[si]
for gi in 0..<drivers.len:
var preds: array[len(PowerBins), GunPrediction]
for i in 0..<len(PowerBins):
preds[i] = drivers[gi].predictCb(state, bulletSpeed(PowerBins[i]))
tracker.spawnBullets(gi, preds, state, tid)
let act = fx.states[si]
var enemyPositions: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]]
var lst = act.tick
if si < fx.lastSeen.len and fx.lastSeen[si] >= 0: lst = fx.lastSeen[si]
enemyPositions[tid] = (x: act.enemyX, y: act.enemyY, lastSeenTick: lst, alive: true)
let dref = drivers
tracker.tickBullets(state, enemyPositions,
proc(gunId: GunId, binIdx: int, e: FeedbackEvent) =
dref[gunId].resultCb(e)
perGunHits[gunId].add e.hit)
let fit = tracker.fitnessFor(tid)
for gi in 0..<drivers.len:
var r = GunReport(name: drivers[gi].name)
for binIdx in 0..<len(PowerBins):
let fw = fit[gi].bins[binIdx]
let n = min(fw.count, WindowSize)
for k in 0..<n:
if fw.hits[k]: inc r.hits
r.shots += n
result.add r
proc rate(h, n: int): float = (if n == 0: 0.0 else: h.float / n.float * 100.0)
proc runMetric(mname: string, metric: BulletMetric, tag: string) =
echo "################################################################"
echo "### METRIC = ", mname, " (", metric, ")"
echo "################################################################"
var pooled = [0, 0, 0]
var pooledN = [0, 0, 0]
var perGunHits: seq[seq[bool]]
perGunHits.setLen(3)
var perFixtureRate: array[3, seq[float]]
var files: seq[string]
for f in walkFiles(fixturesDir / "tr_drussgt_*.jsonl"): files.add f
for f in walkFiles(fixturesDir / "drussgt_*.jsonl"): files.add f
files.sort()
# Totals accumulated from FRESH guns per fixture (no cross-fixture learning).
var totResolved, totMispaired, totStarved, totPushes: array[3, int]
for path in files:
var fx = loadFixture(path)
if fx.states.len < 50: continue
let gfRef = mkRef(initGFGun())
let decRef = mkRef(initDecayGFGun())
let knnRef = mkRef(initKNNGun())
let drivers = @[driver("GuessFactor", gfRef), driver("DecayGF", decRef),
driver("KNN", knnRef)]
let thisHits = new(seq[seq[bool]])
thisHits[].setLen(3)
let reps = collectOne(fx, drivers, metric, thisHits)
var line = fmt"{extractFilename(path):<42}"
for gi in 0..<3:
let r = reps[gi]
pooled[gi] += r.hits; pooledN[gi] += r.shots
perFixtureRate[gi].add rate(r.hits, r.shots)
for h in thisHits[gi]: perGunHits[gi].add h
line.add fmt" {GunNames[gi]}={r.hits:>4}/{r.shots:<4}"
echo line
totResolved[0] += gfRef.g.waveResolved; totMispaired[0] += gfRef.g.waveMispaired
totMispaired[0] += gfRef.g.waveMispaired; totStarved[0] += gfRef.g.waveStarved
totPushes[0] += gfRef.g.wavePushes
totResolved[1] += decRef.g.waveResolved; totMispaired[1] += decRef.g.waveMispaired
totStarved[1] += decRef.g.waveStarved
totPushes[1] += decRef.g.wavePushes
totResolved[2] += knnRef.g.waveResolved; totMispaired[2] += knnRef.g.waveMispaired
totStarved[2] += knnRef.g.waveStarved
totPushes[2] += knnRef.g.wavePushes
echo ""
echo "── pooled hit rate (", mname, ") ──"
for gi in 0..<3:
echo fmt"{GunNames[gi]:<12} {pooled[gi]:>5}/{pooledN[gi]:<6} {rate(pooled[gi], pooledN[gi]):>6.2f}% per-fixture min/max {min(perFixtureRate[gi]):.1f}/{max(perFixtureRate[gi]):.1f}"
echo ""
echo "── pairing audit (", mname, ") ──"
echo "gun resolved mispaired mispair% starved pushes"
for gi, name in GunNames:
echo fmt"{name:<12} {totResolved[gi]:>8} {totMispaired[gi]:>10} {rate(totMispaired[gi], totResolved[gi]):>8.2f} {totStarved[gi]:>7} {totPushes[gi]:>6}"
# Dump per-gun hit booleans for the cross-build permutation test.
for gi in 0..<3:
let path = fmt"/tmp/wavepair_{tag}_{mname}_{GunNames[gi]}.txt"
var f = open(path, fmWrite)
defer: f.close()
for h in perGunHits[gi]:
f.writeLine(if h: "1" else: "0")
echo fmt"dumped {perGunHits[gi].len} outcomes -> {path}"
proc main() =
let tag = if paramCount() >= 1: paramStr(1) else: "run"
let metricArg = if paramCount() >= 2: paramStr(2) else: "path"
case metricArg
of "point": runMetric("point", bmPoint, tag)
of "both": (runMetric("path", bmPath, tag); runMetric("point", bmPoint, tag))
else: runMetric("path", bmPath, tag)
echo "\ndone."
when isMainModule:
randomize(1)
main()
+146
View File
@@ -0,0 +1,146 @@
## Task 3 analysis: compare the FIFO (before) and fireTick-keyed (after) pairing
## for the three learned GF guns.
##
## Two analyses:
## 1. Per-fixture (per-run) paired comparison — the repo's convention. The 10
## DrussGT fixtures are the independent runs; the paired delta is
## after - before. An exact sign-flip permutation test (2^10 = 1024 sign
## patterns) gives the p-value, and the per-run ranges give the overlap.
## 2. Bullet-level two-sample permutation test on the raw hit booleans dumped
## by audit_wave_pairing.nim. ANTI-CONSERVATIVE: bullets within a fixture
## share a trajectory and are correlated, so treat this as an upper bound on
## significance, not the headline.
##
## Run: nim c -r common_libs/tests/compare_pairing.nim
import std/[math, strformat, random, os, strutils]
const
FixtureNames = ["drussgt_vs_corners", "drussgt_vs_crazy", "drussgt_vs_drussgt",
"drussgt_vs_ramfire", "drussgt_vs_spinbot",
"tr_drussgt_vs_corners", "tr_drussgt_vs_crazy",
"tr_drussgt_vs_modularbot", "tr_drussgt_vs_modularbot_shield",
"tr_drussgt_vs_spinbot"]
ShotsPerFixture = 400 # WindowSize(100) x 4 power bins
GunNames = ["GuessFactor", "DecayGF", "KNN"]
# Captured from `audit_wave_pairing.nim <tag> both` (fresh guns per fixture;
# hits out of 400). Deterministic guns -> reproducible.
BeforePath: array[3, array[10, int]] = [
[36, 139, 23, 202, 233, 155, 9, 47, 42, 42], # GuessFactor
[61, 108, 8, 204, 232, 118, 9, 44, 108, 60], # DecayGF
[16, 94, 33, 188, 176, 123, 12, 33, 19, 37], # KNN
]
AfterPath: array[3, array[10, int]] = [
[36, 139, 23, 202, 226, 155, 12, 44, 42, 42],
[61, 105, 19, 213, 226, 140, 9, 47, 108, 42],
[38, 94, 38, 193, 158, 129, 11, 29, 25, 37],
]
BeforePoint: array[3, array[10, int]] = [
[5, 33, 15, 59, 59, 36, 0, 26, 3, 21],
[1, 8, 2, 37, 52, 48, 0, 0, 33, 6],
[0, 19, 14, 53, 34, 39, 13, 15, 5, 25],
]
AfterPoint: array[3, array[10, int]] = [
[5, 33, 15, 59, 59, 36, 0, 28, 3, 21],
[1, 9, 2, 37, 52, 48, 0, 0, 45, 6],
[0, 20, 15, 48, 33, 38, 13, 14, 5, 25],
]
proc sum(a: array[10, int]): int =
for x in a: result += x
proc meanPct(a: array[10, int]): float = sum(a).float / 10.0 / ShotsPerFixture.float * 100.0
proc minPct(a: array[10, int]): float =
result = 1e9
for x in a: result = min(result, x.float / ShotsPerFixture.float * 100.0)
proc maxPct(a: array[10, int]): float =
result = -1e9
for x in a: result = max(result, x.float / ShotsPerFixture.float * 100.0)
proc signFlipP(before, after: array[10, int]): tuple[p, obsMeanPp: float, nPos, nNeg, nZero: int] =
## Exact sign-flip permutation test on the paired per-fixture deltas.
var deltas: array[10, float]
for i in 0..<10:
deltas[i] = (after[i] - before[i]).float / ShotsPerFixture.float * 100.0
if deltas[i] > 1e-9: inc result.nPos
elif deltas[i] < -1e-9: inc result.nNeg
else: inc result.nZero
result.obsMeanPp += deltas[i] / 10.0
let obs = abs(result.obsMeanPp)
var ge = 0
for mask in 0..<(1 shl 10):
var m = 0.0
for i in 0..<10:
let s = if ((mask shr i) and 1) == 1: -1.0 else: 1.0
m += s * deltas[i] / 10.0
if abs(m) >= obs - 1e-12: inc ge
result.p = ge.float / 1024.0
proc loadDump(path: string): seq[bool] =
if not fileExists(path):
return @[]
for line in lines(path):
let s = line.strip()
if s.len == 0: continue
result.add (s == "1")
proc zTest(a, b: seq[bool]): tuple[p, diffPp, z: float] =
## Two-proportion z-test (analytic; the permutation equivalent is exact but
## 344k-element shuffles are needlessly slow). ANTI-CONSERVATIVE because the
## bullets are correlated within a fixture.
if a.len == 0 or b.len == 0: return (1.0, 0.0, 0.0)
var ha, hb: int
for x in a: (if x: inc ha)
for x in b: (if x: inc hb)
let p1 = ha.float / a.len.float
let p2 = hb.float / b.len.float
result.diffPp = (p2 - p1) * 100.0
let p = (ha + hb).float / (a.len + b.len).float
let se = sqrt(max(1e-30, p * (1.0 - p) * (1.0/a.len.float + 1.0/b.len.float)))
result.z = (p2 - p1) / se
result.p = erfc(abs(result.z) / sqrt(2.0))
proc reportMetric(mname: string,
before, after: array[3, array[10, int]]) =
echo "══════════════════════════════════════════════════════════════════"
echo " METRIC = ", mname
echo "══════════════════════════════════════════════════════════════════"
for gi in 0..<2:
let b = before[gi]
let a = after[gi]
echo fmt"{GunNames[gi]}:"
echo fmt" before {sum(b):>4}/{ShotsPerFixture*10} = {meanPct(b):5.2f}% per-run {minPct(b):5.2f}..{maxPct(b):5.2f}%"
echo fmt" after {sum(a):>4}/{ShotsPerFixture*10} = {meanPct(a):5.2f}% per-run {minPct(a):5.2f}..{maxPct(a):5.2f}%"
let sf = signFlipP(b, a)
echo fmt" delta {meanPct(a)-meanPct(b):+5.2f}pp paired sign-flip permutation p={sf.p:.3f} (+{sf.nPos}/-{sf.nNeg}/0:{sf.nZero} of 10)"
let b = before[2]
let a = after[2]
echo fmt"{GunNames[2]}:"
echo fmt" before {sum(b):>4}/{ShotsPerFixture*10} = {meanPct(b):5.2f}% per-run {minPct(b):5.2f}..{maxPct(b):5.2f}%"
echo fmt" after {sum(a):>4}/{ShotsPerFixture*10} = {meanPct(a):5.2f}% per-run {minPct(a):5.2f}..{maxPct(a):5.2f}%"
let sf = signFlipP(b, a)
echo fmt" delta {meanPct(a)-meanPct(b):+5.2f}pp paired sign-flip permutation p={sf.p:.3f} (+{sf.nPos}/-{sf.nNeg}/0:{sf.nZero} of 10)"
echo ""
proc main() =
randomize(12345)
reportMetric("bmPath (shipped)", BeforePath, AfterPath)
reportMetric("bmPoint", BeforePoint, AfterPoint)
echo "══════════════════════════════════════════════════════════════════"
echo " bullet-level two-proportion z-test (bmPath dumps) — ANTI-CONSERVATIVE"
echo "══════════════════════════════════════════════════════════════════"
for gi in 0..<3:
let bf = loadDump(fmt"/tmp/wavepair_before_path_{GunNames[gi]}.txt")
let af = loadDump(fmt"/tmp/wavepair_after_path_{GunNames[gi]}.txt")
let r = zTest(bf, af)
var hb, ha: int
for x in bf: (if x: inc hb)
for x in af: (if x: inc ha)
echo fmt"{GunNames[gi]:<12} before {hb:>6}/{bf.len:<6} {hb.float/bf.len.float*100:5.2f}% after {ha:>6}/{af.len:<6} {ha.float/af.len.float*100:5.2f}% diff {r.diffPp:+5.2f}pp z={r.z:+5.2f} p={r.p:.2g}"
when isMainModule:
main()
+170
View File
@@ -0,0 +1,170 @@
## Pure unit test for the fireTick-keyed wave pairing in the learned GF guns
## (guess_factor.nim, decay_gf.nim, knn_gun.nim). No battle, no Java, no fixtures.
##
## The defect this pins: the guns used to pop the OLDEST queued wave on every
## resolution (FIFO). Under the shipped bmPath metric a later-fired bullet can
## resolve first, so the outcome was attached to the wrong wave. The fix keys
## each wave by (fireTick, powerBin) exactly, as tsetlin.nim / tm_selector.nim do.
##
## The discriminating property is ORDER-INDEPENDENCE: with exact keying, learning
## from the same set of (wave, resolution) pairs must be identical no matter what
## order the resolutions arrive in. Under FIFO, resolving in reverse order pairs
## each event with the wrong wave and the learned state diverges.
##
## Run: nim c -r --path:common_libs common_libs/tests/test_wave_pairing.nim
import std/math
import gun_harness/gun_interface
import guns/guess_factor
import guns/decay_gf
import guns/knn_gun
var failures = 0
proc check(name: string, ok: bool) =
if ok: echo "PASS: ", name
else: echo "FAIL: ", name; inc failures
const Spd = 17.0 # bulletSpeed(PowerBins[0]); all tests use power bin 0
const N = 6 # 6 waves, enough for KNN to leave its <5 cold-start fallback
var Px: array[N, float]
var Py: array[N, float]
for i in 0..<N:
let a = float(i) * PI / 3.0
Px[i] = 100.0 + 300.0 * cos(a)
Py[i] = 100.0 + 300.0 * sin(a)
proc ws(tick: int, ex, ey: float): WorldState =
WorldState(selfX: 100.0, selfY: 100.0, selfSpeed: 0.0, selfHeading: 0.0,
selfRadarHeading: 0.0, selfEnergy: 100.0,
enemyX: ex, enemyY: ey, enemySpeed: 0.0, enemyHeading: 0.0,
enemyEnergy: 100.0,
arenaWidth: 800.0, arenaHeight: 600.0, tick: tick)
proc ev(tick: int, ex, ey: float): FeedbackEvent =
FeedbackEvent(prediction: GunPrediction(x: ex, y: ey),
actualX: ex, actualY: ey, bulletPower: 1.0,
fireTick: tick, powerBin: 0, missDistance: 0.0, hit: true)
proc samePoint(a, b: GunPrediction): bool =
abs(a.x - b.x) < 1e-6 and abs(a.y - b.y) < 1e-6
# ── GuessFactor ───────────────────────────────────────────────────────────────
proc testGF() =
var fwd = initGFGun()
var rev = initGFGun()
for i in 0..<N:
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
check "GF: reverse resolution learns the SAME state as forward (order-independent)",
fwd.peakBin() == rev.peakBin() and samePoint(qf, qr)
check "GF: every wave resolved exactly once in both orders",
fwd.waveResolved == N and rev.waveResolved == N
check "GF: no mispair / no starve",
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
fwd.waveStarved == 0 and rev.waveStarved == 0
proc testGFMissingWave() =
var g = initGFGun()
discard g.predict(ws(0, Px[0], Py[0]), Spd)
g.onResult(ev(999, Px[0], Py[0]))
check "GF: unknown fireTick -> starved, nothing resolved",
g.waveStarved == 1 and g.waveResolved == 0
g.onResult(ev(0, Px[0], Py[0]))
check "GF: the real wave still resolves after an unknown-tick event",
g.waveResolved == 1 and g.waveStarved == 1
g.onResult(ev(0, Px[0], Py[0]))
check "GF: a second resolve of a consumed wave is counted, not applied",
g.waveResolved == 1 and g.waveStarved == 2
# ── DecayGF ───────────────────────────────────────────────────────────────────
proc testDecayGF() =
var fwd = initDecayGFGun()
var rev = initDecayGFGun()
for i in 0..<N:
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
check "DecayGF: reverse resolution learns the SAME state as forward (order-independent)",
fwd.peakBin() == rev.peakBin() and samePoint(qf, qr)
check "DecayGF: every wave resolved exactly once in both orders",
fwd.waveResolved == N and rev.waveResolved == N
check "DecayGF: no mispair / no starve",
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
fwd.waveStarved == 0 and rev.waveStarved == 0
proc testDecayGFMissingWave() =
var g = initDecayGFGun()
discard g.predict(ws(0, Px[0], Py[0]), Spd)
g.onResult(ev(999, Px[0], Py[0]))
check "DecayGF: unknown fireTick -> starved, nothing resolved",
g.waveStarved == 1 and g.waveResolved == 0
g.onResult(ev(0, Px[0], Py[0]))
check "DecayGF: the real wave still resolves after an unknown-tick event",
g.waveResolved == 1 and g.waveStarved == 1
# ── KNN ───────────────────────────────────────────────────────────────────────
proc testKNN() =
var fwd = initKNNGun()
var rev = initKNNGun()
for i in 0..<N:
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
check "KNN: reverse resolution produces the SAME query prediction as forward",
samePoint(qf, qr)
check "KNN: every wave resolved exactly once in both orders",
fwd.waveResolved == N and rev.waveResolved == N
check "KNN: no mispair / no starve",
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
fwd.waveStarved == 0 and rev.waveStarved == 0
proc testKNNMissingWave() =
var g = initKNNGun()
discard g.predict(ws(0, Px[0], Py[0]), Spd)
g.onResult(ev(999, Px[0], Py[0]))
check "KNN: unknown fireTick -> starved, nothing resolved",
g.waveStarved == 1 and g.waveResolved == 0
g.onResult(ev(0, Px[0], Py[0]))
check "KNN: the real wave still resolves after an unknown-tick event",
g.waveResolved == 1 and g.waveStarved == 1
# ── the ring slot is the (fireTick, bin) key ──────────────────────────────────
proc testNonFifoResolveOrder() =
## Explicit non-FIFO sequence: fire 0,1,2; resolve 2,0,1. All three must find
## their own wave (0 starved), which FIFO cannot distinguish but exact keying
## must satisfy alongside the order-independence property above.
var g = initGFGun()
for i in 0..2: discard g.predict(ws(i, Px[i], Py[i]), Spd)
g.onResult(ev(2, Px[2], Py[2]))
g.onResult(ev(0, Px[0], Py[0]))
g.onResult(ev(1, Px[1], Py[1]))
check "GF: resolve order 2,0,1 -> all three waves found, none starved",
g.waveResolved == 3 and g.waveStarved == 0
testGF()
testGFMissingWave()
testDecayGF()
testDecayGFMissingWave()
testKNN()
testKNNMissingWave()
testNonFifoResolveOrder()
if failures > 0:
echo "\n", failures, " check(s) FAILED"
quit(1)
echo "\nAll wave-pairing checks passed."