wave pairing: 36-58% of GF/DecayGF/KNN learning samples were MISLABELLED
The audit inferred (from code) that GF/DecayGF/KNN pop the OLDEST wave on resolution, while under bmPath bullets leave the arena in NON-FIFO order - so an outcome could be attached to the wrong wave. It also noted that `starved=0` does NOT rule this out. Both halves are now MEASURED. MISPAIRING RATE (10 DrussGT fixtures, real VirtualTracker, 344k resolutions/gun): gun bmPath mispair label err bmPoint mispair label err GuessFactor 36.48% 19.39% 18.24% 7.62% DecayGF 36.85% 19.52% 20.57% 8.64% KNN 57.91% 27.63% 29.75% 11.58% (starved = 0 everywhere, exactly as the audit predicted) So ~1 in 5 GF/DecayGF learning samples and ~1 in 4 KNN samples carried a WRONG guess-factor bin. This is a material corruption of the learning signal. FIX: the same fireTick-keyed ring scheme `tsetlin.nim`/`tm_selector.nim` already use - `slot = (fireTick*4 + bin) mod 1024` (period 256 ticks, longer than the ~91-tick max flight), looked up by exact key. Public interfaces unchanged; added `waveResolved`/`waveMispaired` integrity counters. AFTER: mispaired = 0 and starved = 0, both metrics, all three guns. EFFECT ON HIT RATE: SMALL AND NOT SIGNIFICANT. bmPath 4000 samples/gun: GuessFactor 23.20% -> 23.02% (-0.18pp, per-run sign-flip p=0.750) DecayGF 23.80% -> 24.25% (+0.45pp, p=0.625) KNN 18.27% -> 18.80% (+0.53pp, p=0.547) bmPoint: +0.05 / +0.33 / -0.15pp, p = 1.00 / 0.50 / 0.50. Per-run ranges overlap almost completely. A bullet-level z-test is anti-conservative (bullets within a fixture share a trajectory) and its KNN p=1.9e-16 cannot be trusted given ~10 effective independent runs. PLAIN READING: this is a CORRECTNESS fix, not a measurable hit-rate win. It removes a 36-58% mislabelling of the learning signal; the point estimates move by at most ~0.5pp, within run-to-run noise. Stated plainly rather than oversold. A REGRESSION IT CAUGHT IN ITSELF (and this explains the SIGSEGV another job saw and correctly attributed to a concurrent knn_gun.nim rewrite): the first implementation put an inline `array[1024, KNNWave]` (~100KB) inside each gun, which overflowed the default 8MB stack and made `test_power_selection` SIGSEGV. Causation was proven by stashing only the three gun files (test passed), then fixed by making the rings heap-backed `seq`. Verified: `test_power_selection` 3 PASS on the default stack, and zero inline `array[1024]` remain. Guards: test_wave_pairing 17 (new, pure), test_gun_harness 39, test_vbullet_metric 11, test_power_selection 3, test_adaptive_radar 41, test_tfil_ring_weights 24, test_power_policy 26, test_ram_decision 28. ModularBot compiles. Adds audit_wave_pairing.nim and compare_pairing.nim.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
## Pure unit test for the fireTick-keyed wave pairing in the learned GF guns
|
||||
## (guess_factor.nim, decay_gf.nim, knn_gun.nim). No battle, no Java, no fixtures.
|
||||
##
|
||||
## The defect this pins: the guns used to pop the OLDEST queued wave on every
|
||||
## resolution (FIFO). Under the shipped bmPath metric a later-fired bullet can
|
||||
## resolve first, so the outcome was attached to the wrong wave. The fix keys
|
||||
## each wave by (fireTick, powerBin) exactly, as tsetlin.nim / tm_selector.nim do.
|
||||
##
|
||||
## The discriminating property is ORDER-INDEPENDENCE: with exact keying, learning
|
||||
## from the same set of (wave, resolution) pairs must be identical no matter what
|
||||
## order the resolutions arrive in. Under FIFO, resolving in reverse order pairs
|
||||
## each event with the wrong wave and the learned state diverges.
|
||||
##
|
||||
## Run: nim c -r --path:common_libs common_libs/tests/test_wave_pairing.nim
|
||||
|
||||
import std/math
|
||||
import gun_harness/gun_interface
|
||||
import guns/guess_factor
|
||||
import guns/decay_gf
|
||||
import guns/knn_gun
|
||||
|
||||
var failures = 0
|
||||
proc check(name: string, ok: bool) =
|
||||
if ok: echo "PASS: ", name
|
||||
else: echo "FAIL: ", name; inc failures
|
||||
|
||||
const Spd = 17.0 # bulletSpeed(PowerBins[0]); all tests use power bin 0
|
||||
const N = 6 # 6 waves, enough for KNN to leave its <5 cold-start fallback
|
||||
|
||||
var Px: array[N, float]
|
||||
var Py: array[N, float]
|
||||
for i in 0..<N:
|
||||
let a = float(i) * PI / 3.0
|
||||
Px[i] = 100.0 + 300.0 * cos(a)
|
||||
Py[i] = 100.0 + 300.0 * sin(a)
|
||||
|
||||
proc ws(tick: int, ex, ey: float): WorldState =
|
||||
WorldState(selfX: 100.0, selfY: 100.0, selfSpeed: 0.0, selfHeading: 0.0,
|
||||
selfRadarHeading: 0.0, selfEnergy: 100.0,
|
||||
enemyX: ex, enemyY: ey, enemySpeed: 0.0, enemyHeading: 0.0,
|
||||
enemyEnergy: 100.0,
|
||||
arenaWidth: 800.0, arenaHeight: 600.0, tick: tick)
|
||||
|
||||
proc ev(tick: int, ex, ey: float): FeedbackEvent =
|
||||
FeedbackEvent(prediction: GunPrediction(x: ex, y: ey),
|
||||
actualX: ex, actualY: ey, bulletPower: 1.0,
|
||||
fireTick: tick, powerBin: 0, missDistance: 0.0, hit: true)
|
||||
|
||||
proc samePoint(a, b: GunPrediction): bool =
|
||||
abs(a.x - b.x) < 1e-6 and abs(a.y - b.y) < 1e-6
|
||||
|
||||
# ── GuessFactor ───────────────────────────────────────────────────────────────
|
||||
|
||||
proc testGF() =
|
||||
var fwd = initGFGun()
|
||||
var rev = initGFGun()
|
||||
for i in 0..<N:
|
||||
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
|
||||
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
|
||||
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
|
||||
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
|
||||
check "GF: reverse resolution learns the SAME state as forward (order-independent)",
|
||||
fwd.peakBin() == rev.peakBin() and samePoint(qf, qr)
|
||||
check "GF: every wave resolved exactly once in both orders",
|
||||
fwd.waveResolved == N and rev.waveResolved == N
|
||||
check "GF: no mispair / no starve",
|
||||
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
|
||||
fwd.waveStarved == 0 and rev.waveStarved == 0
|
||||
|
||||
proc testGFMissingWave() =
|
||||
var g = initGFGun()
|
||||
discard g.predict(ws(0, Px[0], Py[0]), Spd)
|
||||
g.onResult(ev(999, Px[0], Py[0]))
|
||||
check "GF: unknown fireTick -> starved, nothing resolved",
|
||||
g.waveStarved == 1 and g.waveResolved == 0
|
||||
g.onResult(ev(0, Px[0], Py[0]))
|
||||
check "GF: the real wave still resolves after an unknown-tick event",
|
||||
g.waveResolved == 1 and g.waveStarved == 1
|
||||
g.onResult(ev(0, Px[0], Py[0]))
|
||||
check "GF: a second resolve of a consumed wave is counted, not applied",
|
||||
g.waveResolved == 1 and g.waveStarved == 2
|
||||
|
||||
# ── DecayGF ───────────────────────────────────────────────────────────────────
|
||||
|
||||
proc testDecayGF() =
|
||||
var fwd = initDecayGFGun()
|
||||
var rev = initDecayGFGun()
|
||||
for i in 0..<N:
|
||||
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
|
||||
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
|
||||
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
|
||||
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
|
||||
check "DecayGF: reverse resolution learns the SAME state as forward (order-independent)",
|
||||
fwd.peakBin() == rev.peakBin() and samePoint(qf, qr)
|
||||
check "DecayGF: every wave resolved exactly once in both orders",
|
||||
fwd.waveResolved == N and rev.waveResolved == N
|
||||
check "DecayGF: no mispair / no starve",
|
||||
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
|
||||
fwd.waveStarved == 0 and rev.waveStarved == 0
|
||||
|
||||
proc testDecayGFMissingWave() =
|
||||
var g = initDecayGFGun()
|
||||
discard g.predict(ws(0, Px[0], Py[0]), Spd)
|
||||
g.onResult(ev(999, Px[0], Py[0]))
|
||||
check "DecayGF: unknown fireTick -> starved, nothing resolved",
|
||||
g.waveStarved == 1 and g.waveResolved == 0
|
||||
g.onResult(ev(0, Px[0], Py[0]))
|
||||
check "DecayGF: the real wave still resolves after an unknown-tick event",
|
||||
g.waveResolved == 1 and g.waveStarved == 1
|
||||
|
||||
# ── KNN ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
proc testKNN() =
|
||||
var fwd = initKNNGun()
|
||||
var rev = initKNNGun()
|
||||
for i in 0..<N:
|
||||
discard fwd.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
discard rev.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
for i in 0..<N: fwd.onResult(ev(i, Px[i], Py[i]))
|
||||
for i in countdown(N-1, 0): rev.onResult(ev(i, Px[i], Py[i]))
|
||||
let qf = fwd.predict(ws(100, 400.0, 300.0), Spd)
|
||||
let qr = rev.predict(ws(100, 400.0, 300.0), Spd)
|
||||
check "KNN: reverse resolution produces the SAME query prediction as forward",
|
||||
samePoint(qf, qr)
|
||||
check "KNN: every wave resolved exactly once in both orders",
|
||||
fwd.waveResolved == N and rev.waveResolved == N
|
||||
check "KNN: no mispair / no starve",
|
||||
fwd.waveMispaired == 0 and rev.waveMispaired == 0 and
|
||||
fwd.waveStarved == 0 and rev.waveStarved == 0
|
||||
|
||||
proc testKNNMissingWave() =
|
||||
var g = initKNNGun()
|
||||
discard g.predict(ws(0, Px[0], Py[0]), Spd)
|
||||
g.onResult(ev(999, Px[0], Py[0]))
|
||||
check "KNN: unknown fireTick -> starved, nothing resolved",
|
||||
g.waveStarved == 1 and g.waveResolved == 0
|
||||
g.onResult(ev(0, Px[0], Py[0]))
|
||||
check "KNN: the real wave still resolves after an unknown-tick event",
|
||||
g.waveResolved == 1 and g.waveStarved == 1
|
||||
|
||||
# ── the ring slot is the (fireTick, bin) key ──────────────────────────────────
|
||||
|
||||
proc testNonFifoResolveOrder() =
|
||||
## Explicit non-FIFO sequence: fire 0,1,2; resolve 2,0,1. All three must find
|
||||
## their own wave (0 starved), which FIFO cannot distinguish but exact keying
|
||||
## must satisfy alongside the order-independence property above.
|
||||
var g = initGFGun()
|
||||
for i in 0..2: discard g.predict(ws(i, Px[i], Py[i]), Spd)
|
||||
g.onResult(ev(2, Px[2], Py[2]))
|
||||
g.onResult(ev(0, Px[0], Py[0]))
|
||||
g.onResult(ev(1, Px[1], Py[1]))
|
||||
check "GF: resolve order 2,0,1 -> all three waves found, none starved",
|
||||
g.waveResolved == 3 and g.waveStarved == 0
|
||||
|
||||
testGF()
|
||||
testGFMissingWave()
|
||||
testDecayGF()
|
||||
testDecayGFMissingWave()
|
||||
testKNN()
|
||||
testKNNMissingWave()
|
||||
testNonFifoResolveOrder()
|
||||
|
||||
if failures > 0:
|
||||
echo "\n", failures, " check(s) FAILED"
|
||||
quit(1)
|
||||
echo "\nAll wave-pairing checks passed."
|
||||
Reference in New Issue
Block a user