selector: arrival-accuracy tie-break measured NEGATIVE; randomness is load-bearing
Hypothesis under test (from the gun audit, which named the tie-band as "the lever that matters most"): `bmPath` is deliberately generous (2.3-3.6x `bmPoint`), so a gun can sit in the tied band on a ray that sweeps the target's path while its bullets ARRIVE badly. So: keep the `path`-ranked band (path beat point on real hit rate 7.43% vs 4.70%, z=5.56), but narrow the random draw inside it using a parallel `point` (arrival-accuracy) window. RESULT: NO EFFECT. Real DrussGT, ONE frozen binary (/tmp/ModularBot_tieband, md5 2c0c56e6...), env knobs only, 7 runs x 7 rounds per arm, server-side events sidecar, exact two-sided permutation test on per-run rates. arm runs shots real % dmg/run d p tbbase (shipped) 7 4128 7.17 175 -- -- tbpt path-rank + point-narrow 7 3938 7.08 165 +0.14 0.88 tbpc =commit control 7 3759 4.44 98 +2.74 0.0012 tbpt25 point margin 0.25 7 3683 5.59 119 +1.65 0.20 tbtie05 / tbtie40 (band width) 7 3937/3917 5.84/6.28 133/144 1.49/1.00 0.11/0.25 tbwin50 (SelectorWindow=50) 7 3983 6.05 139 +1.20 0.11 tbfloor10 (FloorPeakFrac=0.10) 7 3829 5.33 118 +2.12 0.11 tbpt vs base: fully overlapping ranges, p=0.88. This is a REAL null, not a dead arm - the mechanism was live, and it visibly changed the selected-gun mix (Pattern 24%->16%, Accel 6%->16%, Tsetlin ~0%->13%). CONTROL VALIDATED, AND THIS IS THE THIRD TIME: removing the random draw inside the band is SIGNIFICANTLY WORSE (4.44%, p=0.0012). Combined with the earlier hysteresis A/B (7.02% -> 5.10% for commitment) and the light-hysteresis result, the selector's per-tick randomness is now load-bearing on three independent measurements. Narrowing the band on ANY second virtual statistic has not helped. Every knob swept (band width, floor, window) is nominally worse than shipped at n=7; that is "no credible win" rather than "proven harm" (sd ~1.8pp, ~1pp resolution, underpowered). Shipped default stays `GUN_SELECTOR_TIEBREAK=off`; the feature is opt-in, fully guarded, and costs zero extra work on the default path (point windows are scored only when the mode is on). Guards: test_selector_tiebreak 19 (new, pure), test_gun_harness 39, test_vbullet_metric 11, test_adaptive_radar 41, test_tfil_ring_weights 24, test_power_policy 26, test_ram_decision 28, test_rack_membership 38, acceptance_offline_vs_online 12/12 PASS (offline path calls neither chooseFromFit nor the tie-break). STRATEGIC CONCLUSION: three selection-side attempts have now failed (hysteresis, commitment, point tie-break). The selector is at a local optimum and the remaining lever is the QUALITY OF THE GUNS, not the selection among them.
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
## Pure guard for the selector's ARRIVAL-ACCURACY TIE-BREAK
|
||||
## (`GUN_SELECTOR_TIEBREAK`, see common_libs/gun_harness/virtual_bullets.nim).
|
||||
##
|
||||
## No Java, no server, no battle:
|
||||
## nim c -r common_libs/tests/test_selector_tiebreak.nim
|
||||
##
|
||||
## Two things are pinned here:
|
||||
## 1. the RANKING rule — `chooseFromFit` narrows a `path` tie band by the
|
||||
## parallel `point` (arrival-accuracy) window while still drawing the shot
|
||||
## at random inside the narrowed band;
|
||||
## 2. the RECORDING rule — `tickBullets` fills that parallel window from the
|
||||
## exact tick the bullet reaches its aim distance, leaving the `path` score
|
||||
## untouched.
|
||||
|
||||
import std/[math, random, tables, strformat]
|
||||
import gun_harness/gun_interface
|
||||
import gun_harness/virtual_bullets
|
||||
|
||||
var failures = 0
|
||||
proc check(name: string, ok: bool) =
|
||||
if ok: echo "PASS: ", name
|
||||
else: echo "FAIL: ", name; inc failures
|
||||
|
||||
proc recordHit(fw: var FitnessWindow, hit: bool) =
|
||||
fw.hits[fw.head] = hit
|
||||
fw.head = (fw.head + 1) mod WindowSize
|
||||
inc fw.count
|
||||
|
||||
proc seed(fit: var GunFitness, binIdx, hits, misses: int, point: bool) =
|
||||
## Record `hits`/`misses` into the gun's path window (or the parallel point
|
||||
## window when `point`).
|
||||
var fw = if point: addr fit.pointBins[binIdx] else: addr fit.bins[binIdx]
|
||||
for _ in 0..<hits: recordHit(fw[], true)
|
||||
for _ in 0..<misses: recordHit(fw[], false)
|
||||
|
||||
proc mkFit(rates: openArray[tuple[pHits, pMiss, aHits, aMiss: int]]): seq[GunFitness] =
|
||||
## One gun per entry: (path hits, path misses, point hits, point misses) in
|
||||
## power bin 0. Both windows get the same >= MinObsBeforeCompete sample count.
|
||||
result = newSeq[GunFitness](rates.len)
|
||||
for i, r in rates:
|
||||
result[i].seed(0, r.pHits, r.pMiss, point = false)
|
||||
result[i].seed(0, r.aHits, r.aMiss, point = true)
|
||||
|
||||
# ── parsing / default ─────────────────────────────────────────────────────────
|
||||
|
||||
proc testParsing() =
|
||||
check "tiebreak parse: empty -> shipped default (off)", parseTieBreak("") == tbOff
|
||||
check "tiebreak parse: 'off' -> tbOff", parseTieBreak("off") == tbOff
|
||||
check "tiebreak parse: 'point' -> tbPoint", parseTieBreak("point") == tbPoint
|
||||
check "tiebreak parse: 'commit' -> tbPointCommit", parseTieBreak("commit") == tbPointCommit
|
||||
check "tiebreak parse: case/space insensitive", parseTieBreak(" PoInT ") == tbPoint
|
||||
check "tiebreak parse: unknown -> off (safe fallback, warns)",
|
||||
parseTieBreak("definitely-not-a-mode") == tbOff
|
||||
|
||||
# ── ranking rule ──────────────────────────────────────────────────────────────
|
||||
|
||||
proc testNarrowingKeepsRandomness() =
|
||||
## Three path-tied guns (10.0 / 9.5 / 9.0 %), all inside the 20% relative
|
||||
## band. Arrival accuracy is sharply different: 1 / 4 / 3 %. With a 0.5
|
||||
## relative point margin the band must lose gun 0 (0.01 < 0.5 * 0.04) but keep
|
||||
## the uniform random draw over guns 1 and 2.
|
||||
randomize(20240921)
|
||||
let fit = mkFit([(100, 0, 1, 99), (95, 5, 40, 60), (90, 10, 30, 70)])
|
||||
var seen: array[3, int]
|
||||
var narrowed = 0
|
||||
for _ in 0..<400:
|
||||
var d: SelectorDiag
|
||||
let g = chooseFromFit(fit, addr d, mode = smRelative,
|
||||
tieBreak = tbPoint, pointTieMargin = 0.5)
|
||||
if g >= 0 and g < 3: inc seen[g]
|
||||
if d.pointTieFired and d.pointTiedCount == 2: inc narrowed
|
||||
check "tiebreak(point): the bad-arrival gun is never drawn", seen[0] == 0
|
||||
let bothDrawn = seen[1] > 0 and seen[2] > 0
|
||||
check "tiebreak(point): both point-accurate tied guns are still drawn (randomness kept)", bothDrawn
|
||||
check "tiebreak(point): the band really was narrowed (diag)", narrowed == 400
|
||||
echo fmt" point draws: gun0={seen[0]} gun1={seen[1]} gun2={seen[2]} (n=400)"
|
||||
|
||||
proc testOffIsBaseline() =
|
||||
## With the tie-break OFF the identical fitness set must still expose the
|
||||
## bad-arrival gun — i.e. the shipped behaviour is unchanged.
|
||||
randomize(20240921)
|
||||
let fit = mkFit([(100, 0, 1, 99), (95, 5, 40, 60), (90, 10, 30, 70)])
|
||||
var seen: array[3, int]
|
||||
for _ in 0..<400:
|
||||
let g = chooseFromFit(fit, nil, mode = smRelative, tieBreak = tbOff)
|
||||
if g >= 0 and g < 3: inc seen[g]
|
||||
check "tiebreak(off): the bad-arrival gun is still drawn (no behaviour change)",
|
||||
seen[0] > 0 and seen[1] > 0 and seen[2] > 0
|
||||
echo fmt" off draws: gun0={seen[0]} gun1={seen[1]} gun2={seen[2]} (n=400)"
|
||||
|
||||
proc testCommitIsDeterministic() =
|
||||
## Control arm: `tbPointCommit` removes the random draw and takes the best
|
||||
## in-band arrival rate.
|
||||
randomize(1)
|
||||
let fit = mkFit([(100, 0, 1, 99), (95, 5, 40, 60), (90, 10, 30, 70)])
|
||||
var allOne = true
|
||||
for _ in 0..<200:
|
||||
if chooseFromFit(fit, nil, mode = smRelative,
|
||||
tieBreak = tbPointCommit) != 1: allOne = false
|
||||
check "tiebreak(commit): deterministically returns the best arrival gun", allOne
|
||||
|
||||
proc testColdPointIsNoOp() =
|
||||
## With no point samples at all the tie-break must leave the band untouched.
|
||||
randomize(7)
|
||||
let fit = mkFit([(100, 0, 0, 0), (95, 5, 0, 0), (90, 10, 0, 0)])
|
||||
var seen: array[3, int]
|
||||
var fired = 0
|
||||
for _ in 0..<400:
|
||||
var d: SelectorDiag
|
||||
let g = chooseFromFit(fit, addr d, mode = smRelative, tieBreak = tbPoint)
|
||||
if g >= 0 and g < 3: inc seen[g]
|
||||
if d.pointTieFired: inc fired
|
||||
check "tiebreak(cold point data): band untouched, all tied guns drawn",
|
||||
seen[0] > 0 and seen[1] > 0 and seen[2] > 0
|
||||
check "tiebreak(cold point data): never reports a narrowing", fired == 0
|
||||
|
||||
proc testColdGunIsKept() =
|
||||
## A gun with ZERO point samples cannot be judged and must be KEPT, while a
|
||||
## gun with a measured bad arrival rate is dropped.
|
||||
randomize(11)
|
||||
let fit = mkFit([(100, 0, 0, 0), (95, 5, 40, 60), (90, 10, 1, 99)])
|
||||
var seen: array[3, int]
|
||||
for _ in 0..<400:
|
||||
let g = chooseFromFit(fit, nil, mode = smRelative,
|
||||
tieBreak = tbPoint, pointTieMargin = 0.5)
|
||||
if g >= 0 and g < 3: inc seen[g]
|
||||
check "tiebreak: a cold-on-point gun is kept (not judged bad)", seen[0] > 0
|
||||
check "tiebreak: a measured bad-arrival gun is dropped", seen[2] == 0
|
||||
check "tiebreak: the point-accurate gun is still drawn", seen[1] > 0
|
||||
echo fmt" cold/mixed draws: gun0={seen[0]} gun1={seen[1]} gun2={seen[2]}"
|
||||
|
||||
# ── recording rule (tickBullets) ──────────────────────────────────────────────
|
||||
|
||||
proc mkState(tick: int, ex, ey: float): WorldState =
|
||||
WorldState(selfX: 100.0, selfY: 100.0,
|
||||
enemyX: ex, enemyY: ey, enemySpeed: 0.0, enemyHeading: 0.0,
|
||||
arenaWidth: 1000.0, arenaHeight: 1000.0, tick: tick)
|
||||
|
||||
proc runOneBullet(tieBreak: TieBreakMode): tuple[samples, hits: int] =
|
||||
## Spawn one bullet per power bin aimed at a stationary enemy 200 px away, then
|
||||
## tick until it passes its aim distance. The parallel window must fill only
|
||||
## when the tie-break is active.
|
||||
let targetId = 7
|
||||
var t = initTracker(1, bmPath)
|
||||
let s0 = mkState(0, 300.0, 100.0)
|
||||
let preds = [GunPrediction(x: 300.0, y: 100.0), GunPrediction(x: 300.0, y: 100.0),
|
||||
GunPrediction(x: 300.0, y: 100.0), GunPrediction(x: 300.0, y: 100.0)]
|
||||
t.spawnBullets(0, preds, s0, targetId)
|
||||
for i in 1..<80:
|
||||
let st = mkState(i, 300.0, 100.0)
|
||||
var enemies: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]]
|
||||
enemies[targetId] = (x: st.enemyX, y: st.enemyY, lastSeenTick: st.tick, alive: true)
|
||||
t.tickBullets(st, enemies,
|
||||
proc(gid: GunId, bin: int, e: FeedbackEvent) = discard, tieBreak = tieBreak)
|
||||
let fit = t.fitnessFor(targetId)
|
||||
for bin in 0..<len(PowerBins):
|
||||
let n = min(fit[0].pointBins[bin].count, WindowSize)
|
||||
result.samples += n
|
||||
for k in 0..<n:
|
||||
if fit[0].pointBins[bin].hits[k]: inc result.hits
|
||||
|
||||
proc testRecording() =
|
||||
let on = runOneBullet(tbPoint)
|
||||
let off = runOneBullet(tbOff)
|
||||
check "recording: tie-break OFF records no parallel samples (zero overhead)",
|
||||
off.samples == 0
|
||||
check "recording: tie-break ON records one parallel sample per bullet",
|
||||
on.samples == len(PowerBins)
|
||||
check "recording: an aimed stationary target is a point hit",
|
||||
on.hits == on.samples
|
||||
|
||||
# ── driver ────────────────────────────────────────────────────────────────────
|
||||
|
||||
testParsing()
|
||||
testNarrowingKeepsRandomness()
|
||||
testOffIsBaseline()
|
||||
testCommitIsDeterministic()
|
||||
testColdPointIsNoOp()
|
||||
testColdGunIsKept()
|
||||
testRecording()
|
||||
|
||||
if failures > 0:
|
||||
echo "\n", failures, " check(s) FAILED"
|
||||
quit(1)
|
||||
echo "\nAll selector tie-break checks passed."
|
||||
Reference in New Issue
Block a user