gun selector: hysteresis measured NEGATIVE, shipped at the lightest setting
Hypothesis under test: the selector chatters (~54 switches/100 ticks) and that chatter suppresses firing, so committing to the virtual-best gun should raise real hit rate. MEASURED AGAINST THE REAL DRUSSGT: it does not. setting switches/100t real hit % dmg/run shots/run no hysteresis 0/0 54.29 7.02% 217 244.8 light 10/0.05 1.95 6.22% 191 235.3 moderate 30/0.15 1.33 5.10% 156 233.9 aggressive 60/0.30 - 5.72% 175 242.5 (16 runs x 7 rounds per config except aggressive = 8; server-side events sidecar; permutation test baseline-vs-moderate p=0.002, baseline-vs-light p=0.18.) Hysteresis cuts chatter 28-54x but every variant fires slightly FEWER shots and deals LESS damage than baseline. Mechanism [INFERRED, consistent with docs/gun_rack_analysis.md 2/4]: the per-tick random tie-break among the tied band is a hedge, and hysteresis destroys it by committing to the virtual-best gun - which is not the real-best, because the virtual metric is a weak, sign-unstable ranker. The chattering was load-bearing. Shipped: GunDwellTicks=10, GunSwitchMargin=0.05 (GUN_SELECTOR_DWELL / GUN_SELECTOR_MARGIN) - the only setting within the baseline's run-to-run spread. GUN_SELECTOR_DWELL=0 GUN_SELECTOR_MARGIN=0 reproduces the pre-change selector exactly. Seam: VirtualTracker, which already owns the other selection state (fitness, the relative floor's peakRateRef), so the bot needs no new fields. bestGun and chooseFromFit stay pure/memoryless, which is why the existing random-tiebreak test needed no change. Guards: test_gun_harness 39/39 (33 original + 6 new hysteresis checks), test_vbullet_metric, test_power_selection, acceptance_offline_vs_online 12/12.
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
## Offline chatter measurement for the gun selector's hysteresis.
|
||||
##
|
||||
## Replays real recorded DrussGT movement (and the synthetic set) through the 13
|
||||
## ModularBot guns and drives the SAME stateful selector the live bot uses
|
||||
## (`selectGun`), counting how often the chosen gun changes. This is the
|
||||
## deterministic analogue of counting the live `[config]` gun-switch lines: the
|
||||
## replay feeds the identical per-tick fitness the live tracker builds, so the
|
||||
## switch sequence is the same up to the seeded tie-break RNG.
|
||||
##
|
||||
## The env knobs (`GUN_SELECTOR_DWELL`, `GUN_SELECTOR_MARGIN`) are read once per
|
||||
## process at module init, so sweep by running the binary under different values:
|
||||
##
|
||||
## nim c -r common_libs/tests/measure_hysteresis.nim # shipped
|
||||
## GUN_SELECTOR_DWELL=0 GUN_SELECTOR_MARGIN=0 \
|
||||
## common_libs/tests/measure_hysteresis # pre-hysteresis baseline
|
||||
##
|
||||
## This is a MEASUREMENT tool, not a test; it always exits 0.
|
||||
|
||||
import std/[os, strformat, algorithm, random]
|
||||
import gun_harness/offline_range
|
||||
import gun_harness/virtual_bullets
|
||||
import range_guns
|
||||
|
||||
const
|
||||
repoRoot = currentSourcePath().parentDir.parentDir.parentDir
|
||||
fixturesDir = repoRoot / "tools" / "fixtures"
|
||||
|
||||
type RunStat = object
|
||||
name: string
|
||||
ticks: int
|
||||
switches: int
|
||||
|
||||
proc measure(path: string, seed: int): RunStat =
|
||||
let fx = loadFixture(path)
|
||||
let drivers = buildAllGunDrivers(seed = seed, enableTmSelector = false)
|
||||
var last = -1
|
||||
var t = 0
|
||||
var switches = 0
|
||||
discard replayFixture(fx, drivers,
|
||||
liveActual = (fx.meta.source == "live"),
|
||||
tickCb = proc(tr: ptr VirtualTracker) =
|
||||
let g = selectGun(tr[], fx.enemyId, t)
|
||||
if last >= 0 and g != last: inc switches
|
||||
last = g
|
||||
inc t)
|
||||
RunStat(name: extractFilename(path), ticks: fx.states.len, switches: switches)
|
||||
|
||||
proc main() =
|
||||
randomize(12345)
|
||||
var paths: seq[string]
|
||||
for f in walkFiles(fixturesDir / "drussgt_vs_*.jsonl"): paths.add f
|
||||
for f in walkFiles(fixturesDir / "tr_drussgt_vs_*.jsonl"): paths.add f
|
||||
paths.sort()
|
||||
|
||||
var totalTicks = 0
|
||||
var totalSwitches = 0
|
||||
echo fmt"dwell={ActiveDwellTicks} ticks margin={ActiveSwitchMargin:.3f} " &
|
||||
fmt"mode={ActiveSelectorMode} rank={ActiveRank}"
|
||||
echo "fixture ticks switches sw/100t"
|
||||
echo "----------------------------------------------------------------"
|
||||
var stats: seq[RunStat]
|
||||
for p in paths:
|
||||
let s = measure(p, seed = 12345)
|
||||
stats.add s
|
||||
totalTicks += s.ticks
|
||||
totalSwitches += s.switches
|
||||
echo fmt"{s.name:<34}{s.ticks:>7}{s.switches:>10}{100.0*float(s.switches)/float(max(1,s.ticks)):>9.2f}"
|
||||
echo "----------------------------------------------------------------"
|
||||
let label = "TOTAL"
|
||||
echo fmt"{label:<34}{totalTicks:>7}{totalSwitches:>10}{100.0*float(totalSwitches)/float(max(1,totalTicks)):>9.2f}"
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
@@ -84,6 +84,59 @@ proc testBestGunDeterministicWinner() =
|
||||
check "gun with clearly best rate and >= MinObsBeforeCompete wins deterministically",
|
||||
allTwo
|
||||
|
||||
# ── sticky selector (hysteresis) ──────────────────────────────────────────────
|
||||
# `selectGun` is the live path; `bestGun`/`chooseFromFit` stay memoryless. These
|
||||
# checks pin the dwell, switch-margin, disqualification and floor behaviours.
|
||||
|
||||
proc testHysteresisDwell() =
|
||||
var t = initTracker(2)
|
||||
seedWindow(t, 7, gunId = 0, binIdx = 0, hits = 60, misses = 40) # 60%
|
||||
seedWindow(t, 7, gunId = 1, binIdx = 0, hits = 100, misses = 0) # 100%
|
||||
t.currentGun = 0
|
||||
t.currentSince = 0
|
||||
check "hysteresis: a clearly better gun is held off inside the dwell window",
|
||||
t.selectGun(7, tick = 5) == 0
|
||||
check "hysteresis: dwell expiry lets the better gun take over",
|
||||
t.selectGun(7, tick = GunDwellTicks) == 1
|
||||
|
||||
proc testHysteresisSwitchMargin() =
|
||||
# 80% incumbent vs 83% challenger: inside a 5% relative margin -> hold.
|
||||
var t = initTracker(2)
|
||||
seedWindow(t, 7, gunId = 0, binIdx = 0, hits = 80, misses = 20)
|
||||
seedWindow(t, 7, gunId = 1, binIdx = 0, hits = 83, misses = 17)
|
||||
t.currentGun = 0
|
||||
t.currentSince = 0
|
||||
check "hysteresis: a challenger inside the switch margin does not displace the incumbent",
|
||||
t.selectGun(7, tick = 100) == 0
|
||||
|
||||
# 80% incumbent vs 90% challenger: beyond the margin -> switch.
|
||||
var t2 = initTracker(2)
|
||||
seedWindow(t2, 7, gunId = 0, binIdx = 0, hits = 80, misses = 20)
|
||||
seedWindow(t2, 7, gunId = 1, binIdx = 0, hits = 90, misses = 10)
|
||||
t2.currentGun = 0
|
||||
t2.currentSince = 0
|
||||
check "hysteresis: a challenger beyond the switch margin displaces the incumbent",
|
||||
t2.selectGun(7, tick = 100) == 1
|
||||
|
||||
proc testHysteresisDisqualified() =
|
||||
var t = initTracker(2)
|
||||
seedWindow(t, 7, gunId = 0, binIdx = 0, hits = 10, misses = 0) # < MinObsBeforeCompete
|
||||
seedWindow(t, 7, gunId = 1, binIdx = 0, hits = 60, misses = 0)
|
||||
t.currentGun = 0
|
||||
t.currentSince = 0
|
||||
check "hysteresis: a disqualified incumbent is replaced even inside dwell",
|
||||
t.selectGun(7, tick = 3) == 1
|
||||
|
||||
proc testHysteresisFloor() =
|
||||
var t = initTracker(2)
|
||||
seedWindow(t, 7, gunId = 0, binIdx = 0, hits = 0, misses = 60)
|
||||
seedWindow(t, 7, gunId = 1, binIdx = 0, hits = 0, misses = 60)
|
||||
t.currentGun = 1
|
||||
t.currentSince = 0
|
||||
t.peakRateRef = 1.0
|
||||
check "hysteresis: floor collapse returns HeadOn even mid-dwell",
|
||||
t.selectGun(7, tick = 3) == 0
|
||||
|
||||
proc testBestPowerCold() =
|
||||
var t = initTracker(3)
|
||||
let (bin, power) = t.bestPower(0, -1)
|
||||
@@ -357,6 +410,10 @@ randomize()
|
||||
testColdBestGun()
|
||||
testRandomTiebreak()
|
||||
testBestGunDeterministicWinner()
|
||||
testHysteresisDwell()
|
||||
testHysteresisSwitchMargin()
|
||||
testHysteresisDisqualified()
|
||||
testHysteresisFloor()
|
||||
testBestPowerCold()
|
||||
testBestPowerWarmBin3()
|
||||
testFitnessForDeterministic()
|
||||
|
||||
Reference in New Issue
Block a user