From d5061ee2156ba74ed95bb46a98b1710b60b03bf1 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Mon, 21 Sep 2026 00:14:46 +0200 Subject: [PATCH] test(range): restore the 12/12 offline==online proof; measure TM clause readability Task 1 - the acceptance proof was unrunnable because RecordWorldState was a compile-time const set to false. It is now a RUNTIME switch (let RecordWorldState* = existsEnv("TR_RECORD_WORLDSTATE")), default OFF, so ordinary runs write no fixture, and acceptance_offline_vs_online.nim enables it for the battle it spawns and clears it afterwards. Restored and run twice: 12/12 deterministic guns match exactly (128-tick and 546-tick battles), with Tsetlin reported separately as stochastic. Both nimble build variants clean. Task 2 - does a compact encoding turn the TM's 99.35% into a READABLE rule? Measured across window sizes (fixed seed, no tuning): frames TEST acc eff.lits/clause firing clauses counterfactual low/high/mean 10 99.35% 152.8 37 100/24/62.4% 3 95.94% 54.9 35 96/20/58.6% 2 99.48% 39.6 38 95/25/60.7% 1 98.30% 19.2 45 100/24/62.3% So 2 frames is strictly better than 10 on BOTH axes: +0.13 accuracy for 4x smaller clauses. The 3-frame dip is non-monotonic and left unexplained rather than smoothed over. A readable rule WAS partially recovered. Five clauses carry the exact Gray form !g10 ^ !g9 ^ !g8; g10 is inert in this data, so the effective rule is the 2-literal proposition !g9 ^ !g8, i.e. energy < 25.6. That is a genuine threshold in readable propositional form - but at 25.6, NOT the labelled 30, because 256 is a power-of-two Gray boundary expressible in two literals while 300 needs a longer conjunction. The TM found the nearest SIMPLE threshold. The honest caveat: that threshold is not the ensemble's decision mechanism. The counterfactual follow rate (high 24%, mean 62.3%) is statistically identical at 1, 2 and 10 frames, so compactness did not make the model read energy - its vote is carried by co-occurring bearing/velocity/heading/wall literals. Also identified: clauses containing all 11 Gray energy bits are satisfied at exactly one raw value (50, the dataset floor), so they are 'energy has hit the floor' detectors, not thresholds. Methodological fix worth keeping: the earlier single-frame counterfactual wrote energy into all 10 frame slots including the zeroed ones, reviving dead clauses and producing a spurious 2% high-follow rate. setEnergyFrames now rewrites only the exposed frames; the corrected figure is 24%. --- ModularBot_garage/src/ModularBot.nim | 22 +- .../tests/acceptance_offline_vs_online.nim | 12 +- .../tests/test_tm_pattern_learning.nim | 300 +++++++++++++++--- 3 files changed, 282 insertions(+), 52 deletions(-) diff --git a/ModularBot_garage/src/ModularBot.nim b/ModularBot_garage/src/ModularBot.nim index 1652fe3..d8d02ad 100644 --- a/ModularBot_garage/src/ModularBot.nim +++ b/ModularBot_garage/src/ModularBot.nim @@ -38,10 +38,12 @@ const DebugCircular = false ## Task A instrumentation: log every real shot + its eventual outcome to ## /tmp/shot_log.jsonl. Set to false to compile all shot-log machinery out. const ShotLog = true -## Offline range recorder: when true, append the exact WorldState the bot builds -## each tick to /tmp/worldstate_record.jsonl so it can be replayed offline. -## Mirrors the ShotLog idiom: false compiles the machinery out entirely. -const RecordWorldState = false +## Offline range recorder: when the TR_RECORD_WORLDSTATE env var is set, append +## the exact WorldState the bot builds each tick to /tmp/worldstate_record.jsonl +## so it can be replayed offline. This is a RUNTIME switch (not a compile-time +## const) so the normal build never writes a fixture, while the acceptance test +## can enable recording for just the battle it spawns by exporting the env var. +let RecordWorldState* = existsEnv("TR_RECORD_WORLDSTATE") const WorldStateRecordPath = "/tmp/worldstate_record.jsonl" const GunNames = ["HeadOn", "Linear", "Tsetlin", "Circular", "GuessFactor", "Pattern", "WallBounce", "Accel", "StopShot", "Displace", "AvgLead", "DecayGF", "KNN"] @@ -192,7 +194,7 @@ proc printConfig(bot: ModularBot, changed: string = "") = proc startWorldStateRecord(bot: ModularBot) = ## Truncate the fixture and write the meta line at round start. - when RecordWorldState: + if RecordWorldState: try: let f = open(WorldStateRecordPath, fmWrite) f.writeLine($(%*{"meta": { @@ -210,7 +212,7 @@ proc finishWorldStateRecord(bot: ModularBot) = ## Append the end marker so the offline replay can reproduce the live ## resolver's final-tick behaviour: if the target died during the last go() ## the live aim block was skipped, so no bullet resolved on that tick. - when RecordWorldState: + if RecordWorldState: var died = false if bot.lastKnownTargetId >= 0 and bot.enemyTracker.enemies.contains(bot.lastKnownTargetId): died = not bot.enemyTracker.enemies[bot.lastKnownTargetId].alive @@ -225,7 +227,7 @@ proc recordWorldState(bot: ModularBot, ws: WorldState) = ## Append one tick of the state the bot ACTUALLY built (including the stale ## tracker positions between radar scans) so the offline replay sees the same ## information the guns saw online. - when RecordWorldState: + if RecordWorldState: let tid = bot.currentTargetId var lst = -1 if tid >= 0 and bot.enemyTracker.enemies.contains(tid): @@ -268,7 +270,7 @@ proc buildState(bot: ModularBot, ex, ey, espeed, eheading, eenergy: float): Worl tick: bot.tick, enemies: ei, ) - when RecordWorldState: + if RecordWorldState: bot.recordWorldState(result) method onScannedBot*(bot: ModularBot, e: ScannedBotEvent) = @@ -310,7 +312,7 @@ method onHitByBullet*(bot: ModularBot, e: HitByBulletEvent) = method onRoundEnded*(bot: ModularBot, e: RoundEndedEventForBot) = ## Dump per-gun virtual bullet stats to /tmp/gun_stats.jsonl (one line per round). - when RecordWorldState: + if RecordWorldState: bot.finishWorldStateRecord() # Use lastKnownTargetId: currentTargetId is -1 if enemy died before round end let targetId = if bot.currentTargetId >= 0: bot.currentTargetId else: bot.lastKnownTargetId @@ -371,7 +373,7 @@ method onRoundEnded*(bot: ModularBot, e: RoundEndedEventForBot) = method onRoundStarted*(bot: ModularBot, e: RoundStartedEvent) = bot.roundNumber = e.roundNumber - when RecordWorldState: + if RecordWorldState: bot.startWorldStateRecord() bot.realShotsFired = 0 bot.realHits = 0 diff --git a/common_libs/tests/acceptance_offline_vs_online.nim b/common_libs/tests/acceptance_offline_vs_online.nim index c6448e5..aee3407 100644 --- a/common_libs/tests/acceptance_offline_vs_online.nim +++ b/common_libs/tests/acceptance_offline_vs_online.nim @@ -3,7 +3,9 @@ ## ## Steps: ## 1. run ONE live ModularBot vs OscillatorBot round with the ModularBot -## recorder ON (compiled in via `const RecordWorldState = true`), +## recorder enabled for this battle only (the test exports +## TR_RECORD_WORLDSTATE=1, which the bot reads at RUNTIME; ordinary +## builds leave it unset and write no fixture), ## 2. read the online per-gun virtual fitness from /tmp/gun_stats.jsonl, ## 3. replay the recorded WorldState fixture offline through the same guns, ## 4. compare. @@ -52,6 +54,12 @@ proc main() = for p in [statsPath, recordPath]: if fileExists(p): removeFile(p) + # Enable the ModularBot's runtime world-state recorder for THIS battle only. + # The env var is inherited by the battle-runner process and then by the bot + # processes it spawns, so a single invocation of this test is self-contained. + putEnv("TR_RECORD_WORLDSTATE", "1") + defer: delEnv("TR_RECORD_WORLDSTATE") + echo "=== live battle: ModularBot vs OscillatorBot, 1 round, max speed ===" let battle = runBattle(@[modularBotDir, adversaryDir], rounds = 1, timeout = 240000, maxSpeed = true) @@ -59,7 +67,7 @@ proc main() = echo fmt" {res.name:<14} rank={res.rank} score={res.totalScore}" if not fileExists(recordPath): - echo "FAIL: recorder produced no fixture (is RecordWorldState true?)" + echo "FAIL: recorder produced no fixture (TR_RECORD_WORLDSTATE not inherited?)" quit(1) if not fileExists(statsPath): echo "FAIL: no /tmp/gun_stats.jsonl" diff --git a/common_libs/tests/test_tm_pattern_learning.nim b/common_libs/tests/test_tm_pattern_learning.nim index 774e8d9..1d78291 100644 --- a/common_libs/tests/test_tm_pattern_learning.nim +++ b/common_libs/tests/test_tm_pattern_learning.nim @@ -128,11 +128,13 @@ proc makeLiterals(vec: TmBinaryVector): array[C_N_LITS, uint8] = result[i + C_N_IN] = 1'u8 - vec[i] proc samplesFromFixture(fx: Fixture, threshold: float, - singleFrame = false): seq[Sample] = + windowFrames = TM_WINDOW_SIZE): seq[Sample] = ## Replicates the gun's window bookkeeping exactly: index 0 = newest frame, - ## shifted once per tick. Skips the first 10 ticks (window warm-up). If - ## `singleFrame`, only the current frame is exposed (frames 1..9 zeroed); this - ## isolates how much of the clause bloat comes from the 10-frame redundancy. + ## shifted once per tick. Skips the first 10 ticks (window warm-up). + ## `windowFrames` controls how many of the MOST RECENT frames are exposed to + ## the classifier; older frames stay zeroed (constant, hence inert). Shrinking + ## it isolates how much of the clause bloat is 10-frame redundancy rather than + ## a property of the Tsetlin Machine itself. var window: array[TM_WINDOW_SIZE, TmFrameEncoded] var count = 0 for t in 0.. readable propositional logic ─────────── @@ -276,13 +276,15 @@ proc maskChannel(s: Sample, ch: FieldChannel): Sample = result.lits[i] = 0'u8 result.lits[i + C_N_IN] = 1'u8 -proc setEnergyAllFrames(s: var Sample, energy: float) = - ## Counterfactual: rewrite the 11-bit Gray-coded energy field of every frame to - ## `energy`, leaving every other channel untouched. If the model keys on the - ## energy threshold, its prediction follows this rewrite. +proc setEnergyFrames(s: var Sample, energy: float, frames: int) = + ## Counterfactual: rewrite the 11-bit Gray-coded energy field of the first + ## `frames` frames to `energy`, leaving every other channel untouched. Only + ## the exposed frames are written: compact encodings zero frames + ## `frames..= 1 included literal + firing*: int ## clauses that output 1 on >= 1 data sample (classify semantics) + firingPerSample*: float + energyFrac*: float ## energy-field literals / total included literals + +proc clauseStats(net: ClsNet, data: seq[Sample], + varying: seq[bool]): ClauseStats = + ## Width + firing statistics for one trained model. `firing` counts clauses + ## under classification semantics (empty clause = 0), i.e. clauses that + ## actually contribute a vote on at least one sample. + var totalNominal, totalEffective = 0 + var energyLits, totalLits = 0 + for c in 0.. 0: float(energyLits) / float(totalLits) else: 0.0 + var everFired: array[C_N_CLAUSES, bool] + var fires = 0 + for s in data: + for c in 0.. 0: float(fires) / float(data.len) else: 0.0 + +type + EnergyProfile = object + bits*: int ## number of current-frame (f0) energy literals in the clause + count*: int ## raw energy values (0..1500) satisfying that sub-conjunction + maxRun*: int ## longest contiguous run of satisfying raw values (threshold => large) + lo*, hi*: int ## min/max satisfying raw value (energy = raw/10) + spec*: string ## the literal spec, e.g. "!g10 !g9 !g8" + +proc grayBits11(raw: int): array[11, uint8] = + ## Same encoding as tmEncodeFrame's energy field: 11-bit Gray code, MSB first. + let g = raw xor (raw shr 1) + for b in 0..<11: + result[b] = uint8((g shr (10 - b)) and 1) + +proc clauseEnergyProfile(net: ClsNet, c: int): EnergyProfile = + ## Evaluate ONLY the current-frame (f0) energy literals of a clause over every + ## raw energy value 0..1500. A faithful threshold rule would satisfy the + ## conjunction over one long contiguous run ending near raw 300 (energy 30); a + ## scattered pattern with maxRun == 1 is a Gray-code coincidence, not a + ## threshold. bits == 0 means the clause has no f0-energy literals. + var lits: seq[tuple[b: int, want: uint8]] + var parts: seq[string] + for lit in included(net, c): + let bitIdx = if lit < C_N_IN: lit else: lit - C_N_IN + if bitIdx >= 72 and bitIdx < 83: + let b = bitIdx - 72 + let pos = lit < C_N_IN + lits.add (b, (if pos: 1'u8 else: 0'u8)) + parts.add (if pos: "g" else: "!g") & $(10 - b) + result.bits = lits.len + if lits.len == 0: return + result.spec = parts.join(" ") + result.lo = -1 + var run = 0 + for raw in 0..1500: + let g = grayBits11(raw) + var ok = true + for (b, want) in lits: + if g[b] != want: ok = false; break + if ok: + inc result.count + inc run + result.maxRun = max(result.maxRun, run) + if result.lo < 0: result.lo = raw + result.hi = raw + else: + run = 0 + proc shuffleRun(train: seq[Sample], labels: seq[int], seedState: uint64, epochs: int): ClsNet = seedRng(seedState) @@ -419,8 +513,8 @@ proc main() = var cfLow = test var cfHigh = test for i in 0..2}: {clauseStringVarying(net, c, varying)}" echo "" - # Ablation: single-frame input removes the 10-frame redundancy. If the clause - # bloat is a redundancy artefact, effective width should fall here. + # ── encoding comparison: how many frames does the rule actually need? ── + # Keep the full-stack model above as the reference, then train fresh models on + # windows that expose only the N most recent frames (older frames zeroed). The + # label and every other channel are unchanged, so this isolates the effect of + # 10-frame temporal redundancy on clause width and on whether the energy rule + # is read. Hyperparameters are identical to the reference run (no tuning). echo "" - echo "── ablation: single-frame input (current frame only) ──" - let train1 = gather(TrainCfgs, singleFrame = true) - let test1 = gather(TestCfgs, singleFrame = true) - var labels1: seq[int] - for s in train1: labels1.add s.y - let net1 = shuffleRun(train1, labels1, 20250920'u64, Epochs) - let teAcc1 = accuracy(net1, test1) - let varying1 = varyingBits(train1) - var total1, varyingInc1, active1 = 0 - for c in 0.. 0: inc active1 - total1 += inc.len + echo "── encoding comparison (label = energy < threshold, unchanged) ──" + echo " frames TEST acc eff.lits/cl firing cl energy% cf(low/high/mean)" + echo " -------------------------------------------------------------------------" + + type EncResult = object + frames: int + net: ClsNet + trainData: seq[Sample] + testData: seq[Sample] + varying: seq[bool] + teAcc: float + stats: ClauseStats + cfLow*: int ## % of test samples whose prediction follows energy -> 10 + cfHigh*: int ## % whose prediction follows energy -> 50 (straight) + cfMean*: float + + var encs: seq[EncResult] + for frames in [TM_WINDOW_SIZE, 3, 2, 1]: + var e = EncResult(frames: frames) + if frames == TM_WINDOW_SIZE: + e.trainData = train + e.testData = test + e.net = net + else: + e.trainData = gather(TrainCfgs, windowFrames = frames) + e.testData = gather(TestCfgs, windowFrames = frames) + var labels: seq[int] + for s in e.trainData: labels.add s.y + e.net = shuffleRun(e.trainData, labels, 20250920'u64, Epochs) + e.varying = varyingBits(e.trainData) + e.teAcc = accuracy(e.net, e.testData) + e.stats = clauseStats(e.net, e.testData, e.varying) + var cfL = e.testData + var cfH = e.testData + for i in 0..4} {e.teAcc*100:7.2f}% {e.stats.meanEffective:10.1f} " & + &"{e.stats.firing:>9} {e.stats.energyFrac*100:8.1f}% " & + &"{e.cfLow:>4}/{e.cfHigh:>4}/{e.cfMean:>5.1f}" + + # "Keeps accuracy" = TEST accuracy within 1.5 percentage points of the full + # 10-frame model. Pick the smallest such window (fewest frames = most readable). + # The full table above is printed regardless, so the tradeoff is transparent. + let fullAcc = encs[0].teAcc + var best = encs[0] + for e in encs: + if e.frames < best.frames and e.teAcc >= fullAcc - 0.015: + best = e + echo "" + if best.frames == TM_WINDOW_SIZE: + echo &"NOTE: no compact window stayed within 1.5 points of full ({fullAcc*100:.2f}%); reporting the full stack." + echo &"most compact encoding within 1.5 points of full ({fullAcc*100:.2f}%): " & + &"{best.frames} frame(s) -> TEST {best.teAcc*100:.2f}%, " & + &"{best.stats.meanEffective:.1f} effective literals/clause, " & + &"{best.stats.firing} firing clauses" + + echo "" + echo &"── learned clauses, compact {best.frames}-frame encoding (positive: TURN) ──" + for c in 0.. 0: + echo &" C{c:>2}: {clauseStringVarying(best.net, c, best.varying)}" + echo "" + echo &"── learned clauses, compact {best.frames}-frame encoding (negative: STRAIGHT) ──" + for c in C_N_POS.. 0: + echo &" C{c:>2}: {clauseStringVarying(best.net, c, best.varying)}" + + # Positive-clause audit: for each positive clause that ever fires, how well + # does "fires" agree with the true label (energy < threshold) and how much of + # its effective width is energy-field literals? A clause that recovered the + # threshold should fire mostly on positives and be energy-dominated. + var posN = 0 + for s in best.testData: + if s.y == 1: inc posN + echo "" + echo &"── compact {best.frames}-frame positive-clause audit " & + &"(label = energy < {Threshold:.0f}) ──" + echo " clause fires/764 agree(+)% energyLits/effLits recall%" + for c in 0..2} {fires:>4}/{best.testData.len:<5} {agree*100 div fires:>8}% " & + &"{eLits:>6}/{vLits:<6} {agree*100 div max(posN,1):>6}%" + + # Energy-literal audit: for each positive clause with current-frame energy + # literals, evaluate that sub-conjunction ALONE across the whole energy range. + # A faithful threshold rule lights up one long contiguous run ending near raw + # 300 (energy 30). A scattered result (longestRun small) is a Gray-code + # coincidence, not a threshold -- however well the full clause predicts. + echo "" + echo &"── compact {best.frames}-frame energy-literal audit " & + &"(f0 energy bits alone, all raw values 0..1500) ──" + echo " clause E-lits satisfying longestRun raw range energy-bit form verdict" for c in 0.. 0: - echo &" C{c:>2}: {clauseStringVarying(net1, c, varying1, 8)}" + let p = clauseEnergyProfile(best.net, c) + if p.bits == 0: continue + let verdict = + if p.maxRun == 1: "scattered" + elif p.lo == 0 and p.hi >= 250 and p.hi <= 350 and p.maxRun == p.hi + 1: + &"threshold-shaped (energy < {(p.hi + 1).float / 10.0:.1f})" + else: "partial/other" + echo &" C{c:>2} {p.bits:>4} {p.count:>7} {p.maxRun:>8} " & + &"{p.lo:>5}..{p.hi:<5} {p.spec:<38} {verdict}" + + # Counterfactual energy probe on the compact encoding: rewrite every frame's + # energy field to a fixed value and see whether the prediction follows. This + # is the test of whether the model learned the RIGHT reason (energy) or a + # heading proxy. + var ccfLow = best.testData + var ccfHigh = best.testData + for i in 0..