test(range): restore the 12/12 offline==online proof; measure TM clause readability
Task 1 - the acceptance proof was unrunnable because RecordWorldState was a
compile-time const set to false. It is now a RUNTIME switch
(let RecordWorldState* = existsEnv("TR_RECORD_WORLDSTATE")), default OFF, so
ordinary runs write no fixture, and acceptance_offline_vs_online.nim enables
it for the battle it spawns and clears it afterwards. Restored and run twice:
12/12 deterministic guns match exactly (128-tick and 546-tick battles), with
Tsetlin reported separately as stochastic. Both nimble build variants clean.
Task 2 - does a compact encoding turn the TM's 99.35% into a READABLE rule?
Measured across window sizes (fixed seed, no tuning):
frames TEST acc eff.lits/clause firing clauses counterfactual low/high/mean
10 99.35% 152.8 37 100/24/62.4%
3 95.94% 54.9 35 96/20/58.6%
2 99.48% 39.6 38 95/25/60.7%
1 98.30% 19.2 45 100/24/62.3%
So 2 frames is strictly better than 10 on BOTH axes: +0.13 accuracy for 4x
smaller clauses. The 3-frame dip is non-monotonic and left unexplained rather
than smoothed over.
A readable rule WAS partially recovered. Five clauses carry the exact Gray
form !g10 ^ !g9 ^ !g8; g10 is inert in this data, so the effective rule is the
2-literal proposition !g9 ^ !g8, i.e. energy < 25.6. That is a genuine
threshold in readable propositional form - but at 25.6, NOT the labelled 30,
because 256 is a power-of-two Gray boundary expressible in two literals while
300 needs a longer conjunction. The TM found the nearest SIMPLE threshold.
The honest caveat: that threshold is not the ensemble's decision mechanism.
The counterfactual follow rate (high 24%, mean 62.3%) is statistically
identical at 1, 2 and 10 frames, so compactness did not make the model read
energy - its vote is carried by co-occurring bearing/velocity/heading/wall
literals. Also identified: clauses containing all 11 Gray energy bits are
satisfied at exactly one raw value (50, the dataset floor), so they are
'energy has hit the floor' detectors, not thresholds.
Methodological fix worth keeping: the earlier single-frame counterfactual
wrote energy into all 10 frame slots including the zeroed ones, reviving dead
clauses and producing a spurious 2% high-follow rate. setEnergyFrames now
rewrites only the exposed frames; the corrected figure is 24%.
This commit is contained in:
@@ -38,10 +38,12 @@ const DebugCircular = false
|
||||
## Task A instrumentation: log every real shot + its eventual outcome to
|
||||
## /tmp/shot_log.jsonl. Set to false to compile all shot-log machinery out.
|
||||
const ShotLog = true
|
||||
## Offline range recorder: when true, append the exact WorldState the bot builds
|
||||
## each tick to /tmp/worldstate_record.jsonl so it can be replayed offline.
|
||||
## Mirrors the ShotLog idiom: false compiles the machinery out entirely.
|
||||
const RecordWorldState = false
|
||||
## Offline range recorder: when the TR_RECORD_WORLDSTATE env var is set, append
|
||||
## the exact WorldState the bot builds each tick to /tmp/worldstate_record.jsonl
|
||||
## so it can be replayed offline. This is a RUNTIME switch (not a compile-time
|
||||
## const) so the normal build never writes a fixture, while the acceptance test
|
||||
## can enable recording for just the battle it spawns by exporting the env var.
|
||||
let RecordWorldState* = existsEnv("TR_RECORD_WORLDSTATE")
|
||||
const WorldStateRecordPath = "/tmp/worldstate_record.jsonl"
|
||||
const GunNames = ["HeadOn", "Linear", "Tsetlin", "Circular", "GuessFactor", "Pattern", "WallBounce", "Accel", "StopShot", "Displace", "AvgLead", "DecayGF", "KNN"]
|
||||
|
||||
@@ -192,7 +194,7 @@ proc printConfig(bot: ModularBot, changed: string = "") =
|
||||
|
||||
proc startWorldStateRecord(bot: ModularBot) =
|
||||
## Truncate the fixture and write the meta line at round start.
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
try:
|
||||
let f = open(WorldStateRecordPath, fmWrite)
|
||||
f.writeLine($(%*{"meta": {
|
||||
@@ -210,7 +212,7 @@ proc finishWorldStateRecord(bot: ModularBot) =
|
||||
## Append the end marker so the offline replay can reproduce the live
|
||||
## resolver's final-tick behaviour: if the target died during the last go()
|
||||
## the live aim block was skipped, so no bullet resolved on that tick.
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
var died = false
|
||||
if bot.lastKnownTargetId >= 0 and bot.enemyTracker.enemies.contains(bot.lastKnownTargetId):
|
||||
died = not bot.enemyTracker.enemies[bot.lastKnownTargetId].alive
|
||||
@@ -225,7 +227,7 @@ proc recordWorldState(bot: ModularBot, ws: WorldState) =
|
||||
## Append one tick of the state the bot ACTUALLY built (including the stale
|
||||
## tracker positions between radar scans) so the offline replay sees the same
|
||||
## information the guns saw online.
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
let tid = bot.currentTargetId
|
||||
var lst = -1
|
||||
if tid >= 0 and bot.enemyTracker.enemies.contains(tid):
|
||||
@@ -268,7 +270,7 @@ proc buildState(bot: ModularBot, ex, ey, espeed, eheading, eenergy: float): Worl
|
||||
tick: bot.tick,
|
||||
enemies: ei,
|
||||
)
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
bot.recordWorldState(result)
|
||||
|
||||
method onScannedBot*(bot: ModularBot, e: ScannedBotEvent) =
|
||||
@@ -310,7 +312,7 @@ method onHitByBullet*(bot: ModularBot, e: HitByBulletEvent) =
|
||||
|
||||
method onRoundEnded*(bot: ModularBot, e: RoundEndedEventForBot) =
|
||||
## Dump per-gun virtual bullet stats to /tmp/gun_stats.jsonl (one line per round).
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
bot.finishWorldStateRecord()
|
||||
# Use lastKnownTargetId: currentTargetId is -1 if enemy died before round end
|
||||
let targetId = if bot.currentTargetId >= 0: bot.currentTargetId else: bot.lastKnownTargetId
|
||||
@@ -371,7 +373,7 @@ method onRoundEnded*(bot: ModularBot, e: RoundEndedEventForBot) =
|
||||
|
||||
method onRoundStarted*(bot: ModularBot, e: RoundStartedEvent) =
|
||||
bot.roundNumber = e.roundNumber
|
||||
when RecordWorldState:
|
||||
if RecordWorldState:
|
||||
bot.startWorldStateRecord()
|
||||
bot.realShotsFired = 0
|
||||
bot.realHits = 0
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
##
|
||||
## Steps:
|
||||
## 1. run ONE live ModularBot vs OscillatorBot round with the ModularBot
|
||||
## recorder ON (compiled in via `const RecordWorldState = true`),
|
||||
## recorder enabled for this battle only (the test exports
|
||||
## TR_RECORD_WORLDSTATE=1, which the bot reads at RUNTIME; ordinary
|
||||
## builds leave it unset and write no fixture),
|
||||
## 2. read the online per-gun virtual fitness from /tmp/gun_stats.jsonl,
|
||||
## 3. replay the recorded WorldState fixture offline through the same guns,
|
||||
## 4. compare.
|
||||
@@ -52,6 +54,12 @@ proc main() =
|
||||
for p in [statsPath, recordPath]:
|
||||
if fileExists(p): removeFile(p)
|
||||
|
||||
# Enable the ModularBot's runtime world-state recorder for THIS battle only.
|
||||
# The env var is inherited by the battle-runner process and then by the bot
|
||||
# processes it spawns, so a single invocation of this test is self-contained.
|
||||
putEnv("TR_RECORD_WORLDSTATE", "1")
|
||||
defer: delEnv("TR_RECORD_WORLDSTATE")
|
||||
|
||||
echo "=== live battle: ModularBot vs OscillatorBot, 1 round, max speed ==="
|
||||
let battle = runBattle(@[modularBotDir, adversaryDir], rounds = 1,
|
||||
timeout = 240000, maxSpeed = true)
|
||||
@@ -59,7 +67,7 @@ proc main() =
|
||||
echo fmt" {res.name:<14} rank={res.rank} score={res.totalScore}"
|
||||
|
||||
if not fileExists(recordPath):
|
||||
echo "FAIL: recorder produced no fixture (is RecordWorldState true?)"
|
||||
echo "FAIL: recorder produced no fixture (TR_RECORD_WORLDSTATE not inherited?)"
|
||||
quit(1)
|
||||
if not fileExists(statsPath):
|
||||
echo "FAIL: no /tmp/gun_stats.jsonl"
|
||||
|
||||
@@ -128,11 +128,13 @@ proc makeLiterals(vec: TmBinaryVector): array[C_N_LITS, uint8] =
|
||||
result[i + C_N_IN] = 1'u8 - vec[i]
|
||||
|
||||
proc samplesFromFixture(fx: Fixture, threshold: float,
|
||||
singleFrame = false): seq[Sample] =
|
||||
windowFrames = TM_WINDOW_SIZE): seq[Sample] =
|
||||
## Replicates the gun's window bookkeeping exactly: index 0 = newest frame,
|
||||
## shifted once per tick. Skips the first 10 ticks (window warm-up). If
|
||||
## `singleFrame`, only the current frame is exposed (frames 1..9 zeroed); this
|
||||
## isolates how much of the clause bloat comes from the 10-frame redundancy.
|
||||
## shifted once per tick. Skips the first 10 ticks (window warm-up).
|
||||
## `windowFrames` controls how many of the MOST RECENT frames are exposed to
|
||||
## the classifier; older frames stay zeroed (constant, hence inert). Shrinking
|
||||
## it isolates how much of the clause bloat is 10-frame redundancy rather than
|
||||
## a property of the Tsetlin Machine itself.
|
||||
var window: array[TM_WINDOW_SIZE, TmFrameEncoded]
|
||||
var count = 0
|
||||
for t in 0..<fx.states.len:
|
||||
@@ -153,12 +155,10 @@ proc samplesFromFixture(fx: Fixture, threshold: float,
|
||||
s.arenaWidth - s.selfX, s.selfX,
|
||||
s.selfEnergy, true)
|
||||
var vec: TmBinaryVector
|
||||
if singleFrame:
|
||||
var w1: array[TM_WINDOW_SIZE, TmFrameEncoded]
|
||||
w1[0] = window[0]
|
||||
vec = tmEncodeFullVector(w1, selfState)
|
||||
else:
|
||||
vec = tmEncodeFullVector(window, selfState)
|
||||
var w1: array[TM_WINDOW_SIZE, TmFrameEncoded]
|
||||
for i in 0 ..< min(windowFrames, TM_WINDOW_SIZE):
|
||||
w1[i] = window[i]
|
||||
vec = tmEncodeFullVector(w1, selfState)
|
||||
result.add Sample(lits: makeLiterals(vec),
|
||||
y: (if s.enemyEnergy < threshold: 1 else: 0))
|
||||
|
||||
@@ -190,8 +190,8 @@ proc makeFixture(c: RuleCfg): Fixture =
|
||||
e0 = c.e0, decay = c.decay, threshold = Threshold, speed = c.speed,
|
||||
hardTurnDeg = c.turn)
|
||||
|
||||
proc gather(cfgs: openArray[RuleCfg], singleFrame = false): seq[Sample] =
|
||||
for c in cfgs: result.add samplesFromFixture(makeFixture(c), Threshold, singleFrame)
|
||||
proc gather(cfgs: openArray[RuleCfg], windowFrames = TM_WINDOW_SIZE): seq[Sample] =
|
||||
for c in cfgs: result.add samplesFromFixture(makeFixture(c), Threshold, windowFrames)
|
||||
|
||||
# ── clause decoding: literal index -> readable propositional logic ───────────
|
||||
|
||||
@@ -276,13 +276,15 @@ proc maskChannel(s: Sample, ch: FieldChannel): Sample =
|
||||
result.lits[i] = 0'u8
|
||||
result.lits[i + C_N_IN] = 1'u8
|
||||
|
||||
proc setEnergyAllFrames(s: var Sample, energy: float) =
|
||||
## Counterfactual: rewrite the 11-bit Gray-coded energy field of every frame to
|
||||
## `energy`, leaving every other channel untouched. If the model keys on the
|
||||
## energy threshold, its prediction follows this rewrite.
|
||||
proc setEnergyFrames(s: var Sample, energy: float, frames: int) =
|
||||
## Counterfactual: rewrite the 11-bit Gray-coded energy field of the first
|
||||
## `frames` frames to `energy`, leaving every other channel untouched. Only
|
||||
## the exposed frames are written: compact encodings zero frames
|
||||
## `frames..<TM_WINDOW_SIZE`, and those constant bits must stay constant or the
|
||||
## probe would revive clauses that never fire in the real encoding.
|
||||
let raw = clamp(int(energy * 10.0), 0, 1500)
|
||||
let gray = raw xor (raw shr 1)
|
||||
for f in 0..<TM_WINDOW_SIZE:
|
||||
for f in 0..<min(frames, TM_WINDOW_SIZE):
|
||||
for b in 0..<11:
|
||||
let bit = uint8((gray shr (10 - b)) and 1)
|
||||
let idx = f * TM_FRAME_BITS + 72 + b
|
||||
@@ -331,6 +333,98 @@ proc confusion(net: ClsNet, data: seq[Sample]): tuple[tp, tn, fp, fn: int] =
|
||||
elif p == 1 and s.y == 0: inc result.fp
|
||||
else: inc result.fn
|
||||
|
||||
type
|
||||
ClauseStats = object
|
||||
meanNominal*: float
|
||||
meanEffective*: float ## incl. literals on bits that actually vary (excludes inert)
|
||||
maxInc*: int
|
||||
active*: int ## clauses with >= 1 included literal
|
||||
firing*: int ## clauses that output 1 on >= 1 data sample (classify semantics)
|
||||
firingPerSample*: float
|
||||
energyFrac*: float ## energy-field literals / total included literals
|
||||
|
||||
proc clauseStats(net: ClsNet, data: seq[Sample],
|
||||
varying: seq[bool]): ClauseStats =
|
||||
## Width + firing statistics for one trained model. `firing` counts clauses
|
||||
## under classification semantics (empty clause = 0), i.e. clauses that
|
||||
## actually contribute a vote on at least one sample.
|
||||
var totalNominal, totalEffective = 0
|
||||
var energyLits, totalLits = 0
|
||||
for c in 0..<C_N_CLAUSES:
|
||||
let inc = included(net, c)
|
||||
if inc.len == 0: continue
|
||||
inc result.active
|
||||
totalNominal += inc.len
|
||||
result.maxInc = max(result.maxInc, inc.len)
|
||||
for lit in inc:
|
||||
inc totalLits
|
||||
if isEnergyField(lit): inc energyLits
|
||||
let bitIdx = if lit < C_N_IN: lit else: lit - C_N_IN
|
||||
if varying[bitIdx]: inc totalEffective
|
||||
let denom = max(result.active, 1)
|
||||
result.meanNominal = float(totalNominal) / float(denom)
|
||||
result.meanEffective = float(totalEffective) / float(denom)
|
||||
result.energyFrac = if totalLits > 0: float(energyLits) / float(totalLits) else: 0.0
|
||||
var everFired: array[C_N_CLAUSES, bool]
|
||||
var fires = 0
|
||||
for s in data:
|
||||
for c in 0..<C_N_CLAUSES:
|
||||
if evalClause(net, c, s.lits, learning = false) == 1'u8:
|
||||
everFired[c] = true
|
||||
inc fires
|
||||
for b in everFired:
|
||||
if b: inc result.firing
|
||||
result.firingPerSample =
|
||||
if data.len > 0: float(fires) / float(data.len) else: 0.0
|
||||
|
||||
type
|
||||
EnergyProfile = object
|
||||
bits*: int ## number of current-frame (f0) energy literals in the clause
|
||||
count*: int ## raw energy values (0..1500) satisfying that sub-conjunction
|
||||
maxRun*: int ## longest contiguous run of satisfying raw values (threshold => large)
|
||||
lo*, hi*: int ## min/max satisfying raw value (energy = raw/10)
|
||||
spec*: string ## the literal spec, e.g. "!g10 !g9 !g8"
|
||||
|
||||
proc grayBits11(raw: int): array[11, uint8] =
|
||||
## Same encoding as tmEncodeFrame's energy field: 11-bit Gray code, MSB first.
|
||||
let g = raw xor (raw shr 1)
|
||||
for b in 0..<11:
|
||||
result[b] = uint8((g shr (10 - b)) and 1)
|
||||
|
||||
proc clauseEnergyProfile(net: ClsNet, c: int): EnergyProfile =
|
||||
## Evaluate ONLY the current-frame (f0) energy literals of a clause over every
|
||||
## raw energy value 0..1500. A faithful threshold rule would satisfy the
|
||||
## conjunction over one long contiguous run ending near raw 300 (energy 30); a
|
||||
## scattered pattern with maxRun == 1 is a Gray-code coincidence, not a
|
||||
## threshold. bits == 0 means the clause has no f0-energy literals.
|
||||
var lits: seq[tuple[b: int, want: uint8]]
|
||||
var parts: seq[string]
|
||||
for lit in included(net, c):
|
||||
let bitIdx = if lit < C_N_IN: lit else: lit - C_N_IN
|
||||
if bitIdx >= 72 and bitIdx < 83:
|
||||
let b = bitIdx - 72
|
||||
let pos = lit < C_N_IN
|
||||
lits.add (b, (if pos: 1'u8 else: 0'u8))
|
||||
parts.add (if pos: "g" else: "!g") & $(10 - b)
|
||||
result.bits = lits.len
|
||||
if lits.len == 0: return
|
||||
result.spec = parts.join(" ")
|
||||
result.lo = -1
|
||||
var run = 0
|
||||
for raw in 0..1500:
|
||||
let g = grayBits11(raw)
|
||||
var ok = true
|
||||
for (b, want) in lits:
|
||||
if g[b] != want: ok = false; break
|
||||
if ok:
|
||||
inc result.count
|
||||
inc run
|
||||
result.maxRun = max(result.maxRun, run)
|
||||
if result.lo < 0: result.lo = raw
|
||||
result.hi = raw
|
||||
else:
|
||||
run = 0
|
||||
|
||||
proc shuffleRun(train: seq[Sample], labels: seq[int], seedState: uint64,
|
||||
epochs: int): ClsNet =
|
||||
seedRng(seedState)
|
||||
@@ -419,8 +513,8 @@ proc main() =
|
||||
var cfLow = test
|
||||
var cfHigh = test
|
||||
for i in 0..<test.len:
|
||||
cfLow[i] = test[i]; setEnergyAllFrames(cfLow[i], 10.0) # label would be 1
|
||||
cfHigh[i] = test[i]; setEnergyAllFrames(cfHigh[i], 50.0) # label would be 0
|
||||
cfLow[i] = test[i]; setEnergyFrames(cfLow[i], 10.0, TM_WINDOW_SIZE) # label would be 1
|
||||
cfHigh[i] = test[i]; setEnergyFrames(cfHigh[i], 50.0, TM_WINDOW_SIZE) # label would be 0
|
||||
var followLow, followHigh = 0
|
||||
for i in 0..<test.len:
|
||||
if predictLabel(net, cfLow[i].lits) == 1: inc followLow
|
||||
@@ -449,31 +543,157 @@ proc main() =
|
||||
echo &" C{c:>2}: {clauseStringVarying(net, c, varying)}"
|
||||
echo ""
|
||||
|
||||
# Ablation: single-frame input removes the 10-frame redundancy. If the clause
|
||||
# bloat is a redundancy artefact, effective width should fall here.
|
||||
# ── encoding comparison: how many frames does the rule actually need? ──
|
||||
# Keep the full-stack model above as the reference, then train fresh models on
|
||||
# windows that expose only the N most recent frames (older frames zeroed). The
|
||||
# label and every other channel are unchanged, so this isolates the effect of
|
||||
# 10-frame temporal redundancy on clause width and on whether the energy rule
|
||||
# is read. Hyperparameters are identical to the reference run (no tuning).
|
||||
echo ""
|
||||
echo "── ablation: single-frame input (current frame only) ──"
|
||||
let train1 = gather(TrainCfgs, singleFrame = true)
|
||||
let test1 = gather(TestCfgs, singleFrame = true)
|
||||
var labels1: seq[int]
|
||||
for s in train1: labels1.add s.y
|
||||
let net1 = shuffleRun(train1, labels1, 20250920'u64, Epochs)
|
||||
let teAcc1 = accuracy(net1, test1)
|
||||
let varying1 = varyingBits(train1)
|
||||
var total1, varyingInc1, active1 = 0
|
||||
for c in 0..<C_N_CLAUSES:
|
||||
let inc = included(net1, c)
|
||||
if inc.len > 0: inc active1
|
||||
total1 += inc.len
|
||||
echo "── encoding comparison (label = energy < threshold, unchanged) ──"
|
||||
echo " frames TEST acc eff.lits/cl firing cl energy% cf(low/high/mean)"
|
||||
echo " -------------------------------------------------------------------------"
|
||||
|
||||
type EncResult = object
|
||||
frames: int
|
||||
net: ClsNet
|
||||
trainData: seq[Sample]
|
||||
testData: seq[Sample]
|
||||
varying: seq[bool]
|
||||
teAcc: float
|
||||
stats: ClauseStats
|
||||
cfLow*: int ## % of test samples whose prediction follows energy -> 10
|
||||
cfHigh*: int ## % whose prediction follows energy -> 50 (straight)
|
||||
cfMean*: float
|
||||
|
||||
var encs: seq[EncResult]
|
||||
for frames in [TM_WINDOW_SIZE, 3, 2, 1]:
|
||||
var e = EncResult(frames: frames)
|
||||
if frames == TM_WINDOW_SIZE:
|
||||
e.trainData = train
|
||||
e.testData = test
|
||||
e.net = net
|
||||
else:
|
||||
e.trainData = gather(TrainCfgs, windowFrames = frames)
|
||||
e.testData = gather(TestCfgs, windowFrames = frames)
|
||||
var labels: seq[int]
|
||||
for s in e.trainData: labels.add s.y
|
||||
e.net = shuffleRun(e.trainData, labels, 20250920'u64, Epochs)
|
||||
e.varying = varyingBits(e.trainData)
|
||||
e.teAcc = accuracy(e.net, e.testData)
|
||||
e.stats = clauseStats(e.net, e.testData, e.varying)
|
||||
var cfL = e.testData
|
||||
var cfH = e.testData
|
||||
for i in 0..<e.testData.len:
|
||||
cfL[i] = e.testData[i]; setEnergyFrames(cfL[i], 10.0, e.frames)
|
||||
cfH[i] = e.testData[i]; setEnergyFrames(cfH[i], 50.0, e.frames)
|
||||
var fL, fH = 0
|
||||
for i in 0..<e.testData.len:
|
||||
if predictLabel(e.net, cfL[i].lits) == 1: inc fL
|
||||
if predictLabel(e.net, cfH[i].lits) == 0: inc fH
|
||||
e.cfLow = fL * 100 div e.testData.len
|
||||
e.cfHigh = fH * 100 div e.testData.len
|
||||
e.cfMean = (fL.float + fH.float) / (2.0 * e.testData.len.float) * 100.0
|
||||
encs.add e
|
||||
echo &" {frames:>4} {e.teAcc*100:7.2f}% {e.stats.meanEffective:10.1f} " &
|
||||
&"{e.stats.firing:>9} {e.stats.energyFrac*100:8.1f}% " &
|
||||
&"{e.cfLow:>4}/{e.cfHigh:>4}/{e.cfMean:>5.1f}"
|
||||
|
||||
# "Keeps accuracy" = TEST accuracy within 1.5 percentage points of the full
|
||||
# 10-frame model. Pick the smallest such window (fewest frames = most readable).
|
||||
# The full table above is printed regardless, so the tradeoff is transparent.
|
||||
let fullAcc = encs[0].teAcc
|
||||
var best = encs[0]
|
||||
for e in encs:
|
||||
if e.frames < best.frames and e.teAcc >= fullAcc - 0.015:
|
||||
best = e
|
||||
echo ""
|
||||
if best.frames == TM_WINDOW_SIZE:
|
||||
echo &"NOTE: no compact window stayed within 1.5 points of full ({fullAcc*100:.2f}%); reporting the full stack."
|
||||
echo &"most compact encoding within 1.5 points of full ({fullAcc*100:.2f}%): " &
|
||||
&"{best.frames} frame(s) -> TEST {best.teAcc*100:.2f}%, " &
|
||||
&"{best.stats.meanEffective:.1f} effective literals/clause, " &
|
||||
&"{best.stats.firing} firing clauses"
|
||||
|
||||
echo ""
|
||||
echo &"── learned clauses, compact {best.frames}-frame encoding (positive: TURN) ──"
|
||||
for c in 0..<C_N_POS:
|
||||
if included(best.net, c).len > 0:
|
||||
echo &" C{c:>2}: {clauseStringVarying(best.net, c, best.varying)}"
|
||||
echo ""
|
||||
echo &"── learned clauses, compact {best.frames}-frame encoding (negative: STRAIGHT) ──"
|
||||
for c in C_N_POS..<C_N_CLAUSES:
|
||||
if included(best.net, c).len > 0:
|
||||
echo &" C{c:>2}: {clauseStringVarying(best.net, c, best.varying)}"
|
||||
|
||||
# Positive-clause audit: for each positive clause that ever fires, how well
|
||||
# does "fires" agree with the true label (energy < threshold) and how much of
|
||||
# its effective width is energy-field literals? A clause that recovered the
|
||||
# threshold should fire mostly on positives and be energy-dominated.
|
||||
var posN = 0
|
||||
for s in best.testData:
|
||||
if s.y == 1: inc posN
|
||||
echo ""
|
||||
echo &"── compact {best.frames}-frame positive-clause audit " &
|
||||
&"(label = energy < {Threshold:.0f}) ──"
|
||||
echo " clause fires/764 agree(+)% energyLits/effLits recall%"
|
||||
for c in 0..<C_N_POS:
|
||||
let inc = included(best.net, c)
|
||||
var vLits, eLits = 0
|
||||
for lit in inc:
|
||||
let bitIdx = if lit < C_N_IN: lit else: lit - C_N_IN
|
||||
if varying1[bitIdx]: inc varyingInc1
|
||||
echo &"single-frame TEST accuracy = {teAcc1*100:.2f}% (majority {testMajority*100:.2f}%), " &
|
||||
&"mean {float(total1)/float(max(active1,1)):.1f} nominal / " &
|
||||
&"{float(varyingInc1)/float(max(active1,1)):.1f} effective literals/clause"
|
||||
if best.varying[bitIdx]:
|
||||
inc vLits
|
||||
if isEnergyField(lit): inc eLits
|
||||
if vLits == 0: continue
|
||||
var fires, agree = 0
|
||||
for s in best.testData:
|
||||
if evalClause(best.net, c, s.lits, learning = false) == 1'u8:
|
||||
inc fires
|
||||
if s.y == 1: inc agree
|
||||
if fires == 0: continue
|
||||
echo &" C{c:>2} {fires:>4}/{best.testData.len:<5} {agree*100 div fires:>8}% " &
|
||||
&"{eLits:>6}/{vLits:<6} {agree*100 div max(posN,1):>6}%"
|
||||
|
||||
# Energy-literal audit: for each positive clause with current-frame energy
|
||||
# literals, evaluate that sub-conjunction ALONE across the whole energy range.
|
||||
# A faithful threshold rule lights up one long contiguous run ending near raw
|
||||
# 300 (energy 30). A scattered result (longestRun small) is a Gray-code
|
||||
# coincidence, not a threshold -- however well the full clause predicts.
|
||||
echo ""
|
||||
echo &"── compact {best.frames}-frame energy-literal audit " &
|
||||
&"(f0 energy bits alone, all raw values 0..1500) ──"
|
||||
echo " clause E-lits satisfying longestRun raw range energy-bit form verdict"
|
||||
for c in 0..<C_N_POS:
|
||||
if included(net1, c).len > 0:
|
||||
echo &" C{c:>2}: {clauseStringVarying(net1, c, varying1, 8)}"
|
||||
let p = clauseEnergyProfile(best.net, c)
|
||||
if p.bits == 0: continue
|
||||
let verdict =
|
||||
if p.maxRun == 1: "scattered"
|
||||
elif p.lo == 0 and p.hi >= 250 and p.hi <= 350 and p.maxRun == p.hi + 1:
|
||||
&"threshold-shaped (energy < {(p.hi + 1).float / 10.0:.1f})"
|
||||
else: "partial/other"
|
||||
echo &" C{c:>2} {p.bits:>4} {p.count:>7} {p.maxRun:>8} " &
|
||||
&"{p.lo:>5}..{p.hi:<5} {p.spec:<38} {verdict}"
|
||||
|
||||
# Counterfactual energy probe on the compact encoding: rewrite every frame's
|
||||
# energy field to a fixed value and see whether the prediction follows. This
|
||||
# is the test of whether the model learned the RIGHT reason (energy) or a
|
||||
# heading proxy.
|
||||
var ccfLow = best.testData
|
||||
var ccfHigh = best.testData
|
||||
for i in 0..<best.testData.len:
|
||||
ccfLow[i] = best.testData[i]; setEnergyFrames(ccfLow[i], 10.0, best.frames)
|
||||
ccfHigh[i] = best.testData[i]; setEnergyFrames(ccfHigh[i], 50.0, best.frames)
|
||||
var cfLowFollow, cfHighFollow = 0
|
||||
for i in 0..<best.testData.len:
|
||||
if predictLabel(best.net, ccfLow[i].lits) == 1: inc cfLowFollow
|
||||
if predictLabel(best.net, ccfHigh[i].lits) == 0: inc cfHighFollow
|
||||
let cfMean = (cfLowFollow.float + cfHighFollow.float) / (2.0 * best.testData.len.float)
|
||||
echo ""
|
||||
echo &"counterfactual energy flip on the compact {best.frames}-frame encoding: " &
|
||||
&"follows low {cfLowFollow*100 div best.testData.len}%, " &
|
||||
&"high {cfHighFollow*100 div best.testData.len}%, mean {cfMean*100:.1f}% " &
|
||||
&"(full-stack was 100% / 24% / 62.4%)"
|
||||
echo ""
|
||||
|
||||
# The two classes are separable by the rule, so a model that found it should
|
||||
|
||||
Reference in New Issue
Block a user