From ab8d3831212396166fc28d15a20acd138cbe7fed Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Tue, 22 Sep 2026 21:40:27 +0200 Subject: [PATCH] Automata metrics: settledness alone does NOT separate learning from fidgeting Added the four automata-level metrics to the TM diagnostics kit (settledness, clause diversity, churn, vote disagreement) plus a state histogram, a per-input confidence table and a one-line health summary, and validated them on a learnable-vs-noise pair. STATE CONVENTIONS, read off OUR code rather than from memory: range [-nStates, nStates] as int16; nStates = 64 for tm_pattern, 32 for tsetlin initial value 0 = the Exclude boundary INCLUDE iff state > 0; EXCLUDE iff state <= 0 flip boundary sits between state 0 and 1; commitment = abs(st)/nStates in [0,1] === THE GATE, AND A RESULT THAT MATTERS === Case A (learnable planted rule) vs Case B (shuffled labels), 49 bits, N=64: metric A (learnable) B (shuffled) settledness mean 0.970 0.719 churn flip/sample 0.000055 FALLING 0.000788 FLAT clause-change/sample 0.00263 falling 0.0595 flat diversity (Jaccard) 0.176 0.014 disagreement 0.003 0.298 verdict settling mixed (NOT settling) **SETTLEDNESS ALONE DOES NOT WORK.** On noise the automata still COMMIT (0.719) - they just commit to the wrong thing. The decisive separators are **churn TREND (falling vs flat)** and **vote DISAGREEMENT (0.003 vs 0.298)**. Had we built only the settledness metric - the one that seems most obvious - we would have been misled. That is now recorded in the README. INERTIA SWEEP: A vs B separate at N=16/32/64/128. **Raising N raises A's commitment but does NOT reduce B's noise-fitting** - so more inertia does not rescue a noise-fitting TM. === REAL READING ON THE SHIPPED GUN, AND THE INFERENCE IT SUPPORTS === tm_pattern GF head over the DrussGT fixtures: settledness 0.484 (settling), diversity 0.267 (moderate), churn 0.094/100 FALLING, disagreement 0.145 (coherent). **VERDICT: SETTLING** - not fidgeting, not collapsed. Constant inputs flagged: 38/39 (the known never-written bits) plus 19/36/37. Context: pooled warm accuracy 35.72% vs 34.24% majority = +1.48pp. So: **the old gun was NOT failing because of inertia or instability - it settled properly and its settled rules still barely beat a lazy guess.** Its settledness (0.484) is LOWER than both synthetic cases (0.97/0.72), which is the signature of WEAK OR CONFLICTING SIGNAL rather than too much inertia. CONCLUSION: **N and s are not the observed bottleneck. The target/representation is.** That is exactly why the new design changes the target and the label pipeline rather than sweeping knobs - and it means we should NOT spend effort on an N/s sweep expecting it to fix anything. Also adds `diag_automata_validation.nim` (Case A/B/C + inertia sweep) and `test_tm_automata_diag.nim` (55 pure checks); `test_tm_diag` 48 and `diag_synthetic` 17 still pass, plus all other guards. acceptance_offline_vs_online was NOT run (it needs a live battle and there is no tm_diag dependency). Caveat: churn on the real gun is a PROXY (a tm_core retrain over captured samples in live order) because the live gun exposes no per-sample state trace; the other metrics are read directly off the exported teams. --- .../tests/diag_automata_validation.nim | 145 +++++ common_libs/tests/diag_tm_pattern_offline.nim | 27 + common_libs/tests/test_tm_automata_diag.nim | 241 ++++++++ common_libs/tm_diag/README.md | 143 ++++- common_libs/tm_diag/diagnostics.nim | 558 +++++++++++++++++- 5 files changed, 1109 insertions(+), 5 deletions(-) create mode 100644 common_libs/tests/diag_automata_validation.nim create mode 100644 common_libs/tests/test_tm_automata_diag.nim diff --git a/common_libs/tests/diag_automata_validation.nim b/common_libs/tests/diag_automata_validation.nim new file mode 100644 index 0000000..dce64ae --- /dev/null +++ b/common_libs/tests/diag_automata_validation.nim @@ -0,0 +1,145 @@ +## Task 5 — VALIDATE THE AUTOMATA-LEVEL METRICS against known ground truth. +## +## Case A: the planted-rule set from diag_synthetic.nim (learnable). +## Case B: the SAME inputs with the labels shuffled (pure noise). +## The metrics must SEPARATE learning from fidgeting: +## A: settledness high and rising, churn trend falling, disagreement low. +## B: settledness low, churn high / not falling, disagreement high. +## Case C: a CONSTANT input, to confirm the kit surfaces an information-free bit. +## +## Every number printed here is MEASURED. +## Run: nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim + +import std/[random, strformat, strutils, algorithm] +import tm_diag/diagnostics + +const + NBits = 49 + BitA = 0 + BitB = 45 + NoiseBit = 17 + ConstBit = 20 + NClasses = 3 + Epochs = 15 + +var failures = 0 +proc check(name: string, ok: bool) = + if ok: echo "PASS: ", name + else: echo "FAIL: ", name; inc failures + +proc genDataset(n, seed: int, constant = false): seq[DiagSample] = + var rng = initRand(seed) + for i in 0.. late(full)={lateS.overallMean:.4f} ({settlednessTrend(earlyS, lateS)})" + + echo "\n## CHECKS" + check "A settledness is high at convergence (>= 0.50)", dA.settledness.overallMean >= 0.50 + check "A churn TRENDS DOWN (falling)", + dA.churn.flipTrend == "falling" and dA.churn.flipLast < dA.churn.flipFirst + check "A disagreement is low (< 0.15)", dA.disagreement.overall < 0.15 + check "A settledness RISES from early to late", + lateS.overallMean > earlyS.overallMean + check "B churn does NOT fall (flat/rising/frozen)", + dB.churn.flipTrend in ["flat", "rising", "frozen"] + check "A is clearly more settled than B (A - B >= 0.10)", + dA.settledness.overallMean - dB.settledness.overallMean >= 0.10 + check "B disagrees clearly more than A (B - A >= 0.10)", + dB.disagreement.overall - dA.disagreement.overall >= 0.10 + check "the pair is separated (A verdict settling, B not settling)", + automataVerdict(dA) == "settling" and automataVerdict(dB) != "settling" + + # ── Case C: constant input ── + let trainC = genDataset(2000, 3, constant = true) + let dC = automataDiagnostics(tmpl, trainC, spec, epochs = Epochs, seed = 777) + echo "\n## CASE C — constant input (bit 20 forced to 1)" + var cbit: InputConfidence + for ic in dC.inputConfidence: + if ic.bit == ConstBit: cbit = ic + echo &"# bit{ConstBit} constant={cbit.constant} meanCommit={cbit.meanCommitment:.4f} " & + &"settled={cbit.settledFraction:.4f}" + let consts = constantInputs(trainC, NBits) + echo &"# constantInputs(trainC) = {consts}" + check "the constant bit is flagged in the confidence table", cbit.constant + check "constantInputs() surfaces the constant bit", ConstBit in consts + check "a real planted bit is NOT flagged constant", + not dC.inputConfidence[BitA].constant + + # ── Inertia sweep: how the automata metrics move with N ── + echo "\n## INERTIA SWEEP — settledness / churn / disagreement vs N (10 epochs)" + echo "# case,N,settledness,churnTrend,flipPer100,disagreement,verdict" + for n in [16, 32, 64, 128]: + let tmplN = newMachine(NBits, NClasses, 40, n, 3.0, 1) + let aN = automataDiagnostics(tmplN, trainA, spec, epochs = 10, seed = 777) + let bN = automataDiagnostics(tmplN, trainB, spec, epochs = 10, seed = 777) + echo &"# A,{n},{aN.settledness.overallMean:.3f},{aN.churn.flipTrend}," & + &"{aN.churn.flipRatePer100:.3f},{aN.disagreement.overall:.3f},{automataVerdict(aN)}" + echo &"# B,{n},{bN.settledness.overallMean:.3f},{bN.churn.flipTrend}," & + &"{bN.churn.flipRatePer100:.3f},{bN.disagreement.overall:.3f},{automataVerdict(bN)}" + + echo "" + if failures > 0: + echo &"{failures} check(s) FAILED" + quit(1) + echo "All automata-validation checks passed." diff --git a/common_libs/tests/diag_tm_pattern_offline.nim b/common_libs/tests/diag_tm_pattern_offline.nim index 2938bf6..95aaeb7 100644 --- a/common_libs/tests/diag_tm_pattern_offline.nim +++ b/common_libs/tests/diag_tm_pattern_offline.nim @@ -177,5 +177,32 @@ proc main() = echo &"# class{cls} votes={c.votes:<7} len={c.length} {c.text}" echo &"# -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}" + # ── AUTOMATA-LEVEL metrics on the shipped GF head (Task 5) ── + # settledness / diversity / histogram / per-input confidence / disagreement + # are read DIRECTLY off the exported teams; churn is measured on a tm_core + # retrain (same algorithm) over the captured samples, because the live gun + # does not expose a per-sample state trace. + var ad = AutomataDiag() + ad.machine = m + ad.settledness = settledness(m, 0.5) + ad.diversity = clauseDiversity(m) + ad.histogram = stateHistogram(m, 9) + ad.inputConfidence = perInputConfidence(m, spec, bestSamples, 0.5) + ad.disagreement = voteDisagreement(m, bestSamples) + let churnN = min(bestSamples.len, 10000) + var churnSamples = newSeq[DiagSample](churnN) + for i in 0..0", + stateIncluded(1) and not stateIncluded(0) and not stateIncluded(-1) + +# ── settledness ────────────────────────────────────────────────────────────── + +proc settledMachine(): TmMachine = + ## nBits=4, 2 classes, 4 clauses/class, 8 literals. + ## class0/clause0 = [+64, -64, 0, +32, 0,0,0,0] -> mean commitment 0.625, + ## 3 of 8 automata settled at threshold 0.5. + result = newMachine(4, 2, 4, 64, 3.0, 1) + result.teams[0][0] = 64 + result.teams[0][1] = -64 + result.teams[0][2] = 0 + result.teams[0][3] = 32 + +proc testSettledness() = + let m = settledMachine() + let s = settledness(m, 0.5) + check "settledness covers every clause", + s.clauses.len == m.nClasses * m.nClauses + check "settledness nAutomata = classes*clauses*literals", + s.nAutomata == m.nClasses * m.nClauses * m.nLiterals + var c0: ClauseSettledness + for c in s.clauses: + if c.cls == 0 and c.index == 0: c0 = c + check "clause0 mean commitment = 0.3125", + abs(c0.meanCommitment - 0.3125) < 1e-12 + check "clause0 settled fraction = 3/8", + abs(c0.settledFraction - 3.0 / 8.0) < 1e-12 + check "clause0 positive polarity", c0.polarity == 1 + check "overall settled fraction = 3/64", + abs(s.overallSettledFraction - 3.0 / 64.0) < 1e-12 + check "overall mean commitment = 2.5/64", + abs(s.overallMean - 2.5 / 64.0) < 1e-12 + # trend helper + var lo = s + lo.overallMean = 0.2 + var hi = s + hi.overallMean = 0.8 + check "settlednessTrend rising", settlednessTrend(lo, hi) == "rising" + check "settlednessTrend falling", settlednessTrend(hi, lo) == "falling" + check "settlednessTrend flat", settlednessTrend(s, s) == "flat" + +# ── state histogram ────────────────────────────────────────────────────────── + +proc testHistogram() = + let m = settledMachine() + let h = stateHistogram(m, 9) + var total = 0 + for c in h.counts: total += c + check "histogram counts sum to every automaton", + total == m.nClasses * m.nClauses * m.nLiterals + check "the -64 state lands in the first bin", h.counts[0] >= 1 + check "the +64 state lands in the last bin", h.counts[^1] >= 1 + check "histogram text is non-empty", histogramText(h).len > 0 + +# ── per-input confidence ───────────────────────────────────────────────────── + +proc testInputConfidence() = + let m = settledMachine() + let spec = draftTMSpec() + let samples = @[ + makeSample(49, (block: + var r = newSeq[int](49) + for i in 0..<49: r[i] = 1 + r), 0), + makeSample(49, (block: + var r = newSeq[int](49) + for i in 0..<49: r[i] = 1 + r), 0), + ] + # all-ones samples make every bit constant, so all should be flagged. + let conf = perInputConfidence(m, spec, samples, 0.5) + check "confidence table covers every bit", conf.len == m.nBits + check "confidence nAutomata per bit = classes*clauses*2", + conf[0].nAutomata == m.nClasses * m.nClauses * 2 + check "a zero-variance bit is flagged constant", conf[0].constant + let ranked = rankedInputConfidence(conf) + check "ranked confidence is ascending", + ranked.len == conf.len and + ranked[0].meanCommitment <= ranked[^1].meanCommitment + +# ── diversity / jaccard ────────────────────────────────────────────────────── + +proc testJaccard() = + check "identical sets -> 1", jaccard(@[1, 2, 3], @[1, 2, 3]) == 1.0 + check "disjoint sets -> 0", jaccard(@[1, 2], @[3, 4]) == 0.0 + check "two empty sets -> 1", jaccard(@[], @[]) == 1.0 + check "empty vs non-empty -> 0", jaccard(@[], @[1]) == 0.0 + check "partial overlap -> |n|/|u|", + abs(jaccard(@[1, 2], @[2, 3]) - 1.0 / 3.0) < 1e-12 + +proc diversityMachine(): TmMachine = + ## nBits=4, 2 classes, 6 clauses/class (half=3 positive, 3 negative). + ## class0 positive clauses: {0,1}, {0,1}, {2,3}; negative clauses empty. + result = newMachine(4, 2, 6, 64, 3.0, 1) + for lit in [0, 1]: + result.teams[0][0 * 8 + lit] = 1 + result.teams[0][1 * 8 + lit] = 1 + for lit in [2, 3]: + result.teams[0][2 * 8 + lit] = 1 + +proc testDiversity() = + let m = diversityMachine() + let d = clauseDiversity(m) + check "positive clauses: {0,1},{0,1},{2,3} mean Jaccard = 1/3", + abs(d.jaccardPos - 1.0 / 3.0) < 1e-12 + check "positive diversity is valid", d.posValid + check "empty negative clauses are skipped", not d.negValid + check "overall pools the same 3 pairs", d.nPairs == 3 + check "overall Jaccard = 1/3", + abs(d.jaccardOverall - 1.0 / 3.0) < 1e-12 + +# ── churn ──────────────────────────────────────────────────────────────────── + +proc testChurnHelpers() = + check "binMeans bins [1,1,3,3,5] with window 2 -> [1,3,5]", + binMeans(@[1.0, 1.0, 3.0, 3.0, 5.0], 2) == @[1.0, 3.0, 5.0] + let fl = firstLastMeans(@[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0]) + check "firstLastMeans first ~0", fl.first < 1e-9 + check "firstLastMeans last ~1", abs(fl.last - 1.0) < 1e-9 + check "churn trend falling", + classifyChurnTrend(@[0.9, 0.8, 0.7, 0.6, 0.5, 0.1, 0.05, 0.01]) == "falling" + check "churn trend flat", + classifyChurnTrend(@[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]) == "flat" + check "churn trend rising", + classifyChurnTrend(@[0.01, 0.02, 0.03, 0.1, 0.5, 0.6, 0.7, 0.9]) == "rising" + check "churn trend frozen", + classifyChurnTrend(@[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) == "frozen" + +proc testChurnTrace() = + var samples: seq[DiagSample] + for i in 0..<120: + var raw = newSeq[int](49) + for b in 0..<49: raw[b] = (i + b) mod 2 + raw[0] = 1 + raw[45] = if i mod 2 == 0: 1 else: 0 + samples.add makeSample(49, raw, if raw[45] == 1: 2 else: 1, i) + let tmpl = newMachine(49, 3, 40, 64, 3.0, 1) + let c = churnTrace(tmpl, samples, epochs = 1, seed = 5, window = 30) + check "churn nAutomata = classes*clauses*literals", + c.nAutomata == 3 * 40 * 98 + check "churn nClauses = classes*clauses", c.nClauses == 3 * 40 + check "churn rate is non-negative", c.flipRate >= 0.0 + check "churn per100 = rate*100", + abs(c.flipRatePer100 - c.flipRate * 100.0) < 1e-12 + check "churn windows are non-empty", c.flipWindows.len > 0 + check "churn trained machine has the right shape", + c.trained.teams.len == 3 and c.trained.teams[0].len == 40 * 98 + +# ── vote disagreement ──────────────────────────────────────────────────────── + +proc testVoteDisagreement() = + # all-agree machine: only positive clauses fire -> disagreement 0 + var agree = newMachine(4, 2, 4, 64, 3.0, 1) + agree.teams[0][0 * 8 + 0] = 1 + agree.teams[0][1 * 8 + 1] = 1 + let allOnes = @[makeSample(4, @[1, 1, 1, 1], 0)] + let da = voteDisagreement(agree, allOnes) + check "all-positive firing clauses -> disagreement 0", + abs(da.overall - 0.0) < 1e-12 + check "one class observed", da.perClassSamples[0] == 1 + + # split machine: one positive and one negative clause fire -> tied vote -> 1 + var split = newMachine(4, 2, 4, 64, 3.0, 1) + split.teams[0][0 * 8 + 0] = 1 + split.teams[0][2 * 8 + 2] = 1 + let ds = voteDisagreement(split, allOnes) + check "a tied vote counts every firing clause as disagreeing", + abs(ds.overall - 1.0) < 1e-12 + +# ── health line / verdict / report ─────────────────────────────────────────── + +proc testHealthLine() = + let spec = draftTMSpec() + var samples: seq[DiagSample] + for i in 0..<200: + var raw = newSeq[int](49) + for b in 0..<49: raw[b] = if (i * 7 + b) mod 3 == 0: 1 else: 0 + raw[0] = if i mod 2 == 0: 1 else: 0 + raw[45] = if i mod 3 == 0: 1 else: 0 + let label = if raw[0] == 1 and raw[45] == 1: 2 elif raw[0] == 1: 1 else: 0 + samples.add makeSample(49, raw, label, i) + let tmpl = newMachine(49, 3, 40, 64, 3.0, 1) + let d = automataDiagnostics(tmpl, samples, spec, epochs = 3, seed = 7, + settleThreshold = 0.5, nHistBins = 9) + check "health line is non-empty and mentions every metric", + d.summary.len > 0 and "settledness" in d.summary and + "diversity" in d.summary and "churn" in d.summary and + "disagreement" in d.summary + check "automataVerdict is non-empty", automataVerdict(d).len > 0 + check "formatAutomataReport contains the histogram and table", + "state histogram" in formatAutomataReport(d) and + "per-input confidence" in formatAutomataReport(d) + check "disagreement per-class array is the right size", + d.disagreement.perClass.len == 3 + check "input confidence covers all bits", d.inputConfidence.len == 49 + +when isMainModule: + testStateConventions() + testSettledness() + testHistogram() + testInputConfidence() + testJaccard() + testDiversity() + testChurnHelpers() + testChurnTrace() + testVoteDisagreement() + testHealthLine() + echo "" + if failures > 0: + echo failures, " / ", checks, " check(s) FAILED" + quit(1) + echo "All ", checks, " automata-diag unit checks passed." diff --git a/common_libs/tm_diag/README.md b/common_libs/tm_diag/README.md index e8704c4..60c9b8b 100644 --- a/common_libs/tm_diag/README.md +++ b/common_libs/tm_diag/README.md @@ -16,7 +16,7 @@ Everything is pure and offline — no battles, no Java, no harness. |---|---| | `feature_spec.nim` | `FeatureSpec`, `describe`, `describeClause`, `draftTMSpec()` (49-bit draft), `tmPatternSpec()` (40-bit shipped encoding) | | `tm_core.nim` | compact deterministic Granmo Table 2/3 multiclass TM (mirrors the tm_pattern core), introspectable clause layout | -| `diagnostics.nim` | the six groups (re-exports the two above) | +| `diagnostics.nim` | the seven groups (re-exports the two above) | Import everything with: @@ -93,6 +93,143 @@ let rs = ablateScrambleFeature(tmplMachine, train, eval, spec, bit) input earn its bits"). Pass `baselineAcc` from `ablateBaseline` to avoid recomputing it per block. +## Task 5 — the AUTOMATA level (settledness / diversity / churn / disagreement) + +Task 2 reads the clauses. Task 5 reads the **automata inside them**: one +automaton per input bit per clause, each holding a state that says how +confident it is that its bit belongs in the clause. These are what tell us +whether the TM is locking onto the enemy or just fidgeting, and whether the +inertia `N` (the number of automata states) should go up or down. + +### State conventions — from the ACTUAL code, not the textbook + +Derived from `tm_core.nim` and `guns/tm_pattern.nim` (`tmEval`, `tmLearnDir`, +`tmNewTeam`, `resetMachine`): + +| property | value | +|---|---| +| state range | `[-nStates, nStates]` (int16); `nStates` = 64 for `tm_pattern`, 32 for `tsetlin.nim` | +| initial value | `0` (both cores call it "the Exclude boundary") | +| INCLUDE | `state > 0` (`tmEval` / `clauseLits`) | +| EXCLUDE | `state <= 0` | +| flip boundary | BETWEEN state `0` and state `1` — the middle of the range | +| `commitment(st)` | `abs(st) / nStates` in `[0,1]`: 0 on the boundary, 1 at either extreme | + +So a flip is exactly a change in the predicate `state > 0`. + +### The four metrics + +1. **SETTLEDNESS** — per clause and overall, the mean `commitment` and the + fraction of automata settled at/above a threshold (default `0.5`). High and + **rising** as training proceeds is healthy; low means the clause is wavering + noise. `settledness(m, threshold)`, `settlednessTrend(early, late)`. +2. **CLAUSE DIVERSITY** — mean pairwise **Jaccard** of the included-literal sets, + within the same polarity. Low-to-moderate is healthy; ~1.0 = all clauses are + one rule in 50 hats; ~0 = memorising ticks. Empty clauses carry no rule and + are skipped by default. `clauseDiversity(m, skipEmpty = true)`, `jaccard(a,b)`. +3. **CHURN** — both levels, per training sample: the fraction of **automata** + crossing the flip boundary, and the fraction of **clauses** whose + included-literal set changed. High early and **falling** is healthy; flat-high + = fidgeting; zero from the start = never learned. `churnTrace(tmpl, samples, + epochs, seed, window, shuffle)`. `shuffle = false` replays the given + (temporal) order, mirroring a live gun. +4. **VOTE DISAGREEMENT** — per class, over the samples where it casts a vote: + the fraction of firing clauses whose polarity disagrees with the sign of the + class's total vote. Low is healthy. `voteDisagreement(m, samples)`. + +### The three readouts that make it readable + +- **state histogram** — the distribution of automata states across the range: + `stateHistogram(m, nBins)`, `histogramText(h)`. +- **per-input confidence table** — for each bit, the mean commitment of its + automata (and a `constant` flag for zero-variance inputs): + `perInputConfidence(m, spec, samples, threshold)`, + `rankedInputConfidence(conf)`. +- **one-line health summary** — `healthLine(ad)` gives + `settledness / diversity / churn trend / disagreement`, each with its own + verdict word (`settling`, `fidgeting`, `frozen`, `coherent`, ...), and + `automataVerdict(ad)` reduces the trajectory to one word. `formatAutomataReport(ad)` + prints everything. + +### API + +```nim +import tm_diag/diagnostics + +let ad = automataDiagnostics(tmpl, samples, spec, + epochs = 15, seed = 777, + settleThreshold = 0.5, nHistBins = 9, + window = 100, measureChurn = true) +# ad.machine, ad.settledness, ad.diversity, ad.churn, ad.disagreement, +# ad.histogram, ad.inputConfidence, ad.summary +echo ad.summary # one line +echo automataVerdict(ad) # "settling" | "fidgeting" | "collapsed" | ... +echo formatAutomataReport(ad) # the full readout +``` + +For a machine you already have (e.g. the shipped gun's `exportTeams()`), call +`settledness` / `clauseDiversity` / `stateHistogram` / `perInputConfidence` / +`voteDisagreement` directly and `churnTrace` on a fresh copy. + +### Validation — Case A (learnable) vs Case B (noise) + +`common_libs/tests/diag_automata_validation.nim` uses the planted rule from +`diag_synthetic.nim` and the SAME inputs with shuffled labels. Measured +(`nBits=49`, 3 classes, 40 clauses, `nStates=64`, 3000 samples, 15 epochs): + +| metric | Case A (learnable) | Case B (noise) | +|---|---|---| +| settledness mean | **0.970** | 0.719 | +| settled fraction | 0.999 | 0.799 | +| churn trend | **falling** (`2.5e-4` -> `2e-6`) | **flat** (`1.0e-3` -> `7.2e-4`) | +| clause-change trend | falling (`0.011` -> `2.3e-4`) | flat (`0.069` -> `0.057`) | +| diversity (overall Jaccard) | 0.176 | 0.014 | +| disagreement | **0.003** | **0.298** | +| verdict | settling | mixed (not settling) | + +Case A settledness RISES `0.475` (early prefix) -> `0.970` (full). The pair +**separates learning from fidgeting**: the decisive signals are the churn trend +(falling vs flat) and disagreement (0.003 vs 0.298). Settledness alone is NOT +enough — on noise the automata still commit (0.719), just to the wrong thing. +Case C (a forced-constant bit) is flagged: `perInputConfidence(...).constant` +and `constantInputs` both surface it. Note a constant bit can show HIGH +commitment (one literal is always 1), so the `constant` flag is what + disambiguates. + +### Inertia sweep (`N` = nStates, 10 epochs) + +The validation also sweeps `N` to see whether inertia moves the metrics: + +| N | Case A settled | A churn | A dis. | Case B settled | B churn | B dis. | +|---|---|---|---|---|---|---| +| 16 | 0.904 | falling | 0.003 | 0.720 | flat | 0.363 | +| 32 | 0.943 | falling | 0.000 | 0.700 | flat | 0.388 | +| 64 | 0.970 | falling | 0.003 | 0.715 | flat | 0.368 | +| 128 | 0.971 | falling | 0.000 | 0.660 | flat | 0.304 | + +A and B are separated at EVERY N (churn falling vs flat, disagreement low vs +high). Raising N only raises Case A's commitment (0.90 -> 0.97); it does NOT +reduce noise-fitting in Case B. So **inertia is not the discriminator** the +metrics identify — the churn trend and disagreement are. + +### Real reading — the shipped `tm_pattern` GF head + +`diag_tm_pattern_offline.nim` over the committed DrussGT fixtures (automata read +directly off the exported teams from `tr_drussgt_vs_modularbot`; churn measured +on a `tm_core` temporal one-pass proxy, live-order): + +- **settledness 0.484** (`settling`), settled fraction 0.453 +- **diversity 0.267** (`moderate`) — positive 0.170, negative 0.322 +- **churn 0.094/100 per sample, FALLING** (0.00208 -> 0.00048); clause-change 5.31/100, falling +- **disagreement 0.145** (`coherent`) +- **verdict: settling** — not fidgeting, not collapsed +- constant inputs flagged: bits 38/39 (the known never-written ones) plus 19/36/37 in this fixture +- context: pooled warm accuracy 35.72% vs the 34.24% majority = **+1.48pp** + +The gun settles onto the within-battle labels but its settled rules barely beat +the majority class, which points at the TARGET / representation rather than the +inertia `N`. See the Task 5 report for the inertia discussion. + ## The default-off real-gun hook `guns/tm_pattern.nim` gained only additive, default-off instrumentation: @@ -116,7 +253,9 @@ let m = machineFromTeams(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S, ```sh nim c -r -d:release --path:common_libs common_libs/tests/test_tm_diag.nim # 48 pure unit checks -nim c -r -d:release --path:common_libs common_libs/tests/diag_synthetic.nim # Task 3 proof +nim c -r --path:common_libs common_libs/tests/test_tm_automata_diag.nim # 55 automata-metric unit checks +nim c -r -d:release --path:common_libs common_libs/tests/diag_synthetic.nim # Task 3 proof (17 checks) +nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim # Task 5 A/B/C proof nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim # Task 4 real reading ``` diff --git a/common_libs/tm_diag/diagnostics.nim b/common_libs/tm_diag/diagnostics.nim index e245c29..9d9fe52 100644 --- a/common_libs/tm_diag/diagnostics.nim +++ b/common_libs/tm_diag/diagnostics.nim @@ -1,20 +1,24 @@ -## tm_diag/diagnostics.nim — THE DIAGNOSTICS KIT (Task 2, six groups). +## tm_diag/diagnostics.nim — THE DIAGNOSTICS KIT (Tasks 2 + 5). ## ## Everything runs OFFLINE: a trained `TmMachine` plus a labelled sample set. -## No battles, no harness, no Java. The six groups are: +## No battles, no harness, no Java. The seven groups are: ## 1. pre-flight DATA checks (dataChecks, shuffledLabelControl) ## 2. clause introspection (clauseInfo, clauseSummary, topClauses) ## 3. per-feature contribution (featureContributions, deadInputs, ...) ## 4. accuracy diagnostics (accuracyDiagnostics) ## 5. learning curve (learningCurve) ## 6. ablation hooks (ablateDropBlock, ablateScrambleFeature, ...) +## 7. AUTOMATA level (Task 5) (settledness, clauseDiversity, churnTrace, +## voteDisagreement, stateHistogram, +## perInputConfidence, healthLine, +## automataDiagnostics) ## ## A `DiagSample` carries the LITERAL vector (positive literals [0..nBits), ## negations [nBits..2*nBits)) exactly as the shipped TM stores it, plus the ## true label. This keeps the kit usable both on the reference core and on the ## literal vectors captured from the real gun. -import std/[math, strformat, algorithm] +import std/[math, strformat, algorithm, strutils] import feature_spec import tm_core @@ -587,3 +591,551 @@ proc ablateBaseline*(tmpl: TmMachine, train, eval: openArray[DiagSample], baselineAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval), ablatedAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval), delta: 0.0) + +# ───────────────────────────────────────────────────────────────────────────── +# GROUP 7 — automata-level metrics +# settledness / clause diversity / churn / vote disagreement +# plus the state histogram, the per-input confidence table and a health line. +# +# STATE CONVENTIONS — derived from the ACTUAL code (tm_core.nim and +# guns/tm_pattern.nim), not from the textbook formulation: +# * every clause owns one automaton per literal (2 * nBits of them); +# * states are int16 in [-nStates, nStates]; +# * INCLUDE iff state > 0, EXCLUDE iff state <= 0 (tmEval / clauseLits); +# * both cores initialise every state to 0 ("the Exclude boundary"), so the +# flip boundary sits BETWEEN state 0 (Exclude) and state 1 (Include) — the +# middle of the range. +# * commitment(st) = |st| / nStates in [0, 1]: 0 exactly on the boundary, +# 1 at either extreme. It is symmetric for the two sides. +# ───────────────────────────────────────────────────────────────────────────── + +proc stateCommitment*(nStates, st: int): float {.inline.} = + ## Normalised distance of an automaton state from its flip boundary, in + ## [0, 1]. 0 == on the boundary (state 0), 1 == at either extreme (+/-nStates). + min(abs(st).float / max(1, nStates).float, 1.0) + +proc stateIncluded*(st: int): bool {.inline.} = + ## The kit's include predicate, matching `tmEval` / `clauseLits` exactly. + st > 0 + +# ── 1. SETTLEDNESS ─────────────────────────────────────────────────────────── + +type + ClauseSettledness* = object + cls*: int + index*: int + polarity*: int ## +1 positive clauses, -1 negative clauses + nAutomata*: int + meanCommitment*: float + settledFraction*: float + + SettlednessResult* = object + threshold*: float + nAutomata*: int + overallMean*: float + overallSettledFraction*: float + clauses*: seq[ClauseSettledness] + +proc settledness*(m: TmMachine, threshold = 0.5): SettlednessResult = + ## Per clause and overall: the mean automaton `commitment` and the fraction of + ## automata whose commitment is >= `threshold`. A clause full of automata + ## parked near the boundary is wavering noise; one with automata pushed to an + ## extreme has committed to a rule. + result.threshold = threshold + var sum = 0.0 + var settled = 0 + for c in 0..= threshold: inc csettled + result.clauses.add ClauseSettledness( + cls: c, index: cl, + polarity: (if cl < m.half: 1 else: -1), + nAutomata: m.nLiterals, + meanCommitment: csum / m.nLiterals.float, + settledFraction: csettled.float / m.nLiterals.float) + sum += csum + settled += csettled + inc result.nAutomata, m.nLiterals + result.overallMean = + if result.nAutomata > 0: sum / result.nAutomata.float else: 0.0 + result.overallSettledFraction = + if result.nAutomata > 0: settled.float / result.nAutomata.float else: 0.0 + +proc settlednessTrend*(early, late: SettlednessResult): string = + ## Classify whether settledness ROSE between two snapshots (the healthy sign). + let d = late.overallMean - early.overallMean + if d > 0.05: "rising" + elif d < -0.05: "falling" + else: "flat" + +# ── state histogram (the raw picture behind settledness) ───────────────────── + +type + StateHistogram* = object + nStates*: int + nBins*: int + binLo*: seq[int] ## inclusive low state observed in the bin + binHi*: seq[int] ## inclusive high state observed in the bin + counts*: seq[int] + total*: int + +proc stateHistogram*(m: TmMachine, nBins = 9): StateHistogram = + ## Distribution of every automaton state across [-nStates, nStates]. + let nb = max(1, nBins) + let span = 2 * m.nStates + 1 + result.nStates = m.nStates + result.nBins = nb + result.counts = newSeq[int](nb) + result.binLo = newSeq[int](nb) + result.binHi = newSeq[int](nb) + for b in 0..= nb: bin = nb - 1 + inc result.counts[bin] + if st < result.binLo[bin]: result.binLo[bin] = st + if st > result.binHi[bin]: result.binHi[bin] = st + inc result.total + for b in 0..4}..{h.binHi[b]:>4}] {h.counts[b]:>8} {bar}\n" + +# ── per-input automata confidence table ────────────────────────────────────── + +type + InputConfidence* = object + bit*: int + name*: string + nAutomata*: int + meanCommitment*: float + settledFraction*: float + constant*: bool ## zero variance across the sample set (information-free) + +proc perInputConfidence*(m: TmMachine, spec: FeatureSpec, + samples: openArray[DiagSample], + threshold = 0.5): seq[InputConfidence] = + ## For every raw bit: the mean commitment of the automata of its positive + ## literal and its negation, across every clause and class. Inputs the TM is + ## confident about sit high; a bit whose automata are all parked at the + ## boundary (e.g. a constant input) sits near 0. `constant` flags zero-variance + ## bits so a high commitment on a never-written input is not mistaken for + ## learning (the shipped gun's bits 38/39 are the canonical example). + result = newSeq[InputConfidence](m.nBits) + var consts: seq[int] + if samples.len > 0: + for b in 0..= threshold: inc settled + if cmNeg >= threshold: inc settled + inc n, 2 + result[b].nAutomata = n + result[b].meanCommitment = if n > 0: sum / n.float else: 0.0 + result[b].settledFraction = if n > 0: settled.float / n.float else: 0.0 + +proc rankedInputConfidence*(conf: seq[InputConfidence]): seq[InputConfidence] = + ## Least confident first — the inputs whose automata are still undecided. + result = conf + result.sort(proc(a, b: InputConfidence): int = + result = cmp(a.meanCommitment, b.meanCommitment) + if result == 0: result = cmp(a.bit, b.bit)) + +# ── 2. CLAUSE DIVERSITY ────────────────────────────────────────────────────── + +type + DiversityResult* = object + jaccardPos*: float ## mean pairwise Jaccard among positive clauses + jaccardNeg*: float ## ... among negative clauses + jaccardOverall*: float ## pooled over both polarities + nPosClauses*: int + nNegClauses*: int + nPairs*: int + posValid*: bool + negValid*: bool + valid*: bool + skipEmpty*: bool + +proc jaccard*(a, b: openArray[int]): float = + ## |a n b| / |a u b| for two sorted, unique literal sets. Two empty sets are + ## identical (1.0); an empty against a non-empty set share nothing (0.0). + if a.len == 0 and b.len == 0: return 1.0 + if a.len == 0 or b.len == 0: return 0.0 + var i = 0 + var j = 0 + var inter = 0 + while i < a.len and j < b.len: + if a[i] == b[j]: inc inter; inc i; inc j + elif a[i] < b[j]: inc i + else: inc j + let uni = a.len + b.len - inter + if uni == 0: 1.0 else: inter.float / uni.float + +proc polarityDiversity(m: TmMachine, polarity: int, skipEmpty: bool): + tuple[sum: float, npairs, nclauses: int] = + var sets: seq[seq[int]] + let cl0 = if polarity > 0: 0 else: m.half + let cl1 = if polarity > 0: m.half else: m.nClauses + for c in 0.. 0 + result.negValid = n.npairs > 0 + result.jaccardPos = if p.npairs > 0: p.sum / p.npairs.float else: 0.0 + result.jaccardNeg = if n.npairs > 0: n.sum / n.npairs.float else: 0.0 + let pairs = p.npairs + n.npairs + result.nPairs = pairs + result.valid = pairs > 0 + result.jaccardOverall = + if pairs > 0: (p.sum + n.sum) / pairs.float else: 0.0 + +# ── 3. CHURN ───────────────────────────────────────────────────────────────── + +type + ChurnResult* = object + samples*: int + epochs*: int + nAutomata*: int + nClauses*: int + totalFlips*: int + totalClauseChanges*: int + flipRate*: float ## automaton flips per training sample (fraction) + flipRatePer100*: float ## same, x100 (readable) + clauseChangeRate*: float ## clause-set changes per training sample + clauseChangePer100*: float + flipFirst*: float + flipLast*: float + clauseFirst*: float + clauseLast*: float + flipTrend*: string + clauseTrend*: string + windowSize*: int + flipWindows*: seq[float] ## mean per-sample flip rate per window + clauseWindows*: seq[float] + trained*: TmMachine + +proc binMeans*(s: seq[float], window: int): seq[float] = + if s.len == 0: return + let w = max(1, window) + var i = 0 + while i < s.len: + let j = min(s.len, i + w) + var sum = 0.0 + for k in i.. 1.6: "rising" + else: "flat" + +proc churnTrace*(tmpl: TmMachine, samples: openArray[DiagSample], + epochs = 1, seed = 777'u64, window = 100, + shuffle = true): ChurnResult = + ## Train a fresh machine while measuring, per training sample, both churn + ## levels: the fraction of AUTOMATA crossing the flip boundary, and the + ## fraction of CLAUSES whose included-literal set changed. `shuffle = false` + ## replays the samples in their given (temporal) order once per epoch, which + ## mirrors a live gun training as bullets resolve. + result.epochs = max(1, epochs) + result.samples = samples.len + result.windowSize = max(1, window) + var m = tmpl + m.resetMachine(seed) + result.nAutomata = m.nClasses * m.nClauses * m.nLiterals + result.nClauses = m.nClasses * m.nClauses + if result.nAutomata == 0: + result.trained = m + return + var prevInc = newSeq[uint8](result.nAutomata) + for c in 0.. 0) + var flipSeries = newSeq[float]() + var clauseSeries = newSeq[float]() + for _ in 0.. 0) + if nowInc != prevInc[gb + lit]: + inc flips + changed = true + prevInc[gb + lit] = nowInc + if changed: inc clauseChanges + result.totalFlips += flips + result.totalClauseChanges += clauseChanges + flipSeries.add flips.float / result.nAutomata.float + clauseSeries.add clauseChanges.float / result.nClauses.float + result.trained = m + let nObs = flipSeries.len + if nObs > 0: + result.flipRate = result.totalFlips.float / + (nObs.float * result.nAutomata.float) + result.clauseChangeRate = result.totalClauseChanges.float / + (nObs.float * result.nClauses.float) + result.flipRatePer100 = result.flipRate * 100.0 + result.clauseChangePer100 = result.clauseChangeRate * 100.0 + let ff = firstLastMeans(flipSeries) + let cf = firstLastMeans(clauseSeries) + result.flipFirst = ff.first + result.flipLast = ff.last + result.clauseFirst = cf.first + result.clauseLast = cf.last + result.flipTrend = classifyChurnTrend(flipSeries) + result.clauseTrend = classifyChurnTrend(clauseSeries) + result.flipWindows = binMeans(flipSeries, result.windowSize) + result.clauseWindows = binMeans(clauseSeries, result.windowSize) + +# ── 4. VOTE DISAGREEMENT ───────────────────────────────────────────────────── + +type + VoteDisagreementResult* = object + overall*: float + perClass*: seq[float] + perClassSamples*: seq[int] + nPairs*: int ## (class, sample) observations that had a vote + +proc voteDisagreement*(m: TmMachine, samples: openArray[DiagSample]): + VoteDisagreementResult = + ## For each class, over the samples where it casts at least one clause vote: + ## the fraction of FIRING clauses whose own polarity disagrees with the sign + ## of the class's total vote. A class whose clauses always pull together has + ## disagreement ~0; a fuzzy class boundary or too few clauses makes it high. + ## A tied class vote (mixed signs) counts every firing clause as disagreeing. + result.perClass = newSeq[float](m.nClasses) + result.perClassSamples = newSeq[int](m.nClasses) + var classSum = newSeq[float](m.nClasses) + for c in 0.. 0.0: 1 elif vote < 0.0: -1 else: 0 + var disagree = 0 + for cl in firing: + let pol = if m.tmPolarity(cl) > 0.0: 1 else: -1 + if pol != majSign: inc disagree + classSum[c] += disagree.float / firing.len.float + inc result.perClassSamples[c] + inc result.nPairs + var totalSum = 0.0 + var totalN = 0 + for c in 0.. 0: + result.perClass[c] = classSum[c] / result.perClassSamples[c].float + totalSum += result.perClass[c] + inc totalN + result.overall = if totalN > 0: totalSum / totalN.float else: 0.0 + +# ── the convenience driver + the one-line health summary ───────────────────── + +type + AutomataDiag* = object + machine*: TmMachine + settledness*: SettlednessResult + diversity*: DiversityResult + churn*: ChurnResult + disagreement*: VoteDisagreementResult + histogram*: StateHistogram + inputConfidence*: seq[InputConfidence] + summary*: string + +proc settlednessVerdict*(s: SettlednessResult): string = + if s.overallMean >= 0.65: "settled" + elif s.overallMean >= 0.40: "settling" + else: "wavering" + +proc diversityVerdict*(d: DiversityResult): string = + if not d.valid: "n/a" + elif d.jaccardOverall < 0.15: "diverse" + elif d.jaccardOverall <= 0.70: "moderate" + else: "redundant" + +proc churnVerdict*(c: ChurnResult): string = + case c.flipTrend + of "falling": "settling" + of "rising": "fidgeting (rising)" + of "frozen": "frozen" + of "insufficient", "not-measured": c.flipTrend + else: + if c.flipRatePer100 > 5.0: "fidgeting" else: "quiet" + +proc disagreementVerdict*(d: VoteDisagreementResult): string = + if d.overall < 0.15: "coherent" + elif d.overall < 0.35: "fuzzy" + else: "fragmented" + +proc healthLine*(d: AutomataDiag): string = + ## ONE line to glance at: settledness / diversity / churn trend / disagreement, + ## each with its own verdict word. + &"settledness={d.settledness.overallMean:.3f} ({settlednessVerdict(d.settledness)}) | " & + &"diversity={d.diversity.jaccardOverall:.3f} ({diversityVerdict(d.diversity)}) | " & + &"churn={d.churn.flipRatePer100:.3f}/100 {d.churn.flipTrend} " & + &"({churnVerdict(d.churn)}) | " & + &"disagreement={d.disagreement.overall:.3f} " & + &"({disagreementVerdict(d.disagreement)})" + +proc automataDiagnostics*(tmpl: TmMachine, samples: openArray[DiagSample], + spec: FeatureSpec, epochs = 1, seed = 777'u64, + settleThreshold = 0.5, nHistBins = 9, + window = 100, measureChurn = true): AutomataDiag = + ## Train a fresh machine on `samples` (measuring churn along the way unless + ## `measureChurn` is false) and compute every automata-level metric on it. + if measureChurn: + result.churn = churnTrace(tmpl, samples, epochs, seed, window) + result.machine = result.churn.trained + else: + result.machine = trainModel(tmpl, samples, epochs, seed) + result.churn.epochs = max(1, epochs) + result.churn.samples = samples.len + result.churn.trained = result.machine + result.churn.flipTrend = "not-measured" + result.churn.clauseTrend = "not-measured" + result.settledness = settledness(result.machine, settleThreshold) + result.diversity = clauseDiversity(result.machine) + result.disagreement = voteDisagreement(result.machine, samples) + result.histogram = stateHistogram(result.machine, nHistBins) + result.inputConfidence = perInputConfidence(result.machine, spec, samples, + settleThreshold) + result.summary = healthLine(result) + +proc automataVerdict*(d: AutomataDiag): string = + ## One word for the whole trajectory: settling / fidgeting / collapsed / other. + if d.churn.flipTrend == "falling" and d.settledness.overallMean >= 0.40: + "settling" + elif d.churn.flipTrend in ["flat", "rising"] and d.churn.flipRatePer100 > 5.0: + "fidgeting" + elif d.settledness.overallMean < 0.25 and d.churn.flipTrend == "frozen": + "collapsed" + elif d.disagreement.overall > 0.35 and d.settledness.overallMean < 0.40: + "collapsed" + elif d.settledness.overallMean >= 0.65 and d.churn.flipTrend == "frozen": + "converged-static" + else: + "mixed" + +proc formatAutomataReport*(d: AutomataDiag): string = + ## The full readout: health line, verdict, state histogram, per-input + ## confidence (least confident first) and per-class disagreement. + result.add "health: " & d.summary & "\n" + result.add "verdict: " & automataVerdict(d) & "\n" + result.add &"automata: n={d.settledness.nAutomata} settled>={d.settledness.threshold:.2f} " & + &"frac={d.settledness.overallSettledFraction:.3f}\n" + result.add &"churn: flips/sample={d.churn.flipRate:.6f} ({d.churn.flipRatePer100:.3f}/100, " & + &"{d.churn.flipTrend}: {d.churn.flipFirst:.6f}->{d.churn.flipLast:.6f}) " & + &"clause-change/sample={d.churn.clauseChangeRate:.6f} " & + &"({d.churn.clauseChangePer100:.3f}/100, {d.churn.clauseTrend})\n" + result.add &"diversity: pos={d.diversity.jaccardPos:.3f} " & + &"neg={d.diversity.jaccardNeg:.3f} overall={d.diversity.jaccardOverall:.3f} " & + &"(pairs={d.diversity.nPairs}, posClauses={d.diversity.nPosClauses}, " & + &"negClauses={d.diversity.nNegClauses})\n" + result.add "state histogram:\n" & histogramText(d.histogram) + result.add "per-input confidence (least confident first):\n" + let ranked = rankedInputConfidence(d.inputConfidence) + for i in 0..2} {ranked[i].name:<28} " & + &"meanCommit={ranked[i].meanCommitment:.3f} settled={ranked[i].settledFraction:.3f}{tag}\n" + var cbits: seq[int] + for ic in d.inputConfidence: + if ic.constant: cbits.add ic.bit + result.add &"constant inputs (zero variance): {cbits}\n" + result.add "disagreement per class:" + for c in 0..