Automata metrics: settledness alone does NOT separate learning from fidgeting
Added the four automata-level metrics to the TM diagnostics kit (settledness, clause diversity, churn, vote disagreement) plus a state histogram, a per-input confidence table and a one-line health summary, and validated them on a learnable-vs-noise pair. STATE CONVENTIONS, read off OUR code rather than from memory: range [-nStates, nStates] as int16; nStates = 64 for tm_pattern, 32 for tsetlin initial value 0 = the Exclude boundary INCLUDE iff state > 0; EXCLUDE iff state <= 0 flip boundary sits between state 0 and 1; commitment = abs(st)/nStates in [0,1] === THE GATE, AND A RESULT THAT MATTERS === Case A (learnable planted rule) vs Case B (shuffled labels), 49 bits, N=64: metric A (learnable) B (shuffled) settledness mean 0.970 0.719 churn flip/sample 0.000055 FALLING 0.000788 FLAT clause-change/sample 0.00263 falling 0.0595 flat diversity (Jaccard) 0.176 0.014 disagreement 0.003 0.298 verdict settling mixed (NOT settling) **SETTLEDNESS ALONE DOES NOT WORK.** On noise the automata still COMMIT (0.719) - they just commit to the wrong thing. The decisive separators are **churn TREND (falling vs flat)** and **vote DISAGREEMENT (0.003 vs 0.298)**. Had we built only the settledness metric - the one that seems most obvious - we would have been misled. That is now recorded in the README. INERTIA SWEEP: A vs B separate at N=16/32/64/128. **Raising N raises A's commitment but does NOT reduce B's noise-fitting** - so more inertia does not rescue a noise-fitting TM. === REAL READING ON THE SHIPPED GUN, AND THE INFERENCE IT SUPPORTS === tm_pattern GF head over the DrussGT fixtures: settledness 0.484 (settling), diversity 0.267 (moderate), churn 0.094/100 FALLING, disagreement 0.145 (coherent). **VERDICT: SETTLING** - not fidgeting, not collapsed. Constant inputs flagged: 38/39 (the known never-written bits) plus 19/36/37. Context: pooled warm accuracy 35.72% vs 34.24% majority = +1.48pp. So: **the old gun was NOT failing because of inertia or instability - it settled properly and its settled rules still barely beat a lazy guess.** Its settledness (0.484) is LOWER than both synthetic cases (0.97/0.72), which is the signature of WEAK OR CONFLICTING SIGNAL rather than too much inertia. CONCLUSION: **N and s are not the observed bottleneck. The target/representation is.** That is exactly why the new design changes the target and the label pipeline rather than sweeping knobs - and it means we should NOT spend effort on an N/s sweep expecting it to fix anything. Also adds `diag_automata_validation.nim` (Case A/B/C + inertia sweep) and `test_tm_automata_diag.nim` (55 pure checks); `test_tm_diag` 48 and `diag_synthetic` 17 still pass, plus all other guards. acceptance_offline_vs_online was NOT run (it needs a live battle and there is no tm_diag dependency). Caveat: churn on the real gun is a PROXY (a tm_core retrain over captured samples in live order) because the live gun exposes no per-sample state trace; the other metrics are read directly off the exported teams.
This commit is contained in:
@@ -0,0 +1,145 @@
|
|||||||
|
## Task 5 — VALIDATE THE AUTOMATA-LEVEL METRICS against known ground truth.
|
||||||
|
##
|
||||||
|
## Case A: the planted-rule set from diag_synthetic.nim (learnable).
|
||||||
|
## Case B: the SAME inputs with the labels shuffled (pure noise).
|
||||||
|
## The metrics must SEPARATE learning from fidgeting:
|
||||||
|
## A: settledness high and rising, churn trend falling, disagreement low.
|
||||||
|
## B: settledness low, churn high / not falling, disagreement high.
|
||||||
|
## Case C: a CONSTANT input, to confirm the kit surfaces an information-free bit.
|
||||||
|
##
|
||||||
|
## Every number printed here is MEASURED.
|
||||||
|
## Run: nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim
|
||||||
|
|
||||||
|
import std/[random, strformat, strutils, algorithm]
|
||||||
|
import tm_diag/diagnostics
|
||||||
|
|
||||||
|
const
|
||||||
|
NBits = 49
|
||||||
|
BitA = 0
|
||||||
|
BitB = 45
|
||||||
|
NoiseBit = 17
|
||||||
|
ConstBit = 20
|
||||||
|
NClasses = 3
|
||||||
|
Epochs = 15
|
||||||
|
|
||||||
|
var failures = 0
|
||||||
|
proc check(name: string, ok: bool) =
|
||||||
|
if ok: echo "PASS: ", name
|
||||||
|
else: echo "FAIL: ", name; inc failures
|
||||||
|
|
||||||
|
proc genDataset(n, seed: int, constant = false): seq[DiagSample] =
|
||||||
|
var rng = initRand(seed)
|
||||||
|
for i in 0..<n:
|
||||||
|
var raw = newSeq[int](NBits)
|
||||||
|
for b in 0..<NBits:
|
||||||
|
raw[b] = (if rng.rand(1.0) < 0.5: 1 else: 0)
|
||||||
|
let a = if rng.rand(1.0) < 0.4: 1 else: 0
|
||||||
|
let bb = if rng.rand(1.0) < 0.5: 1 else: 0
|
||||||
|
raw[BitA] = a
|
||||||
|
raw[BitB] = bb
|
||||||
|
if constant: raw[ConstBit] = 1
|
||||||
|
let label =
|
||||||
|
if a == 1 and bb == 1: 2
|
||||||
|
elif a == 1: 1
|
||||||
|
else: 0
|
||||||
|
result.add makeSample(NBits, raw, label, i)
|
||||||
|
|
||||||
|
proc shuffleLabels(s: seq[DiagSample], seed: int): seq[DiagSample] =
|
||||||
|
result = s
|
||||||
|
var rng = initRand(seed)
|
||||||
|
var labels = newSeq[int](s.len)
|
||||||
|
for i in 0..<s.len: labels[i] = s[i].label
|
||||||
|
for i in countdown(s.len - 1, 1):
|
||||||
|
let j = rng.rand(i)
|
||||||
|
swap(labels[i], labels[j])
|
||||||
|
for i in 0..<s.len: result[i].label = labels[i]
|
||||||
|
|
||||||
|
proc report(tag: string, d: AutomataDiag) =
|
||||||
|
echo &"\n## {tag}"
|
||||||
|
echo "# ", d.summary
|
||||||
|
echo "# verdict=", automataVerdict(d)
|
||||||
|
echo &"# settledness: mean={d.settledness.overallMean:.4f} " &
|
||||||
|
&"settledFrac={d.settledness.overallSettledFraction:.4f} " &
|
||||||
|
&"(threshold={d.settledness.threshold:.2f})"
|
||||||
|
echo &"# diversity: pos={d.diversity.jaccardPos:.4f} neg={d.diversity.jaccardNeg:.4f} " &
|
||||||
|
&"overall={d.diversity.jaccardOverall:.4f} pairs={d.diversity.nPairs}"
|
||||||
|
echo &"# churn: flip/sample={d.churn.flipRate:.6f} ({d.churn.flipRatePer100:.4f}/100) " &
|
||||||
|
&"trend={d.churn.flipTrend} first={d.churn.flipFirst:.6f} last={d.churn.flipLast:.6f}"
|
||||||
|
echo &"# clause-change/sample={d.churn.clauseChangeRate:.6f} " &
|
||||||
|
&"({d.churn.clauseChangePer100:.4f}/100) trend={d.churn.clauseTrend} " &
|
||||||
|
&"first={d.churn.clauseFirst:.6f} last={d.churn.clauseLast:.6f}"
|
||||||
|
echo &"# disagreement: overall={d.disagreement.overall:.4f} perClass={d.disagreement.perClass}"
|
||||||
|
|
||||||
|
when isMainModule:
|
||||||
|
let spec = draftTMSpec()
|
||||||
|
let trainA = genDataset(3000, 1)
|
||||||
|
let trainB = shuffleLabels(trainA, 99)
|
||||||
|
let tmpl = newMachine(NBits, NClasses, nClauses = 40, nStates = 64,
|
||||||
|
sValue = 3.0, seed = 1)
|
||||||
|
|
||||||
|
echo &"# automata validation: nBits={NBits} classes={NClasses} clauses=40 states=64 " &
|
||||||
|
&"samples={trainA.len} epochs={Epochs}"
|
||||||
|
|
||||||
|
let dA = automataDiagnostics(tmpl, trainA, spec, epochs = Epochs, seed = 777,
|
||||||
|
settleThreshold = 0.5)
|
||||||
|
let dB = automataDiagnostics(tmpl, trainB, spec, epochs = Epochs, seed = 777,
|
||||||
|
settleThreshold = 0.5)
|
||||||
|
report("CASE A — learnable planted rule", dA)
|
||||||
|
report("CASE B — shuffled (noise) labels", dB)
|
||||||
|
|
||||||
|
# settledness RISING: early prefix vs full training.
|
||||||
|
let early = trainModel(tmpl, trainA[0..<500], epochs = 3, seed = 777)
|
||||||
|
let earlyS = settledness(early, 0.5)
|
||||||
|
let lateS = dA.settledness
|
||||||
|
echo &"\n## CASE A settledness trajectory: early(prefix 500 x3)={earlyS.overallMean:.4f} " &
|
||||||
|
&"-> late(full)={lateS.overallMean:.4f} ({settlednessTrend(earlyS, lateS)})"
|
||||||
|
|
||||||
|
echo "\n## CHECKS"
|
||||||
|
check "A settledness is high at convergence (>= 0.50)", dA.settledness.overallMean >= 0.50
|
||||||
|
check "A churn TRENDS DOWN (falling)",
|
||||||
|
dA.churn.flipTrend == "falling" and dA.churn.flipLast < dA.churn.flipFirst
|
||||||
|
check "A disagreement is low (< 0.15)", dA.disagreement.overall < 0.15
|
||||||
|
check "A settledness RISES from early to late",
|
||||||
|
lateS.overallMean > earlyS.overallMean
|
||||||
|
check "B churn does NOT fall (flat/rising/frozen)",
|
||||||
|
dB.churn.flipTrend in ["flat", "rising", "frozen"]
|
||||||
|
check "A is clearly more settled than B (A - B >= 0.10)",
|
||||||
|
dA.settledness.overallMean - dB.settledness.overallMean >= 0.10
|
||||||
|
check "B disagrees clearly more than A (B - A >= 0.10)",
|
||||||
|
dB.disagreement.overall - dA.disagreement.overall >= 0.10
|
||||||
|
check "the pair is separated (A verdict settling, B not settling)",
|
||||||
|
automataVerdict(dA) == "settling" and automataVerdict(dB) != "settling"
|
||||||
|
|
||||||
|
# ── Case C: constant input ──
|
||||||
|
let trainC = genDataset(2000, 3, constant = true)
|
||||||
|
let dC = automataDiagnostics(tmpl, trainC, spec, epochs = Epochs, seed = 777)
|
||||||
|
echo "\n## CASE C — constant input (bit 20 forced to 1)"
|
||||||
|
var cbit: InputConfidence
|
||||||
|
for ic in dC.inputConfidence:
|
||||||
|
if ic.bit == ConstBit: cbit = ic
|
||||||
|
echo &"# bit{ConstBit} constant={cbit.constant} meanCommit={cbit.meanCommitment:.4f} " &
|
||||||
|
&"settled={cbit.settledFraction:.4f}"
|
||||||
|
let consts = constantInputs(trainC, NBits)
|
||||||
|
echo &"# constantInputs(trainC) = {consts}"
|
||||||
|
check "the constant bit is flagged in the confidence table", cbit.constant
|
||||||
|
check "constantInputs() surfaces the constant bit", ConstBit in consts
|
||||||
|
check "a real planted bit is NOT flagged constant",
|
||||||
|
not dC.inputConfidence[BitA].constant
|
||||||
|
|
||||||
|
# ── Inertia sweep: how the automata metrics move with N ──
|
||||||
|
echo "\n## INERTIA SWEEP — settledness / churn / disagreement vs N (10 epochs)"
|
||||||
|
echo "# case,N,settledness,churnTrend,flipPer100,disagreement,verdict"
|
||||||
|
for n in [16, 32, 64, 128]:
|
||||||
|
let tmplN = newMachine(NBits, NClasses, 40, n, 3.0, 1)
|
||||||
|
let aN = automataDiagnostics(tmplN, trainA, spec, epochs = 10, seed = 777)
|
||||||
|
let bN = automataDiagnostics(tmplN, trainB, spec, epochs = 10, seed = 777)
|
||||||
|
echo &"# A,{n},{aN.settledness.overallMean:.3f},{aN.churn.flipTrend}," &
|
||||||
|
&"{aN.churn.flipRatePer100:.3f},{aN.disagreement.overall:.3f},{automataVerdict(aN)}"
|
||||||
|
echo &"# B,{n},{bN.settledness.overallMean:.3f},{bN.churn.flipTrend}," &
|
||||||
|
&"{bN.churn.flipRatePer100:.3f},{bN.disagreement.overall:.3f},{automataVerdict(bN)}"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
if failures > 0:
|
||||||
|
echo &"{failures} check(s) FAILED"
|
||||||
|
quit(1)
|
||||||
|
echo "All automata-validation checks passed."
|
||||||
@@ -177,5 +177,32 @@ proc main() =
|
|||||||
echo &"# class{cls} votes={c.votes:<7} len={c.length} {c.text}"
|
echo &"# class{cls} votes={c.votes:<7} len={c.length} {c.text}"
|
||||||
echo &"# -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}"
|
echo &"# -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}"
|
||||||
|
|
||||||
|
# ── AUTOMATA-LEVEL metrics on the shipped GF head (Task 5) ──
|
||||||
|
# settledness / diversity / histogram / per-input confidence / disagreement
|
||||||
|
# are read DIRECTLY off the exported teams; churn is measured on a tm_core
|
||||||
|
# retrain (same algorithm) over the captured samples, because the live gun
|
||||||
|
# does not expose a per-sample state trace.
|
||||||
|
var ad = AutomataDiag()
|
||||||
|
ad.machine = m
|
||||||
|
ad.settledness = settledness(m, 0.5)
|
||||||
|
ad.diversity = clauseDiversity(m)
|
||||||
|
ad.histogram = stateHistogram(m, 9)
|
||||||
|
ad.inputConfidence = perInputConfidence(m, spec, bestSamples, 0.5)
|
||||||
|
ad.disagreement = voteDisagreement(m, bestSamples)
|
||||||
|
let churnN = min(bestSamples.len, 10000)
|
||||||
|
var churnSamples = newSeq[DiagSample](churnN)
|
||||||
|
for i in 0..<churnN: churnSamples[i] = bestSamples[i]
|
||||||
|
let tmpl = newMachine(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S,
|
||||||
|
seed = 1)
|
||||||
|
# Mirror the live gun: one TEMPORAL pass (no shuffle) as bullets resolve.
|
||||||
|
ad.churn = churnTrace(tmpl, churnSamples, epochs = 1, seed = 777,
|
||||||
|
window = 200, shuffle = false)
|
||||||
|
ad.summary = healthLine(ad)
|
||||||
|
echo "\n## AUTOMATA-LEVEL metrics (shipped tm_pattern GF head)"
|
||||||
|
echo "# churn measured on a tm_core retrain over ", churnN,
|
||||||
|
" captured samples, ONE temporal pass (live-order proxy)"
|
||||||
|
echo formatAutomataReport(ad)
|
||||||
|
echo "# VERDICT: ", automataVerdict(ad)
|
||||||
|
|
||||||
when isMainModule:
|
when isMainModule:
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
## Pure unit tests for the AUTOMATA-LEVEL metrics added to tm_diag:
|
||||||
|
## settledness, clause diversity, churn, vote disagreement, the state
|
||||||
|
## histogram, per-input confidence and the health/verdict helpers.
|
||||||
|
##
|
||||||
|
## Run: nim c -r --path:common_libs common_libs/tests/test_tm_automata_diag.nim
|
||||||
|
|
||||||
|
import tm_diag/diagnostics
|
||||||
|
import std/strutils
|
||||||
|
|
||||||
|
var checks = 0
|
||||||
|
var failures = 0
|
||||||
|
proc check(name: string, ok: bool) =
|
||||||
|
inc checks
|
||||||
|
if ok: echo "PASS: ", name
|
||||||
|
else: echo "FAIL: ", name; inc failures
|
||||||
|
|
||||||
|
# ── state conventions ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testStateConventions() =
|
||||||
|
check "commitment(0) = 0 (on the flip boundary)",
|
||||||
|
stateCommitment(64, 0) == 0.0
|
||||||
|
check "commitment(+64) = 1 (extreme Include)",
|
||||||
|
stateCommitment(64, 64) == 1.0
|
||||||
|
check "commitment(-64) = 1 (extreme Exclude)",
|
||||||
|
stateCommitment(64, -64) == 1.0
|
||||||
|
check "commitment(+32) = 0.5", abs(stateCommitment(64, 32) - 0.5) < 1e-12
|
||||||
|
check "commitment is symmetric", stateCommitment(64, 17) == stateCommitment(64, -17)
|
||||||
|
check "include predicate matches state>0",
|
||||||
|
stateIncluded(1) and not stateIncluded(0) and not stateIncluded(-1)
|
||||||
|
|
||||||
|
# ── settledness ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc settledMachine(): TmMachine =
|
||||||
|
## nBits=4, 2 classes, 4 clauses/class, 8 literals.
|
||||||
|
## class0/clause0 = [+64, -64, 0, +32, 0,0,0,0] -> mean commitment 0.625,
|
||||||
|
## 3 of 8 automata settled at threshold 0.5.
|
||||||
|
result = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||||
|
result.teams[0][0] = 64
|
||||||
|
result.teams[0][1] = -64
|
||||||
|
result.teams[0][2] = 0
|
||||||
|
result.teams[0][3] = 32
|
||||||
|
|
||||||
|
proc testSettledness() =
|
||||||
|
let m = settledMachine()
|
||||||
|
let s = settledness(m, 0.5)
|
||||||
|
check "settledness covers every clause",
|
||||||
|
s.clauses.len == m.nClasses * m.nClauses
|
||||||
|
check "settledness nAutomata = classes*clauses*literals",
|
||||||
|
s.nAutomata == m.nClasses * m.nClauses * m.nLiterals
|
||||||
|
var c0: ClauseSettledness
|
||||||
|
for c in s.clauses:
|
||||||
|
if c.cls == 0 and c.index == 0: c0 = c
|
||||||
|
check "clause0 mean commitment = 0.3125",
|
||||||
|
abs(c0.meanCommitment - 0.3125) < 1e-12
|
||||||
|
check "clause0 settled fraction = 3/8",
|
||||||
|
abs(c0.settledFraction - 3.0 / 8.0) < 1e-12
|
||||||
|
check "clause0 positive polarity", c0.polarity == 1
|
||||||
|
check "overall settled fraction = 3/64",
|
||||||
|
abs(s.overallSettledFraction - 3.0 / 64.0) < 1e-12
|
||||||
|
check "overall mean commitment = 2.5/64",
|
||||||
|
abs(s.overallMean - 2.5 / 64.0) < 1e-12
|
||||||
|
# trend helper
|
||||||
|
var lo = s
|
||||||
|
lo.overallMean = 0.2
|
||||||
|
var hi = s
|
||||||
|
hi.overallMean = 0.8
|
||||||
|
check "settlednessTrend rising", settlednessTrend(lo, hi) == "rising"
|
||||||
|
check "settlednessTrend falling", settlednessTrend(hi, lo) == "falling"
|
||||||
|
check "settlednessTrend flat", settlednessTrend(s, s) == "flat"
|
||||||
|
|
||||||
|
# ── state histogram ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testHistogram() =
|
||||||
|
let m = settledMachine()
|
||||||
|
let h = stateHistogram(m, 9)
|
||||||
|
var total = 0
|
||||||
|
for c in h.counts: total += c
|
||||||
|
check "histogram counts sum to every automaton",
|
||||||
|
total == m.nClasses * m.nClauses * m.nLiterals
|
||||||
|
check "the -64 state lands in the first bin", h.counts[0] >= 1
|
||||||
|
check "the +64 state lands in the last bin", h.counts[^1] >= 1
|
||||||
|
check "histogram text is non-empty", histogramText(h).len > 0
|
||||||
|
|
||||||
|
# ── per-input confidence ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testInputConfidence() =
|
||||||
|
let m = settledMachine()
|
||||||
|
let spec = draftTMSpec()
|
||||||
|
let samples = @[
|
||||||
|
makeSample(49, (block:
|
||||||
|
var r = newSeq[int](49)
|
||||||
|
for i in 0..<49: r[i] = 1
|
||||||
|
r), 0),
|
||||||
|
makeSample(49, (block:
|
||||||
|
var r = newSeq[int](49)
|
||||||
|
for i in 0..<49: r[i] = 1
|
||||||
|
r), 0),
|
||||||
|
]
|
||||||
|
# all-ones samples make every bit constant, so all should be flagged.
|
||||||
|
let conf = perInputConfidence(m, spec, samples, 0.5)
|
||||||
|
check "confidence table covers every bit", conf.len == m.nBits
|
||||||
|
check "confidence nAutomata per bit = classes*clauses*2",
|
||||||
|
conf[0].nAutomata == m.nClasses * m.nClauses * 2
|
||||||
|
check "a zero-variance bit is flagged constant", conf[0].constant
|
||||||
|
let ranked = rankedInputConfidence(conf)
|
||||||
|
check "ranked confidence is ascending",
|
||||||
|
ranked.len == conf.len and
|
||||||
|
ranked[0].meanCommitment <= ranked[^1].meanCommitment
|
||||||
|
|
||||||
|
# ── diversity / jaccard ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testJaccard() =
|
||||||
|
check "identical sets -> 1", jaccard(@[1, 2, 3], @[1, 2, 3]) == 1.0
|
||||||
|
check "disjoint sets -> 0", jaccard(@[1, 2], @[3, 4]) == 0.0
|
||||||
|
check "two empty sets -> 1", jaccard(@[], @[]) == 1.0
|
||||||
|
check "empty vs non-empty -> 0", jaccard(@[], @[1]) == 0.0
|
||||||
|
check "partial overlap -> |n|/|u|",
|
||||||
|
abs(jaccard(@[1, 2], @[2, 3]) - 1.0 / 3.0) < 1e-12
|
||||||
|
|
||||||
|
proc diversityMachine(): TmMachine =
|
||||||
|
## nBits=4, 2 classes, 6 clauses/class (half=3 positive, 3 negative).
|
||||||
|
## class0 positive clauses: {0,1}, {0,1}, {2,3}; negative clauses empty.
|
||||||
|
result = newMachine(4, 2, 6, 64, 3.0, 1)
|
||||||
|
for lit in [0, 1]:
|
||||||
|
result.teams[0][0 * 8 + lit] = 1
|
||||||
|
result.teams[0][1 * 8 + lit] = 1
|
||||||
|
for lit in [2, 3]:
|
||||||
|
result.teams[0][2 * 8 + lit] = 1
|
||||||
|
|
||||||
|
proc testDiversity() =
|
||||||
|
let m = diversityMachine()
|
||||||
|
let d = clauseDiversity(m)
|
||||||
|
check "positive clauses: {0,1},{0,1},{2,3} mean Jaccard = 1/3",
|
||||||
|
abs(d.jaccardPos - 1.0 / 3.0) < 1e-12
|
||||||
|
check "positive diversity is valid", d.posValid
|
||||||
|
check "empty negative clauses are skipped", not d.negValid
|
||||||
|
check "overall pools the same 3 pairs", d.nPairs == 3
|
||||||
|
check "overall Jaccard = 1/3",
|
||||||
|
abs(d.jaccardOverall - 1.0 / 3.0) < 1e-12
|
||||||
|
|
||||||
|
# ── churn ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testChurnHelpers() =
|
||||||
|
check "binMeans bins [1,1,3,3,5] with window 2 -> [1,3,5]",
|
||||||
|
binMeans(@[1.0, 1.0, 3.0, 3.0, 5.0], 2) == @[1.0, 3.0, 5.0]
|
||||||
|
let fl = firstLastMeans(@[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0])
|
||||||
|
check "firstLastMeans first ~0", fl.first < 1e-9
|
||||||
|
check "firstLastMeans last ~1", abs(fl.last - 1.0) < 1e-9
|
||||||
|
check "churn trend falling",
|
||||||
|
classifyChurnTrend(@[0.9, 0.8, 0.7, 0.6, 0.5, 0.1, 0.05, 0.01]) == "falling"
|
||||||
|
check "churn trend flat",
|
||||||
|
classifyChurnTrend(@[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]) == "flat"
|
||||||
|
check "churn trend rising",
|
||||||
|
classifyChurnTrend(@[0.01, 0.02, 0.03, 0.1, 0.5, 0.6, 0.7, 0.9]) == "rising"
|
||||||
|
check "churn trend frozen",
|
||||||
|
classifyChurnTrend(@[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) == "frozen"
|
||||||
|
|
||||||
|
proc testChurnTrace() =
|
||||||
|
var samples: seq[DiagSample]
|
||||||
|
for i in 0..<120:
|
||||||
|
var raw = newSeq[int](49)
|
||||||
|
for b in 0..<49: raw[b] = (i + b) mod 2
|
||||||
|
raw[0] = 1
|
||||||
|
raw[45] = if i mod 2 == 0: 1 else: 0
|
||||||
|
samples.add makeSample(49, raw, if raw[45] == 1: 2 else: 1, i)
|
||||||
|
let tmpl = newMachine(49, 3, 40, 64, 3.0, 1)
|
||||||
|
let c = churnTrace(tmpl, samples, epochs = 1, seed = 5, window = 30)
|
||||||
|
check "churn nAutomata = classes*clauses*literals",
|
||||||
|
c.nAutomata == 3 * 40 * 98
|
||||||
|
check "churn nClauses = classes*clauses", c.nClauses == 3 * 40
|
||||||
|
check "churn rate is non-negative", c.flipRate >= 0.0
|
||||||
|
check "churn per100 = rate*100",
|
||||||
|
abs(c.flipRatePer100 - c.flipRate * 100.0) < 1e-12
|
||||||
|
check "churn windows are non-empty", c.flipWindows.len > 0
|
||||||
|
check "churn trained machine has the right shape",
|
||||||
|
c.trained.teams.len == 3 and c.trained.teams[0].len == 40 * 98
|
||||||
|
|
||||||
|
# ── vote disagreement ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testVoteDisagreement() =
|
||||||
|
# all-agree machine: only positive clauses fire -> disagreement 0
|
||||||
|
var agree = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||||
|
agree.teams[0][0 * 8 + 0] = 1
|
||||||
|
agree.teams[0][1 * 8 + 1] = 1
|
||||||
|
let allOnes = @[makeSample(4, @[1, 1, 1, 1], 0)]
|
||||||
|
let da = voteDisagreement(agree, allOnes)
|
||||||
|
check "all-positive firing clauses -> disagreement 0",
|
||||||
|
abs(da.overall - 0.0) < 1e-12
|
||||||
|
check "one class observed", da.perClassSamples[0] == 1
|
||||||
|
|
||||||
|
# split machine: one positive and one negative clause fire -> tied vote -> 1
|
||||||
|
var split = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||||
|
split.teams[0][0 * 8 + 0] = 1
|
||||||
|
split.teams[0][2 * 8 + 2] = 1
|
||||||
|
let ds = voteDisagreement(split, allOnes)
|
||||||
|
check "a tied vote counts every firing clause as disagreeing",
|
||||||
|
abs(ds.overall - 1.0) < 1e-12
|
||||||
|
|
||||||
|
# ── health line / verdict / report ───────────────────────────────────────────
|
||||||
|
|
||||||
|
proc testHealthLine() =
|
||||||
|
let spec = draftTMSpec()
|
||||||
|
var samples: seq[DiagSample]
|
||||||
|
for i in 0..<200:
|
||||||
|
var raw = newSeq[int](49)
|
||||||
|
for b in 0..<49: raw[b] = if (i * 7 + b) mod 3 == 0: 1 else: 0
|
||||||
|
raw[0] = if i mod 2 == 0: 1 else: 0
|
||||||
|
raw[45] = if i mod 3 == 0: 1 else: 0
|
||||||
|
let label = if raw[0] == 1 and raw[45] == 1: 2 elif raw[0] == 1: 1 else: 0
|
||||||
|
samples.add makeSample(49, raw, label, i)
|
||||||
|
let tmpl = newMachine(49, 3, 40, 64, 3.0, 1)
|
||||||
|
let d = automataDiagnostics(tmpl, samples, spec, epochs = 3, seed = 7,
|
||||||
|
settleThreshold = 0.5, nHistBins = 9)
|
||||||
|
check "health line is non-empty and mentions every metric",
|
||||||
|
d.summary.len > 0 and "settledness" in d.summary and
|
||||||
|
"diversity" in d.summary and "churn" in d.summary and
|
||||||
|
"disagreement" in d.summary
|
||||||
|
check "automataVerdict is non-empty", automataVerdict(d).len > 0
|
||||||
|
check "formatAutomataReport contains the histogram and table",
|
||||||
|
"state histogram" in formatAutomataReport(d) and
|
||||||
|
"per-input confidence" in formatAutomataReport(d)
|
||||||
|
check "disagreement per-class array is the right size",
|
||||||
|
d.disagreement.perClass.len == 3
|
||||||
|
check "input confidence covers all bits", d.inputConfidence.len == 49
|
||||||
|
|
||||||
|
when isMainModule:
|
||||||
|
testStateConventions()
|
||||||
|
testSettledness()
|
||||||
|
testHistogram()
|
||||||
|
testInputConfidence()
|
||||||
|
testJaccard()
|
||||||
|
testDiversity()
|
||||||
|
testChurnHelpers()
|
||||||
|
testChurnTrace()
|
||||||
|
testVoteDisagreement()
|
||||||
|
testHealthLine()
|
||||||
|
echo ""
|
||||||
|
if failures > 0:
|
||||||
|
echo failures, " / ", checks, " check(s) FAILED"
|
||||||
|
quit(1)
|
||||||
|
echo "All ", checks, " automata-diag unit checks passed."
|
||||||
@@ -16,7 +16,7 @@ Everything is pure and offline — no battles, no Java, no harness.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `feature_spec.nim` | `FeatureSpec`, `describe`, `describeClause`, `draftTMSpec()` (49-bit draft), `tmPatternSpec()` (40-bit shipped encoding) |
|
| `feature_spec.nim` | `FeatureSpec`, `describe`, `describeClause`, `draftTMSpec()` (49-bit draft), `tmPatternSpec()` (40-bit shipped encoding) |
|
||||||
| `tm_core.nim` | compact deterministic Granmo Table 2/3 multiclass TM (mirrors the tm_pattern core), introspectable clause layout |
|
| `tm_core.nim` | compact deterministic Granmo Table 2/3 multiclass TM (mirrors the tm_pattern core), introspectable clause layout |
|
||||||
| `diagnostics.nim` | the six groups (re-exports the two above) |
|
| `diagnostics.nim` | the seven groups (re-exports the two above) |
|
||||||
|
|
||||||
Import everything with:
|
Import everything with:
|
||||||
|
|
||||||
@@ -93,6 +93,143 @@ let rs = ablateScrambleFeature(tmplMachine, train, eval, spec, bit)
|
|||||||
input earn its bits"). Pass `baselineAcc` from `ablateBaseline` to avoid
|
input earn its bits"). Pass `baselineAcc` from `ablateBaseline` to avoid
|
||||||
recomputing it per block.
|
recomputing it per block.
|
||||||
|
|
||||||
|
## Task 5 — the AUTOMATA level (settledness / diversity / churn / disagreement)
|
||||||
|
|
||||||
|
Task 2 reads the clauses. Task 5 reads the **automata inside them**: one
|
||||||
|
automaton per input bit per clause, each holding a state that says how
|
||||||
|
confident it is that its bit belongs in the clause. These are what tell us
|
||||||
|
whether the TM is locking onto the enemy or just fidgeting, and whether the
|
||||||
|
inertia `N` (the number of automata states) should go up or down.
|
||||||
|
|
||||||
|
### State conventions — from the ACTUAL code, not the textbook
|
||||||
|
|
||||||
|
Derived from `tm_core.nim` and `guns/tm_pattern.nim` (`tmEval`, `tmLearnDir`,
|
||||||
|
`tmNewTeam`, `resetMachine`):
|
||||||
|
|
||||||
|
| property | value |
|
||||||
|
|---|---|
|
||||||
|
| state range | `[-nStates, nStates]` (int16); `nStates` = 64 for `tm_pattern`, 32 for `tsetlin.nim` |
|
||||||
|
| initial value | `0` (both cores call it "the Exclude boundary") |
|
||||||
|
| INCLUDE | `state > 0` (`tmEval` / `clauseLits`) |
|
||||||
|
| EXCLUDE | `state <= 0` |
|
||||||
|
| flip boundary | BETWEEN state `0` and state `1` — the middle of the range |
|
||||||
|
| `commitment(st)` | `abs(st) / nStates` in `[0,1]`: 0 on the boundary, 1 at either extreme |
|
||||||
|
|
||||||
|
So a flip is exactly a change in the predicate `state > 0`.
|
||||||
|
|
||||||
|
### The four metrics
|
||||||
|
|
||||||
|
1. **SETTLEDNESS** — per clause and overall, the mean `commitment` and the
|
||||||
|
fraction of automata settled at/above a threshold (default `0.5`). High and
|
||||||
|
**rising** as training proceeds is healthy; low means the clause is wavering
|
||||||
|
noise. `settledness(m, threshold)`, `settlednessTrend(early, late)`.
|
||||||
|
2. **CLAUSE DIVERSITY** — mean pairwise **Jaccard** of the included-literal sets,
|
||||||
|
within the same polarity. Low-to-moderate is healthy; ~1.0 = all clauses are
|
||||||
|
one rule in 50 hats; ~0 = memorising ticks. Empty clauses carry no rule and
|
||||||
|
are skipped by default. `clauseDiversity(m, skipEmpty = true)`, `jaccard(a,b)`.
|
||||||
|
3. **CHURN** — both levels, per training sample: the fraction of **automata**
|
||||||
|
crossing the flip boundary, and the fraction of **clauses** whose
|
||||||
|
included-literal set changed. High early and **falling** is healthy; flat-high
|
||||||
|
= fidgeting; zero from the start = never learned. `churnTrace(tmpl, samples,
|
||||||
|
epochs, seed, window, shuffle)`. `shuffle = false` replays the given
|
||||||
|
(temporal) order, mirroring a live gun.
|
||||||
|
4. **VOTE DISAGREEMENT** — per class, over the samples where it casts a vote:
|
||||||
|
the fraction of firing clauses whose polarity disagrees with the sign of the
|
||||||
|
class's total vote. Low is healthy. `voteDisagreement(m, samples)`.
|
||||||
|
|
||||||
|
### The three readouts that make it readable
|
||||||
|
|
||||||
|
- **state histogram** — the distribution of automata states across the range:
|
||||||
|
`stateHistogram(m, nBins)`, `histogramText(h)`.
|
||||||
|
- **per-input confidence table** — for each bit, the mean commitment of its
|
||||||
|
automata (and a `constant` flag for zero-variance inputs):
|
||||||
|
`perInputConfidence(m, spec, samples, threshold)`,
|
||||||
|
`rankedInputConfidence(conf)`.
|
||||||
|
- **one-line health summary** — `healthLine(ad)` gives
|
||||||
|
`settledness / diversity / churn trend / disagreement`, each with its own
|
||||||
|
verdict word (`settling`, `fidgeting`, `frozen`, `coherent`, ...), and
|
||||||
|
`automataVerdict(ad)` reduces the trajectory to one word. `formatAutomataReport(ad)`
|
||||||
|
prints everything.
|
||||||
|
|
||||||
|
### API
|
||||||
|
|
||||||
|
```nim
|
||||||
|
import tm_diag/diagnostics
|
||||||
|
|
||||||
|
let ad = automataDiagnostics(tmpl, samples, spec,
|
||||||
|
epochs = 15, seed = 777,
|
||||||
|
settleThreshold = 0.5, nHistBins = 9,
|
||||||
|
window = 100, measureChurn = true)
|
||||||
|
# ad.machine, ad.settledness, ad.diversity, ad.churn, ad.disagreement,
|
||||||
|
# ad.histogram, ad.inputConfidence, ad.summary
|
||||||
|
echo ad.summary # one line
|
||||||
|
echo automataVerdict(ad) # "settling" | "fidgeting" | "collapsed" | ...
|
||||||
|
echo formatAutomataReport(ad) # the full readout
|
||||||
|
```
|
||||||
|
|
||||||
|
For a machine you already have (e.g. the shipped gun's `exportTeams()`), call
|
||||||
|
`settledness` / `clauseDiversity` / `stateHistogram` / `perInputConfidence` /
|
||||||
|
`voteDisagreement` directly and `churnTrace` on a fresh copy.
|
||||||
|
|
||||||
|
### Validation — Case A (learnable) vs Case B (noise)
|
||||||
|
|
||||||
|
`common_libs/tests/diag_automata_validation.nim` uses the planted rule from
|
||||||
|
`diag_synthetic.nim` and the SAME inputs with shuffled labels. Measured
|
||||||
|
(`nBits=49`, 3 classes, 40 clauses, `nStates=64`, 3000 samples, 15 epochs):
|
||||||
|
|
||||||
|
| metric | Case A (learnable) | Case B (noise) |
|
||||||
|
|---|---|---|
|
||||||
|
| settledness mean | **0.970** | 0.719 |
|
||||||
|
| settled fraction | 0.999 | 0.799 |
|
||||||
|
| churn trend | **falling** (`2.5e-4` -> `2e-6`) | **flat** (`1.0e-3` -> `7.2e-4`) |
|
||||||
|
| clause-change trend | falling (`0.011` -> `2.3e-4`) | flat (`0.069` -> `0.057`) |
|
||||||
|
| diversity (overall Jaccard) | 0.176 | 0.014 |
|
||||||
|
| disagreement | **0.003** | **0.298** |
|
||||||
|
| verdict | settling | mixed (not settling) |
|
||||||
|
|
||||||
|
Case A settledness RISES `0.475` (early prefix) -> `0.970` (full). The pair
|
||||||
|
**separates learning from fidgeting**: the decisive signals are the churn trend
|
||||||
|
(falling vs flat) and disagreement (0.003 vs 0.298). Settledness alone is NOT
|
||||||
|
enough — on noise the automata still commit (0.719), just to the wrong thing.
|
||||||
|
Case C (a forced-constant bit) is flagged: `perInputConfidence(...).constant`
|
||||||
|
and `constantInputs` both surface it. Note a constant bit can show HIGH
|
||||||
|
commitment (one literal is always 1), so the `constant` flag is what
|
||||||
|
disambiguates.
|
||||||
|
|
||||||
|
### Inertia sweep (`N` = nStates, 10 epochs)
|
||||||
|
|
||||||
|
The validation also sweeps `N` to see whether inertia moves the metrics:
|
||||||
|
|
||||||
|
| N | Case A settled | A churn | A dis. | Case B settled | B churn | B dis. |
|
||||||
|
|---|---|---|---|---|---|---|
|
||||||
|
| 16 | 0.904 | falling | 0.003 | 0.720 | flat | 0.363 |
|
||||||
|
| 32 | 0.943 | falling | 0.000 | 0.700 | flat | 0.388 |
|
||||||
|
| 64 | 0.970 | falling | 0.003 | 0.715 | flat | 0.368 |
|
||||||
|
| 128 | 0.971 | falling | 0.000 | 0.660 | flat | 0.304 |
|
||||||
|
|
||||||
|
A and B are separated at EVERY N (churn falling vs flat, disagreement low vs
|
||||||
|
high). Raising N only raises Case A's commitment (0.90 -> 0.97); it does NOT
|
||||||
|
reduce noise-fitting in Case B. So **inertia is not the discriminator** the
|
||||||
|
metrics identify — the churn trend and disagreement are.
|
||||||
|
|
||||||
|
### Real reading — the shipped `tm_pattern` GF head
|
||||||
|
|
||||||
|
`diag_tm_pattern_offline.nim` over the committed DrussGT fixtures (automata read
|
||||||
|
directly off the exported teams from `tr_drussgt_vs_modularbot`; churn measured
|
||||||
|
on a `tm_core` temporal one-pass proxy, live-order):
|
||||||
|
|
||||||
|
- **settledness 0.484** (`settling`), settled fraction 0.453
|
||||||
|
- **diversity 0.267** (`moderate`) — positive 0.170, negative 0.322
|
||||||
|
- **churn 0.094/100 per sample, FALLING** (0.00208 -> 0.00048); clause-change 5.31/100, falling
|
||||||
|
- **disagreement 0.145** (`coherent`)
|
||||||
|
- **verdict: settling** — not fidgeting, not collapsed
|
||||||
|
- constant inputs flagged: bits 38/39 (the known never-written ones) plus 19/36/37 in this fixture
|
||||||
|
- context: pooled warm accuracy 35.72% vs the 34.24% majority = **+1.48pp**
|
||||||
|
|
||||||
|
The gun settles onto the within-battle labels but its settled rules barely beat
|
||||||
|
the majority class, which points at the TARGET / representation rather than the
|
||||||
|
inertia `N`. See the Task 5 report for the inertia discussion.
|
||||||
|
|
||||||
## The default-off real-gun hook
|
## The default-off real-gun hook
|
||||||
|
|
||||||
`guns/tm_pattern.nim` gained only additive, default-off instrumentation:
|
`guns/tm_pattern.nim` gained only additive, default-off instrumentation:
|
||||||
@@ -116,7 +253,9 @@ let m = machineFromTeams(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S,
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
nim c -r -d:release --path:common_libs common_libs/tests/test_tm_diag.nim # 48 pure unit checks
|
nim c -r -d:release --path:common_libs common_libs/tests/test_tm_diag.nim # 48 pure unit checks
|
||||||
nim c -r -d:release --path:common_libs common_libs/tests/diag_synthetic.nim # Task 3 proof
|
nim c -r --path:common_libs common_libs/tests/test_tm_automata_diag.nim # 55 automata-metric unit checks
|
||||||
|
nim c -r -d:release --path:common_libs common_libs/tests/diag_synthetic.nim # Task 3 proof (17 checks)
|
||||||
|
nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim # Task 5 A/B/C proof
|
||||||
nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim # Task 4 real reading
|
nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim # Task 4 real reading
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -1,20 +1,24 @@
|
|||||||
## tm_diag/diagnostics.nim — THE DIAGNOSTICS KIT (Task 2, six groups).
|
## tm_diag/diagnostics.nim — THE DIAGNOSTICS KIT (Tasks 2 + 5).
|
||||||
##
|
##
|
||||||
## Everything runs OFFLINE: a trained `TmMachine` plus a labelled sample set.
|
## Everything runs OFFLINE: a trained `TmMachine` plus a labelled sample set.
|
||||||
## No battles, no harness, no Java. The six groups are:
|
## No battles, no harness, no Java. The seven groups are:
|
||||||
## 1. pre-flight DATA checks (dataChecks, shuffledLabelControl)
|
## 1. pre-flight DATA checks (dataChecks, shuffledLabelControl)
|
||||||
## 2. clause introspection (clauseInfo, clauseSummary, topClauses)
|
## 2. clause introspection (clauseInfo, clauseSummary, topClauses)
|
||||||
## 3. per-feature contribution (featureContributions, deadInputs, ...)
|
## 3. per-feature contribution (featureContributions, deadInputs, ...)
|
||||||
## 4. accuracy diagnostics (accuracyDiagnostics)
|
## 4. accuracy diagnostics (accuracyDiagnostics)
|
||||||
## 5. learning curve (learningCurve)
|
## 5. learning curve (learningCurve)
|
||||||
## 6. ablation hooks (ablateDropBlock, ablateScrambleFeature, ...)
|
## 6. ablation hooks (ablateDropBlock, ablateScrambleFeature, ...)
|
||||||
|
## 7. AUTOMATA level (Task 5) (settledness, clauseDiversity, churnTrace,
|
||||||
|
## voteDisagreement, stateHistogram,
|
||||||
|
## perInputConfidence, healthLine,
|
||||||
|
## automataDiagnostics)
|
||||||
##
|
##
|
||||||
## A `DiagSample` carries the LITERAL vector (positive literals [0..nBits),
|
## A `DiagSample` carries the LITERAL vector (positive literals [0..nBits),
|
||||||
## negations [nBits..2*nBits)) exactly as the shipped TM stores it, plus the
|
## negations [nBits..2*nBits)) exactly as the shipped TM stores it, plus the
|
||||||
## true label. This keeps the kit usable both on the reference core and on the
|
## true label. This keeps the kit usable both on the reference core and on the
|
||||||
## literal vectors captured from the real gun.
|
## literal vectors captured from the real gun.
|
||||||
|
|
||||||
import std/[math, strformat, algorithm]
|
import std/[math, strformat, algorithm, strutils]
|
||||||
import feature_spec
|
import feature_spec
|
||||||
import tm_core
|
import tm_core
|
||||||
|
|
||||||
@@ -587,3 +591,551 @@ proc ablateBaseline*(tmpl: TmMachine, train, eval: openArray[DiagSample],
|
|||||||
baselineAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval),
|
baselineAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval),
|
||||||
ablatedAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval),
|
ablatedAcc: evalAcc(trainModel(tmpl, train, epochs, seed), eval),
|
||||||
delta: 0.0)
|
delta: 0.0)
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# GROUP 7 — automata-level metrics
|
||||||
|
# settledness / clause diversity / churn / vote disagreement
|
||||||
|
# plus the state histogram, the per-input confidence table and a health line.
|
||||||
|
#
|
||||||
|
# STATE CONVENTIONS — derived from the ACTUAL code (tm_core.nim and
|
||||||
|
# guns/tm_pattern.nim), not from the textbook formulation:
|
||||||
|
# * every clause owns one automaton per literal (2 * nBits of them);
|
||||||
|
# * states are int16 in [-nStates, nStates];
|
||||||
|
# * INCLUDE iff state > 0, EXCLUDE iff state <= 0 (tmEval / clauseLits);
|
||||||
|
# * both cores initialise every state to 0 ("the Exclude boundary"), so the
|
||||||
|
# flip boundary sits BETWEEN state 0 (Exclude) and state 1 (Include) — the
|
||||||
|
# middle of the range.
|
||||||
|
# * commitment(st) = |st| / nStates in [0, 1]: 0 exactly on the boundary,
|
||||||
|
# 1 at either extreme. It is symmetric for the two sides.
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
proc stateCommitment*(nStates, st: int): float {.inline.} =
|
||||||
|
## Normalised distance of an automaton state from its flip boundary, in
|
||||||
|
## [0, 1]. 0 == on the boundary (state 0), 1 == at either extreme (+/-nStates).
|
||||||
|
min(abs(st).float / max(1, nStates).float, 1.0)
|
||||||
|
|
||||||
|
proc stateIncluded*(st: int): bool {.inline.} =
|
||||||
|
## The kit's include predicate, matching `tmEval` / `clauseLits` exactly.
|
||||||
|
st > 0
|
||||||
|
|
||||||
|
# ── 1. SETTLEDNESS ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
ClauseSettledness* = object
|
||||||
|
cls*: int
|
||||||
|
index*: int
|
||||||
|
polarity*: int ## +1 positive clauses, -1 negative clauses
|
||||||
|
nAutomata*: int
|
||||||
|
meanCommitment*: float
|
||||||
|
settledFraction*: float
|
||||||
|
|
||||||
|
SettlednessResult* = object
|
||||||
|
threshold*: float
|
||||||
|
nAutomata*: int
|
||||||
|
overallMean*: float
|
||||||
|
overallSettledFraction*: float
|
||||||
|
clauses*: seq[ClauseSettledness]
|
||||||
|
|
||||||
|
proc settledness*(m: TmMachine, threshold = 0.5): SettlednessResult =
|
||||||
|
## Per clause and overall: the mean automaton `commitment` and the fraction of
|
||||||
|
## automata whose commitment is >= `threshold`. A clause full of automata
|
||||||
|
## parked near the boundary is wavering noise; one with automata pushed to an
|
||||||
|
## extreme has committed to a rule.
|
||||||
|
result.threshold = threshold
|
||||||
|
var sum = 0.0
|
||||||
|
var settled = 0
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
for cl in 0..<m.nClauses:
|
||||||
|
let base = cl * m.nLiterals
|
||||||
|
var csum = 0.0
|
||||||
|
var csettled = 0
|
||||||
|
for lit in 0..<m.nLiterals:
|
||||||
|
let cm = stateCommitment(m.nStates, int(m.teams[c][base + lit]))
|
||||||
|
csum += cm
|
||||||
|
if cm >= threshold: inc csettled
|
||||||
|
result.clauses.add ClauseSettledness(
|
||||||
|
cls: c, index: cl,
|
||||||
|
polarity: (if cl < m.half: 1 else: -1),
|
||||||
|
nAutomata: m.nLiterals,
|
||||||
|
meanCommitment: csum / m.nLiterals.float,
|
||||||
|
settledFraction: csettled.float / m.nLiterals.float)
|
||||||
|
sum += csum
|
||||||
|
settled += csettled
|
||||||
|
inc result.nAutomata, m.nLiterals
|
||||||
|
result.overallMean =
|
||||||
|
if result.nAutomata > 0: sum / result.nAutomata.float else: 0.0
|
||||||
|
result.overallSettledFraction =
|
||||||
|
if result.nAutomata > 0: settled.float / result.nAutomata.float else: 0.0
|
||||||
|
|
||||||
|
proc settlednessTrend*(early, late: SettlednessResult): string =
|
||||||
|
## Classify whether settledness ROSE between two snapshots (the healthy sign).
|
||||||
|
let d = late.overallMean - early.overallMean
|
||||||
|
if d > 0.05: "rising"
|
||||||
|
elif d < -0.05: "falling"
|
||||||
|
else: "flat"
|
||||||
|
|
||||||
|
# ── state histogram (the raw picture behind settledness) ─────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
StateHistogram* = object
|
||||||
|
nStates*: int
|
||||||
|
nBins*: int
|
||||||
|
binLo*: seq[int] ## inclusive low state observed in the bin
|
||||||
|
binHi*: seq[int] ## inclusive high state observed in the bin
|
||||||
|
counts*: seq[int]
|
||||||
|
total*: int
|
||||||
|
|
||||||
|
proc stateHistogram*(m: TmMachine, nBins = 9): StateHistogram =
|
||||||
|
## Distribution of every automaton state across [-nStates, nStates].
|
||||||
|
let nb = max(1, nBins)
|
||||||
|
let span = 2 * m.nStates + 1
|
||||||
|
result.nStates = m.nStates
|
||||||
|
result.nBins = nb
|
||||||
|
result.counts = newSeq[int](nb)
|
||||||
|
result.binLo = newSeq[int](nb)
|
||||||
|
result.binHi = newSeq[int](nb)
|
||||||
|
for b in 0..<nb:
|
||||||
|
result.binLo[b] = high(int)
|
||||||
|
result.binHi[b] = low(int)
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
for i in 0..<m.teams[c].len:
|
||||||
|
let st = int(m.teams[c][i])
|
||||||
|
var bin = ((st + m.nStates) * nb) div span
|
||||||
|
if bin < 0: bin = 0
|
||||||
|
if bin >= nb: bin = nb - 1
|
||||||
|
inc result.counts[bin]
|
||||||
|
if st < result.binLo[bin]: result.binLo[bin] = st
|
||||||
|
if st > result.binHi[bin]: result.binHi[bin] = st
|
||||||
|
inc result.total
|
||||||
|
for b in 0..<nb:
|
||||||
|
if result.counts[b] == 0:
|
||||||
|
result.binLo[b] = 0
|
||||||
|
result.binHi[b] = 0
|
||||||
|
|
||||||
|
proc histogramText*(h: StateHistogram): string =
|
||||||
|
## One line per bin, e.g. `[-64..-57] 1234 #####`.
|
||||||
|
for b in 0..<h.nBins:
|
||||||
|
let bar = "#".repeat(min(40, h.counts[b] div max(1, h.total div 200 + 1)))
|
||||||
|
result.add &" [{h.binLo[b]:>4}..{h.binHi[b]:>4}] {h.counts[b]:>8} {bar}\n"
|
||||||
|
|
||||||
|
# ── per-input automata confidence table ──────────────────────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
InputConfidence* = object
|
||||||
|
bit*: int
|
||||||
|
name*: string
|
||||||
|
nAutomata*: int
|
||||||
|
meanCommitment*: float
|
||||||
|
settledFraction*: float
|
||||||
|
constant*: bool ## zero variance across the sample set (information-free)
|
||||||
|
|
||||||
|
proc perInputConfidence*(m: TmMachine, spec: FeatureSpec,
|
||||||
|
samples: openArray[DiagSample],
|
||||||
|
threshold = 0.5): seq[InputConfidence] =
|
||||||
|
## For every raw bit: the mean commitment of the automata of its positive
|
||||||
|
## literal and its negation, across every clause and class. Inputs the TM is
|
||||||
|
## confident about sit high; a bit whose automata are all parked at the
|
||||||
|
## boundary (e.g. a constant input) sits near 0. `constant` flags zero-variance
|
||||||
|
## bits so a high commitment on a never-written input is not mistaken for
|
||||||
|
## learning (the shipped gun's bits 38/39 are the canonical example).
|
||||||
|
result = newSeq[InputConfidence](m.nBits)
|
||||||
|
var consts: seq[int]
|
||||||
|
if samples.len > 0:
|
||||||
|
for b in 0..<m.nBits:
|
||||||
|
let v0 = samples[0].lits[b]
|
||||||
|
var cst = true
|
||||||
|
for i in 1..<samples.len:
|
||||||
|
if samples[i].lits[b] != v0:
|
||||||
|
cst = false
|
||||||
|
break
|
||||||
|
if cst: consts.add b
|
||||||
|
for b in 0..<m.nBits:
|
||||||
|
result[b].bit = b
|
||||||
|
result[b].name = spec.describe(b)
|
||||||
|
result[b].constant = b in consts
|
||||||
|
var sum = 0.0
|
||||||
|
var settled = 0
|
||||||
|
var n = 0
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
for cl in 0..<m.nClauses:
|
||||||
|
let base = cl * m.nLiterals
|
||||||
|
let cmPos = stateCommitment(m.nStates, int(m.teams[c][base + b]))
|
||||||
|
let cmNeg = stateCommitment(m.nStates,
|
||||||
|
int(m.teams[c][base + b + m.nBits]))
|
||||||
|
sum += cmPos + cmNeg
|
||||||
|
if cmPos >= threshold: inc settled
|
||||||
|
if cmNeg >= threshold: inc settled
|
||||||
|
inc n, 2
|
||||||
|
result[b].nAutomata = n
|
||||||
|
result[b].meanCommitment = if n > 0: sum / n.float else: 0.0
|
||||||
|
result[b].settledFraction = if n > 0: settled.float / n.float else: 0.0
|
||||||
|
|
||||||
|
proc rankedInputConfidence*(conf: seq[InputConfidence]): seq[InputConfidence] =
|
||||||
|
## Least confident first — the inputs whose automata are still undecided.
|
||||||
|
result = conf
|
||||||
|
result.sort(proc(a, b: InputConfidence): int =
|
||||||
|
result = cmp(a.meanCommitment, b.meanCommitment)
|
||||||
|
if result == 0: result = cmp(a.bit, b.bit))
|
||||||
|
|
||||||
|
# ── 2. CLAUSE DIVERSITY ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
DiversityResult* = object
|
||||||
|
jaccardPos*: float ## mean pairwise Jaccard among positive clauses
|
||||||
|
jaccardNeg*: float ## ... among negative clauses
|
||||||
|
jaccardOverall*: float ## pooled over both polarities
|
||||||
|
nPosClauses*: int
|
||||||
|
nNegClauses*: int
|
||||||
|
nPairs*: int
|
||||||
|
posValid*: bool
|
||||||
|
negValid*: bool
|
||||||
|
valid*: bool
|
||||||
|
skipEmpty*: bool
|
||||||
|
|
||||||
|
proc jaccard*(a, b: openArray[int]): float =
|
||||||
|
## |a n b| / |a u b| for two sorted, unique literal sets. Two empty sets are
|
||||||
|
## identical (1.0); an empty against a non-empty set share nothing (0.0).
|
||||||
|
if a.len == 0 and b.len == 0: return 1.0
|
||||||
|
if a.len == 0 or b.len == 0: return 0.0
|
||||||
|
var i = 0
|
||||||
|
var j = 0
|
||||||
|
var inter = 0
|
||||||
|
while i < a.len and j < b.len:
|
||||||
|
if a[i] == b[j]: inc inter; inc i; inc j
|
||||||
|
elif a[i] < b[j]: inc i
|
||||||
|
else: inc j
|
||||||
|
let uni = a.len + b.len - inter
|
||||||
|
if uni == 0: 1.0 else: inter.float / uni.float
|
||||||
|
|
||||||
|
proc polarityDiversity(m: TmMachine, polarity: int, skipEmpty: bool):
|
||||||
|
tuple[sum: float, npairs, nclauses: int] =
|
||||||
|
var sets: seq[seq[int]]
|
||||||
|
let cl0 = if polarity > 0: 0 else: m.half
|
||||||
|
let cl1 = if polarity > 0: m.half else: m.nClauses
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
for cl in cl0..<cl1:
|
||||||
|
let ls = m.clauseLits(c, cl)
|
||||||
|
if skipEmpty and ls.len == 0: continue
|
||||||
|
sets.add ls
|
||||||
|
result.nclauses = sets.len
|
||||||
|
for i in 0..<sets.len:
|
||||||
|
for j in (i + 1)..<sets.len:
|
||||||
|
result.sum += jaccard(sets[i], sets[j])
|
||||||
|
inc result.npairs
|
||||||
|
|
||||||
|
proc clauseDiversity*(m: TmMachine, skipEmpty = true): DiversityResult =
|
||||||
|
## Mean pairwise Jaccard overlap of the INCLUDED-literal sets, within the same
|
||||||
|
## polarity (positive vs positive, negative vs negative). Low-to-moderate is
|
||||||
|
## healthy; near 1.0 means the clauses are the same rule wearing 50 hats;
|
||||||
|
## near 0 with few samples usually means memorising individual ticks. Empty
|
||||||
|
## clauses carry no rule and are skipped by default (`skipEmpty`).
|
||||||
|
result.skipEmpty = skipEmpty
|
||||||
|
let p = polarityDiversity(m, 1, skipEmpty)
|
||||||
|
let n = polarityDiversity(m, -1, skipEmpty)
|
||||||
|
result.nPosClauses = p.nclauses
|
||||||
|
result.nNegClauses = n.nclauses
|
||||||
|
result.posValid = p.npairs > 0
|
||||||
|
result.negValid = n.npairs > 0
|
||||||
|
result.jaccardPos = if p.npairs > 0: p.sum / p.npairs.float else: 0.0
|
||||||
|
result.jaccardNeg = if n.npairs > 0: n.sum / n.npairs.float else: 0.0
|
||||||
|
let pairs = p.npairs + n.npairs
|
||||||
|
result.nPairs = pairs
|
||||||
|
result.valid = pairs > 0
|
||||||
|
result.jaccardOverall =
|
||||||
|
if pairs > 0: (p.sum + n.sum) / pairs.float else: 0.0
|
||||||
|
|
||||||
|
# ── 3. CHURN ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
ChurnResult* = object
|
||||||
|
samples*: int
|
||||||
|
epochs*: int
|
||||||
|
nAutomata*: int
|
||||||
|
nClauses*: int
|
||||||
|
totalFlips*: int
|
||||||
|
totalClauseChanges*: int
|
||||||
|
flipRate*: float ## automaton flips per training sample (fraction)
|
||||||
|
flipRatePer100*: float ## same, x100 (readable)
|
||||||
|
clauseChangeRate*: float ## clause-set changes per training sample
|
||||||
|
clauseChangePer100*: float
|
||||||
|
flipFirst*: float
|
||||||
|
flipLast*: float
|
||||||
|
clauseFirst*: float
|
||||||
|
clauseLast*: float
|
||||||
|
flipTrend*: string
|
||||||
|
clauseTrend*: string
|
||||||
|
windowSize*: int
|
||||||
|
flipWindows*: seq[float] ## mean per-sample flip rate per window
|
||||||
|
clauseWindows*: seq[float]
|
||||||
|
trained*: TmMachine
|
||||||
|
|
||||||
|
proc binMeans*(s: seq[float], window: int): seq[float] =
|
||||||
|
if s.len == 0: return
|
||||||
|
let w = max(1, window)
|
||||||
|
var i = 0
|
||||||
|
while i < s.len:
|
||||||
|
let j = min(s.len, i + w)
|
||||||
|
var sum = 0.0
|
||||||
|
for k in i..<j: sum += s[k]
|
||||||
|
result.add sum / (j - i).float
|
||||||
|
i = j
|
||||||
|
|
||||||
|
proc firstLastMeans*(s: seq[float]): tuple[first, last: float] =
|
||||||
|
if s.len == 0: return (0.0, 0.0)
|
||||||
|
let q = max(1, s.len div 5)
|
||||||
|
var a = 0.0
|
||||||
|
var b = 0.0
|
||||||
|
for i in 0..<q: a += s[i]
|
||||||
|
for i in (s.len - q)..<s.len: b += s[i]
|
||||||
|
(a / q.float, b / q.float)
|
||||||
|
|
||||||
|
proc classifyChurnTrend*(s: seq[float]): string =
|
||||||
|
## "falling" is the healthy sign; "flat"/"rising" at a high rate is
|
||||||
|
## fidgeting; "frozen" means it never moved at all.
|
||||||
|
if s.len < 4: return "insufficient"
|
||||||
|
let (first, last) = firstLastMeans(s)
|
||||||
|
if first <= 1e-12 and last <= 1e-12: return "frozen"
|
||||||
|
if first <= 1e-12: return "rising"
|
||||||
|
let ratio = last / first
|
||||||
|
if ratio < 0.6: "falling"
|
||||||
|
elif ratio > 1.6: "rising"
|
||||||
|
else: "flat"
|
||||||
|
|
||||||
|
proc churnTrace*(tmpl: TmMachine, samples: openArray[DiagSample],
|
||||||
|
epochs = 1, seed = 777'u64, window = 100,
|
||||||
|
shuffle = true): ChurnResult =
|
||||||
|
## Train a fresh machine while measuring, per training sample, both churn
|
||||||
|
## levels: the fraction of AUTOMATA crossing the flip boundary, and the
|
||||||
|
## fraction of CLAUSES whose included-literal set changed. `shuffle = false`
|
||||||
|
## replays the samples in their given (temporal) order once per epoch, which
|
||||||
|
## mirrors a live gun training as bullets resolve.
|
||||||
|
result.epochs = max(1, epochs)
|
||||||
|
result.samples = samples.len
|
||||||
|
result.windowSize = max(1, window)
|
||||||
|
var m = tmpl
|
||||||
|
m.resetMachine(seed)
|
||||||
|
result.nAutomata = m.nClasses * m.nClauses * m.nLiterals
|
||||||
|
result.nClauses = m.nClasses * m.nClauses
|
||||||
|
if result.nAutomata == 0:
|
||||||
|
result.trained = m
|
||||||
|
return
|
||||||
|
var prevInc = newSeq[uint8](result.nAutomata)
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
let team = m.teams[c]
|
||||||
|
let gbase = c * m.nClauses * m.nLiterals
|
||||||
|
for i in 0..<team.len:
|
||||||
|
prevInc[gbase + i] = uint8(team[i] > 0)
|
||||||
|
var flipSeries = newSeq[float]()
|
||||||
|
var clauseSeries = newSeq[float]()
|
||||||
|
for _ in 0..<result.epochs:
|
||||||
|
let idx =
|
||||||
|
if shuffle: shuffledIndices(samples.len, m.rng)
|
||||||
|
else: (block:
|
||||||
|
var id = newSeq[int](samples.len)
|
||||||
|
for i in 0..<samples.len: id[i] = i
|
||||||
|
id)
|
||||||
|
for k in idx:
|
||||||
|
m.trainSample(samples[k].lits, samples[k].label)
|
||||||
|
var flips = 0
|
||||||
|
var clauseChanges = 0
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
let team = m.teams[c]
|
||||||
|
let gbase = c * m.nClauses * m.nLiterals
|
||||||
|
for cl in 0..<m.nClauses:
|
||||||
|
var changed = false
|
||||||
|
let base = cl * m.nLiterals
|
||||||
|
let gb = gbase + base
|
||||||
|
for lit in 0..<m.nLiterals:
|
||||||
|
let nowInc = uint8(team[base + lit] > 0)
|
||||||
|
if nowInc != prevInc[gb + lit]:
|
||||||
|
inc flips
|
||||||
|
changed = true
|
||||||
|
prevInc[gb + lit] = nowInc
|
||||||
|
if changed: inc clauseChanges
|
||||||
|
result.totalFlips += flips
|
||||||
|
result.totalClauseChanges += clauseChanges
|
||||||
|
flipSeries.add flips.float / result.nAutomata.float
|
||||||
|
clauseSeries.add clauseChanges.float / result.nClauses.float
|
||||||
|
result.trained = m
|
||||||
|
let nObs = flipSeries.len
|
||||||
|
if nObs > 0:
|
||||||
|
result.flipRate = result.totalFlips.float /
|
||||||
|
(nObs.float * result.nAutomata.float)
|
||||||
|
result.clauseChangeRate = result.totalClauseChanges.float /
|
||||||
|
(nObs.float * result.nClauses.float)
|
||||||
|
result.flipRatePer100 = result.flipRate * 100.0
|
||||||
|
result.clauseChangePer100 = result.clauseChangeRate * 100.0
|
||||||
|
let ff = firstLastMeans(flipSeries)
|
||||||
|
let cf = firstLastMeans(clauseSeries)
|
||||||
|
result.flipFirst = ff.first
|
||||||
|
result.flipLast = ff.last
|
||||||
|
result.clauseFirst = cf.first
|
||||||
|
result.clauseLast = cf.last
|
||||||
|
result.flipTrend = classifyChurnTrend(flipSeries)
|
||||||
|
result.clauseTrend = classifyChurnTrend(clauseSeries)
|
||||||
|
result.flipWindows = binMeans(flipSeries, result.windowSize)
|
||||||
|
result.clauseWindows = binMeans(clauseSeries, result.windowSize)
|
||||||
|
|
||||||
|
# ── 4. VOTE DISAGREEMENT ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
VoteDisagreementResult* = object
|
||||||
|
overall*: float
|
||||||
|
perClass*: seq[float]
|
||||||
|
perClassSamples*: seq[int]
|
||||||
|
nPairs*: int ## (class, sample) observations that had a vote
|
||||||
|
|
||||||
|
proc voteDisagreement*(m: TmMachine, samples: openArray[DiagSample]):
|
||||||
|
VoteDisagreementResult =
|
||||||
|
## For each class, over the samples where it casts at least one clause vote:
|
||||||
|
## the fraction of FIRING clauses whose own polarity disagrees with the sign
|
||||||
|
## of the class's total vote. A class whose clauses always pull together has
|
||||||
|
## disagreement ~0; a fuzzy class boundary or too few clauses makes it high.
|
||||||
|
## A tied class vote (mixed signs) counts every firing clause as disagreeing.
|
||||||
|
result.perClass = newSeq[float](m.nClasses)
|
||||||
|
result.perClassSamples = newSeq[int](m.nClasses)
|
||||||
|
var classSum = newSeq[float](m.nClasses)
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
for s in samples:
|
||||||
|
var vote = 0.0
|
||||||
|
var firing: seq[int]
|
||||||
|
for cl in 0..<m.nClauses:
|
||||||
|
let o = m.tmEval(m.teams[c], s.lits, cl, learning = false)
|
||||||
|
if o == 1'u8:
|
||||||
|
vote += m.tmPolarity(cl)
|
||||||
|
firing.add cl
|
||||||
|
if firing.len == 0: continue
|
||||||
|
let majSign = if vote > 0.0: 1 elif vote < 0.0: -1 else: 0
|
||||||
|
var disagree = 0
|
||||||
|
for cl in firing:
|
||||||
|
let pol = if m.tmPolarity(cl) > 0.0: 1 else: -1
|
||||||
|
if pol != majSign: inc disagree
|
||||||
|
classSum[c] += disagree.float / firing.len.float
|
||||||
|
inc result.perClassSamples[c]
|
||||||
|
inc result.nPairs
|
||||||
|
var totalSum = 0.0
|
||||||
|
var totalN = 0
|
||||||
|
for c in 0..<m.nClasses:
|
||||||
|
if result.perClassSamples[c] > 0:
|
||||||
|
result.perClass[c] = classSum[c] / result.perClassSamples[c].float
|
||||||
|
totalSum += result.perClass[c]
|
||||||
|
inc totalN
|
||||||
|
result.overall = if totalN > 0: totalSum / totalN.float else: 0.0
|
||||||
|
|
||||||
|
# ── the convenience driver + the one-line health summary ─────────────────────
|
||||||
|
|
||||||
|
type
|
||||||
|
AutomataDiag* = object
|
||||||
|
machine*: TmMachine
|
||||||
|
settledness*: SettlednessResult
|
||||||
|
diversity*: DiversityResult
|
||||||
|
churn*: ChurnResult
|
||||||
|
disagreement*: VoteDisagreementResult
|
||||||
|
histogram*: StateHistogram
|
||||||
|
inputConfidence*: seq[InputConfidence]
|
||||||
|
summary*: string
|
||||||
|
|
||||||
|
proc settlednessVerdict*(s: SettlednessResult): string =
|
||||||
|
if s.overallMean >= 0.65: "settled"
|
||||||
|
elif s.overallMean >= 0.40: "settling"
|
||||||
|
else: "wavering"
|
||||||
|
|
||||||
|
proc diversityVerdict*(d: DiversityResult): string =
|
||||||
|
if not d.valid: "n/a"
|
||||||
|
elif d.jaccardOverall < 0.15: "diverse"
|
||||||
|
elif d.jaccardOverall <= 0.70: "moderate"
|
||||||
|
else: "redundant"
|
||||||
|
|
||||||
|
proc churnVerdict*(c: ChurnResult): string =
|
||||||
|
case c.flipTrend
|
||||||
|
of "falling": "settling"
|
||||||
|
of "rising": "fidgeting (rising)"
|
||||||
|
of "frozen": "frozen"
|
||||||
|
of "insufficient", "not-measured": c.flipTrend
|
||||||
|
else:
|
||||||
|
if c.flipRatePer100 > 5.0: "fidgeting" else: "quiet"
|
||||||
|
|
||||||
|
proc disagreementVerdict*(d: VoteDisagreementResult): string =
|
||||||
|
if d.overall < 0.15: "coherent"
|
||||||
|
elif d.overall < 0.35: "fuzzy"
|
||||||
|
else: "fragmented"
|
||||||
|
|
||||||
|
proc healthLine*(d: AutomataDiag): string =
|
||||||
|
## ONE line to glance at: settledness / diversity / churn trend / disagreement,
|
||||||
|
## each with its own verdict word.
|
||||||
|
&"settledness={d.settledness.overallMean:.3f} ({settlednessVerdict(d.settledness)}) | " &
|
||||||
|
&"diversity={d.diversity.jaccardOverall:.3f} ({diversityVerdict(d.diversity)}) | " &
|
||||||
|
&"churn={d.churn.flipRatePer100:.3f}/100 {d.churn.flipTrend} " &
|
||||||
|
&"({churnVerdict(d.churn)}) | " &
|
||||||
|
&"disagreement={d.disagreement.overall:.3f} " &
|
||||||
|
&"({disagreementVerdict(d.disagreement)})"
|
||||||
|
|
||||||
|
proc automataDiagnostics*(tmpl: TmMachine, samples: openArray[DiagSample],
|
||||||
|
spec: FeatureSpec, epochs = 1, seed = 777'u64,
|
||||||
|
settleThreshold = 0.5, nHistBins = 9,
|
||||||
|
window = 100, measureChurn = true): AutomataDiag =
|
||||||
|
## Train a fresh machine on `samples` (measuring churn along the way unless
|
||||||
|
## `measureChurn` is false) and compute every automata-level metric on it.
|
||||||
|
if measureChurn:
|
||||||
|
result.churn = churnTrace(tmpl, samples, epochs, seed, window)
|
||||||
|
result.machine = result.churn.trained
|
||||||
|
else:
|
||||||
|
result.machine = trainModel(tmpl, samples, epochs, seed)
|
||||||
|
result.churn.epochs = max(1, epochs)
|
||||||
|
result.churn.samples = samples.len
|
||||||
|
result.churn.trained = result.machine
|
||||||
|
result.churn.flipTrend = "not-measured"
|
||||||
|
result.churn.clauseTrend = "not-measured"
|
||||||
|
result.settledness = settledness(result.machine, settleThreshold)
|
||||||
|
result.diversity = clauseDiversity(result.machine)
|
||||||
|
result.disagreement = voteDisagreement(result.machine, samples)
|
||||||
|
result.histogram = stateHistogram(result.machine, nHistBins)
|
||||||
|
result.inputConfidence = perInputConfidence(result.machine, spec, samples,
|
||||||
|
settleThreshold)
|
||||||
|
result.summary = healthLine(result)
|
||||||
|
|
||||||
|
proc automataVerdict*(d: AutomataDiag): string =
|
||||||
|
## One word for the whole trajectory: settling / fidgeting / collapsed / other.
|
||||||
|
if d.churn.flipTrend == "falling" and d.settledness.overallMean >= 0.40:
|
||||||
|
"settling"
|
||||||
|
elif d.churn.flipTrend in ["flat", "rising"] and d.churn.flipRatePer100 > 5.0:
|
||||||
|
"fidgeting"
|
||||||
|
elif d.settledness.overallMean < 0.25 and d.churn.flipTrend == "frozen":
|
||||||
|
"collapsed"
|
||||||
|
elif d.disagreement.overall > 0.35 and d.settledness.overallMean < 0.40:
|
||||||
|
"collapsed"
|
||||||
|
elif d.settledness.overallMean >= 0.65 and d.churn.flipTrend == "frozen":
|
||||||
|
"converged-static"
|
||||||
|
else:
|
||||||
|
"mixed"
|
||||||
|
|
||||||
|
proc formatAutomataReport*(d: AutomataDiag): string =
|
||||||
|
## The full readout: health line, verdict, state histogram, per-input
|
||||||
|
## confidence (least confident first) and per-class disagreement.
|
||||||
|
result.add "health: " & d.summary & "\n"
|
||||||
|
result.add "verdict: " & automataVerdict(d) & "\n"
|
||||||
|
result.add &"automata: n={d.settledness.nAutomata} settled>={d.settledness.threshold:.2f} " &
|
||||||
|
&"frac={d.settledness.overallSettledFraction:.3f}\n"
|
||||||
|
result.add &"churn: flips/sample={d.churn.flipRate:.6f} ({d.churn.flipRatePer100:.3f}/100, " &
|
||||||
|
&"{d.churn.flipTrend}: {d.churn.flipFirst:.6f}->{d.churn.flipLast:.6f}) " &
|
||||||
|
&"clause-change/sample={d.churn.clauseChangeRate:.6f} " &
|
||||||
|
&"({d.churn.clauseChangePer100:.3f}/100, {d.churn.clauseTrend})\n"
|
||||||
|
result.add &"diversity: pos={d.diversity.jaccardPos:.3f} " &
|
||||||
|
&"neg={d.diversity.jaccardNeg:.3f} overall={d.diversity.jaccardOverall:.3f} " &
|
||||||
|
&"(pairs={d.diversity.nPairs}, posClauses={d.diversity.nPosClauses}, " &
|
||||||
|
&"negClauses={d.diversity.nNegClauses})\n"
|
||||||
|
result.add "state histogram:\n" & histogramText(d.histogram)
|
||||||
|
result.add "per-input confidence (least confident first):\n"
|
||||||
|
let ranked = rankedInputConfidence(d.inputConfidence)
|
||||||
|
for i in 0..<min(12, ranked.len):
|
||||||
|
let tag = if ranked[i].constant: " CONST" else: ""
|
||||||
|
result.add &" bit{ranked[i].bit:>2} {ranked[i].name:<28} " &
|
||||||
|
&"meanCommit={ranked[i].meanCommitment:.3f} settled={ranked[i].settledFraction:.3f}{tag}\n"
|
||||||
|
var cbits: seq[int]
|
||||||
|
for ic in d.inputConfidence:
|
||||||
|
if ic.constant: cbits.add ic.bit
|
||||||
|
result.add &"constant inputs (zero variance): {cbits}\n"
|
||||||
|
result.add "disagreement per class:"
|
||||||
|
for c in 0..<d.disagreement.perClass.len:
|
||||||
|
result.add &" c{c}={d.disagreement.perClass[c]:.3f}"
|
||||||
|
result.add "\n"
|
||||||
|
|||||||
Reference in New Issue
Block a user