Automata metrics: settledness alone does NOT separate learning from fidgeting
Added the four automata-level metrics to the TM diagnostics kit (settledness, clause diversity, churn, vote disagreement) plus a state histogram, a per-input confidence table and a one-line health summary, and validated them on a learnable-vs-noise pair. STATE CONVENTIONS, read off OUR code rather than from memory: range [-nStates, nStates] as int16; nStates = 64 for tm_pattern, 32 for tsetlin initial value 0 = the Exclude boundary INCLUDE iff state > 0; EXCLUDE iff state <= 0 flip boundary sits between state 0 and 1; commitment = abs(st)/nStates in [0,1] === THE GATE, AND A RESULT THAT MATTERS === Case A (learnable planted rule) vs Case B (shuffled labels), 49 bits, N=64: metric A (learnable) B (shuffled) settledness mean 0.970 0.719 churn flip/sample 0.000055 FALLING 0.000788 FLAT clause-change/sample 0.00263 falling 0.0595 flat diversity (Jaccard) 0.176 0.014 disagreement 0.003 0.298 verdict settling mixed (NOT settling) **SETTLEDNESS ALONE DOES NOT WORK.** On noise the automata still COMMIT (0.719) - they just commit to the wrong thing. The decisive separators are **churn TREND (falling vs flat)** and **vote DISAGREEMENT (0.003 vs 0.298)**. Had we built only the settledness metric - the one that seems most obvious - we would have been misled. That is now recorded in the README. INERTIA SWEEP: A vs B separate at N=16/32/64/128. **Raising N raises A's commitment but does NOT reduce B's noise-fitting** - so more inertia does not rescue a noise-fitting TM. === REAL READING ON THE SHIPPED GUN, AND THE INFERENCE IT SUPPORTS === tm_pattern GF head over the DrussGT fixtures: settledness 0.484 (settling), diversity 0.267 (moderate), churn 0.094/100 FALLING, disagreement 0.145 (coherent). **VERDICT: SETTLING** - not fidgeting, not collapsed. Constant inputs flagged: 38/39 (the known never-written bits) plus 19/36/37. Context: pooled warm accuracy 35.72% vs 34.24% majority = +1.48pp. So: **the old gun was NOT failing because of inertia or instability - it settled properly and its settled rules still barely beat a lazy guess.** Its settledness (0.484) is LOWER than both synthetic cases (0.97/0.72), which is the signature of WEAK OR CONFLICTING SIGNAL rather than too much inertia. CONCLUSION: **N and s are not the observed bottleneck. The target/representation is.** That is exactly why the new design changes the target and the label pipeline rather than sweeping knobs - and it means we should NOT spend effort on an N/s sweep expecting it to fix anything. Also adds `diag_automata_validation.nim` (Case A/B/C + inertia sweep) and `test_tm_automata_diag.nim` (55 pure checks); `test_tm_diag` 48 and `diag_synthetic` 17 still pass, plus all other guards. acceptance_offline_vs_online was NOT run (it needs a live battle and there is no tm_diag dependency). Caveat: churn on the real gun is a PROXY (a tm_core retrain over captured samples in live order) because the live gun exposes no per-sample state trace; the other metrics are read directly off the exported teams.
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
## Task 5 — VALIDATE THE AUTOMATA-LEVEL METRICS against known ground truth.
|
||||
##
|
||||
## Case A: the planted-rule set from diag_synthetic.nim (learnable).
|
||||
## Case B: the SAME inputs with the labels shuffled (pure noise).
|
||||
## The metrics must SEPARATE learning from fidgeting:
|
||||
## A: settledness high and rising, churn trend falling, disagreement low.
|
||||
## B: settledness low, churn high / not falling, disagreement high.
|
||||
## Case C: a CONSTANT input, to confirm the kit surfaces an information-free bit.
|
||||
##
|
||||
## Every number printed here is MEASURED.
|
||||
## Run: nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim
|
||||
|
||||
import std/[random, strformat, strutils, algorithm]
|
||||
import tm_diag/diagnostics
|
||||
|
||||
const
|
||||
NBits = 49
|
||||
BitA = 0
|
||||
BitB = 45
|
||||
NoiseBit = 17
|
||||
ConstBit = 20
|
||||
NClasses = 3
|
||||
Epochs = 15
|
||||
|
||||
var failures = 0
|
||||
proc check(name: string, ok: bool) =
|
||||
if ok: echo "PASS: ", name
|
||||
else: echo "FAIL: ", name; inc failures
|
||||
|
||||
proc genDataset(n, seed: int, constant = false): seq[DiagSample] =
|
||||
var rng = initRand(seed)
|
||||
for i in 0..<n:
|
||||
var raw = newSeq[int](NBits)
|
||||
for b in 0..<NBits:
|
||||
raw[b] = (if rng.rand(1.0) < 0.5: 1 else: 0)
|
||||
let a = if rng.rand(1.0) < 0.4: 1 else: 0
|
||||
let bb = if rng.rand(1.0) < 0.5: 1 else: 0
|
||||
raw[BitA] = a
|
||||
raw[BitB] = bb
|
||||
if constant: raw[ConstBit] = 1
|
||||
let label =
|
||||
if a == 1 and bb == 1: 2
|
||||
elif a == 1: 1
|
||||
else: 0
|
||||
result.add makeSample(NBits, raw, label, i)
|
||||
|
||||
proc shuffleLabels(s: seq[DiagSample], seed: int): seq[DiagSample] =
|
||||
result = s
|
||||
var rng = initRand(seed)
|
||||
var labels = newSeq[int](s.len)
|
||||
for i in 0..<s.len: labels[i] = s[i].label
|
||||
for i in countdown(s.len - 1, 1):
|
||||
let j = rng.rand(i)
|
||||
swap(labels[i], labels[j])
|
||||
for i in 0..<s.len: result[i].label = labels[i]
|
||||
|
||||
proc report(tag: string, d: AutomataDiag) =
|
||||
echo &"\n## {tag}"
|
||||
echo "# ", d.summary
|
||||
echo "# verdict=", automataVerdict(d)
|
||||
echo &"# settledness: mean={d.settledness.overallMean:.4f} " &
|
||||
&"settledFrac={d.settledness.overallSettledFraction:.4f} " &
|
||||
&"(threshold={d.settledness.threshold:.2f})"
|
||||
echo &"# diversity: pos={d.diversity.jaccardPos:.4f} neg={d.diversity.jaccardNeg:.4f} " &
|
||||
&"overall={d.diversity.jaccardOverall:.4f} pairs={d.diversity.nPairs}"
|
||||
echo &"# churn: flip/sample={d.churn.flipRate:.6f} ({d.churn.flipRatePer100:.4f}/100) " &
|
||||
&"trend={d.churn.flipTrend} first={d.churn.flipFirst:.6f} last={d.churn.flipLast:.6f}"
|
||||
echo &"# clause-change/sample={d.churn.clauseChangeRate:.6f} " &
|
||||
&"({d.churn.clauseChangePer100:.4f}/100) trend={d.churn.clauseTrend} " &
|
||||
&"first={d.churn.clauseFirst:.6f} last={d.churn.clauseLast:.6f}"
|
||||
echo &"# disagreement: overall={d.disagreement.overall:.4f} perClass={d.disagreement.perClass}"
|
||||
|
||||
when isMainModule:
|
||||
let spec = draftTMSpec()
|
||||
let trainA = genDataset(3000, 1)
|
||||
let trainB = shuffleLabels(trainA, 99)
|
||||
let tmpl = newMachine(NBits, NClasses, nClauses = 40, nStates = 64,
|
||||
sValue = 3.0, seed = 1)
|
||||
|
||||
echo &"# automata validation: nBits={NBits} classes={NClasses} clauses=40 states=64 " &
|
||||
&"samples={trainA.len} epochs={Epochs}"
|
||||
|
||||
let dA = automataDiagnostics(tmpl, trainA, spec, epochs = Epochs, seed = 777,
|
||||
settleThreshold = 0.5)
|
||||
let dB = automataDiagnostics(tmpl, trainB, spec, epochs = Epochs, seed = 777,
|
||||
settleThreshold = 0.5)
|
||||
report("CASE A — learnable planted rule", dA)
|
||||
report("CASE B — shuffled (noise) labels", dB)
|
||||
|
||||
# settledness RISING: early prefix vs full training.
|
||||
let early = trainModel(tmpl, trainA[0..<500], epochs = 3, seed = 777)
|
||||
let earlyS = settledness(early, 0.5)
|
||||
let lateS = dA.settledness
|
||||
echo &"\n## CASE A settledness trajectory: early(prefix 500 x3)={earlyS.overallMean:.4f} " &
|
||||
&"-> late(full)={lateS.overallMean:.4f} ({settlednessTrend(earlyS, lateS)})"
|
||||
|
||||
echo "\n## CHECKS"
|
||||
check "A settledness is high at convergence (>= 0.50)", dA.settledness.overallMean >= 0.50
|
||||
check "A churn TRENDS DOWN (falling)",
|
||||
dA.churn.flipTrend == "falling" and dA.churn.flipLast < dA.churn.flipFirst
|
||||
check "A disagreement is low (< 0.15)", dA.disagreement.overall < 0.15
|
||||
check "A settledness RISES from early to late",
|
||||
lateS.overallMean > earlyS.overallMean
|
||||
check "B churn does NOT fall (flat/rising/frozen)",
|
||||
dB.churn.flipTrend in ["flat", "rising", "frozen"]
|
||||
check "A is clearly more settled than B (A - B >= 0.10)",
|
||||
dA.settledness.overallMean - dB.settledness.overallMean >= 0.10
|
||||
check "B disagrees clearly more than A (B - A >= 0.10)",
|
||||
dB.disagreement.overall - dA.disagreement.overall >= 0.10
|
||||
check "the pair is separated (A verdict settling, B not settling)",
|
||||
automataVerdict(dA) == "settling" and automataVerdict(dB) != "settling"
|
||||
|
||||
# ── Case C: constant input ──
|
||||
let trainC = genDataset(2000, 3, constant = true)
|
||||
let dC = automataDiagnostics(tmpl, trainC, spec, epochs = Epochs, seed = 777)
|
||||
echo "\n## CASE C — constant input (bit 20 forced to 1)"
|
||||
var cbit: InputConfidence
|
||||
for ic in dC.inputConfidence:
|
||||
if ic.bit == ConstBit: cbit = ic
|
||||
echo &"# bit{ConstBit} constant={cbit.constant} meanCommit={cbit.meanCommitment:.4f} " &
|
||||
&"settled={cbit.settledFraction:.4f}"
|
||||
let consts = constantInputs(trainC, NBits)
|
||||
echo &"# constantInputs(trainC) = {consts}"
|
||||
check "the constant bit is flagged in the confidence table", cbit.constant
|
||||
check "constantInputs() surfaces the constant bit", ConstBit in consts
|
||||
check "a real planted bit is NOT flagged constant",
|
||||
not dC.inputConfidence[BitA].constant
|
||||
|
||||
# ── Inertia sweep: how the automata metrics move with N ──
|
||||
echo "\n## INERTIA SWEEP — settledness / churn / disagreement vs N (10 epochs)"
|
||||
echo "# case,N,settledness,churnTrend,flipPer100,disagreement,verdict"
|
||||
for n in [16, 32, 64, 128]:
|
||||
let tmplN = newMachine(NBits, NClasses, 40, n, 3.0, 1)
|
||||
let aN = automataDiagnostics(tmplN, trainA, spec, epochs = 10, seed = 777)
|
||||
let bN = automataDiagnostics(tmplN, trainB, spec, epochs = 10, seed = 777)
|
||||
echo &"# A,{n},{aN.settledness.overallMean:.3f},{aN.churn.flipTrend}," &
|
||||
&"{aN.churn.flipRatePer100:.3f},{aN.disagreement.overall:.3f},{automataVerdict(aN)}"
|
||||
echo &"# B,{n},{bN.settledness.overallMean:.3f},{bN.churn.flipTrend}," &
|
||||
&"{bN.churn.flipRatePer100:.3f},{bN.disagreement.overall:.3f},{automataVerdict(bN)}"
|
||||
|
||||
echo ""
|
||||
if failures > 0:
|
||||
echo &"{failures} check(s) FAILED"
|
||||
quit(1)
|
||||
echo "All automata-validation checks passed."
|
||||
@@ -177,5 +177,32 @@ proc main() =
|
||||
echo &"# class{cls} votes={c.votes:<7} len={c.length} {c.text}"
|
||||
echo &"# -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}"
|
||||
|
||||
# ── AUTOMATA-LEVEL metrics on the shipped GF head (Task 5) ──
|
||||
# settledness / diversity / histogram / per-input confidence / disagreement
|
||||
# are read DIRECTLY off the exported teams; churn is measured on a tm_core
|
||||
# retrain (same algorithm) over the captured samples, because the live gun
|
||||
# does not expose a per-sample state trace.
|
||||
var ad = AutomataDiag()
|
||||
ad.machine = m
|
||||
ad.settledness = settledness(m, 0.5)
|
||||
ad.diversity = clauseDiversity(m)
|
||||
ad.histogram = stateHistogram(m, 9)
|
||||
ad.inputConfidence = perInputConfidence(m, spec, bestSamples, 0.5)
|
||||
ad.disagreement = voteDisagreement(m, bestSamples)
|
||||
let churnN = min(bestSamples.len, 10000)
|
||||
var churnSamples = newSeq[DiagSample](churnN)
|
||||
for i in 0..<churnN: churnSamples[i] = bestSamples[i]
|
||||
let tmpl = newMachine(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S,
|
||||
seed = 1)
|
||||
# Mirror the live gun: one TEMPORAL pass (no shuffle) as bullets resolve.
|
||||
ad.churn = churnTrace(tmpl, churnSamples, epochs = 1, seed = 777,
|
||||
window = 200, shuffle = false)
|
||||
ad.summary = healthLine(ad)
|
||||
echo "\n## AUTOMATA-LEVEL metrics (shipped tm_pattern GF head)"
|
||||
echo "# churn measured on a tm_core retrain over ", churnN,
|
||||
" captured samples, ONE temporal pass (live-order proxy)"
|
||||
echo formatAutomataReport(ad)
|
||||
echo "# VERDICT: ", automataVerdict(ad)
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
## Pure unit tests for the AUTOMATA-LEVEL metrics added to tm_diag:
|
||||
## settledness, clause diversity, churn, vote disagreement, the state
|
||||
## histogram, per-input confidence and the health/verdict helpers.
|
||||
##
|
||||
## Run: nim c -r --path:common_libs common_libs/tests/test_tm_automata_diag.nim
|
||||
|
||||
import tm_diag/diagnostics
|
||||
import std/strutils
|
||||
|
||||
var checks = 0
|
||||
var failures = 0
|
||||
proc check(name: string, ok: bool) =
|
||||
inc checks
|
||||
if ok: echo "PASS: ", name
|
||||
else: echo "FAIL: ", name; inc failures
|
||||
|
||||
# ── state conventions ────────────────────────────────────────────────────────
|
||||
|
||||
proc testStateConventions() =
|
||||
check "commitment(0) = 0 (on the flip boundary)",
|
||||
stateCommitment(64, 0) == 0.0
|
||||
check "commitment(+64) = 1 (extreme Include)",
|
||||
stateCommitment(64, 64) == 1.0
|
||||
check "commitment(-64) = 1 (extreme Exclude)",
|
||||
stateCommitment(64, -64) == 1.0
|
||||
check "commitment(+32) = 0.5", abs(stateCommitment(64, 32) - 0.5) < 1e-12
|
||||
check "commitment is symmetric", stateCommitment(64, 17) == stateCommitment(64, -17)
|
||||
check "include predicate matches state>0",
|
||||
stateIncluded(1) and not stateIncluded(0) and not stateIncluded(-1)
|
||||
|
||||
# ── settledness ──────────────────────────────────────────────────────────────
|
||||
|
||||
proc settledMachine(): TmMachine =
|
||||
## nBits=4, 2 classes, 4 clauses/class, 8 literals.
|
||||
## class0/clause0 = [+64, -64, 0, +32, 0,0,0,0] -> mean commitment 0.625,
|
||||
## 3 of 8 automata settled at threshold 0.5.
|
||||
result = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||
result.teams[0][0] = 64
|
||||
result.teams[0][1] = -64
|
||||
result.teams[0][2] = 0
|
||||
result.teams[0][3] = 32
|
||||
|
||||
proc testSettledness() =
|
||||
let m = settledMachine()
|
||||
let s = settledness(m, 0.5)
|
||||
check "settledness covers every clause",
|
||||
s.clauses.len == m.nClasses * m.nClauses
|
||||
check "settledness nAutomata = classes*clauses*literals",
|
||||
s.nAutomata == m.nClasses * m.nClauses * m.nLiterals
|
||||
var c0: ClauseSettledness
|
||||
for c in s.clauses:
|
||||
if c.cls == 0 and c.index == 0: c0 = c
|
||||
check "clause0 mean commitment = 0.3125",
|
||||
abs(c0.meanCommitment - 0.3125) < 1e-12
|
||||
check "clause0 settled fraction = 3/8",
|
||||
abs(c0.settledFraction - 3.0 / 8.0) < 1e-12
|
||||
check "clause0 positive polarity", c0.polarity == 1
|
||||
check "overall settled fraction = 3/64",
|
||||
abs(s.overallSettledFraction - 3.0 / 64.0) < 1e-12
|
||||
check "overall mean commitment = 2.5/64",
|
||||
abs(s.overallMean - 2.5 / 64.0) < 1e-12
|
||||
# trend helper
|
||||
var lo = s
|
||||
lo.overallMean = 0.2
|
||||
var hi = s
|
||||
hi.overallMean = 0.8
|
||||
check "settlednessTrend rising", settlednessTrend(lo, hi) == "rising"
|
||||
check "settlednessTrend falling", settlednessTrend(hi, lo) == "falling"
|
||||
check "settlednessTrend flat", settlednessTrend(s, s) == "flat"
|
||||
|
||||
# ── state histogram ──────────────────────────────────────────────────────────
|
||||
|
||||
proc testHistogram() =
|
||||
let m = settledMachine()
|
||||
let h = stateHistogram(m, 9)
|
||||
var total = 0
|
||||
for c in h.counts: total += c
|
||||
check "histogram counts sum to every automaton",
|
||||
total == m.nClasses * m.nClauses * m.nLiterals
|
||||
check "the -64 state lands in the first bin", h.counts[0] >= 1
|
||||
check "the +64 state lands in the last bin", h.counts[^1] >= 1
|
||||
check "histogram text is non-empty", histogramText(h).len > 0
|
||||
|
||||
# ── per-input confidence ─────────────────────────────────────────────────────
|
||||
|
||||
proc testInputConfidence() =
|
||||
let m = settledMachine()
|
||||
let spec = draftTMSpec()
|
||||
let samples = @[
|
||||
makeSample(49, (block:
|
||||
var r = newSeq[int](49)
|
||||
for i in 0..<49: r[i] = 1
|
||||
r), 0),
|
||||
makeSample(49, (block:
|
||||
var r = newSeq[int](49)
|
||||
for i in 0..<49: r[i] = 1
|
||||
r), 0),
|
||||
]
|
||||
# all-ones samples make every bit constant, so all should be flagged.
|
||||
let conf = perInputConfidence(m, spec, samples, 0.5)
|
||||
check "confidence table covers every bit", conf.len == m.nBits
|
||||
check "confidence nAutomata per bit = classes*clauses*2",
|
||||
conf[0].nAutomata == m.nClasses * m.nClauses * 2
|
||||
check "a zero-variance bit is flagged constant", conf[0].constant
|
||||
let ranked = rankedInputConfidence(conf)
|
||||
check "ranked confidence is ascending",
|
||||
ranked.len == conf.len and
|
||||
ranked[0].meanCommitment <= ranked[^1].meanCommitment
|
||||
|
||||
# ── diversity / jaccard ──────────────────────────────────────────────────────
|
||||
|
||||
proc testJaccard() =
|
||||
check "identical sets -> 1", jaccard(@[1, 2, 3], @[1, 2, 3]) == 1.0
|
||||
check "disjoint sets -> 0", jaccard(@[1, 2], @[3, 4]) == 0.0
|
||||
check "two empty sets -> 1", jaccard(@[], @[]) == 1.0
|
||||
check "empty vs non-empty -> 0", jaccard(@[], @[1]) == 0.0
|
||||
check "partial overlap -> |n|/|u|",
|
||||
abs(jaccard(@[1, 2], @[2, 3]) - 1.0 / 3.0) < 1e-12
|
||||
|
||||
proc diversityMachine(): TmMachine =
|
||||
## nBits=4, 2 classes, 6 clauses/class (half=3 positive, 3 negative).
|
||||
## class0 positive clauses: {0,1}, {0,1}, {2,3}; negative clauses empty.
|
||||
result = newMachine(4, 2, 6, 64, 3.0, 1)
|
||||
for lit in [0, 1]:
|
||||
result.teams[0][0 * 8 + lit] = 1
|
||||
result.teams[0][1 * 8 + lit] = 1
|
||||
for lit in [2, 3]:
|
||||
result.teams[0][2 * 8 + lit] = 1
|
||||
|
||||
proc testDiversity() =
|
||||
let m = diversityMachine()
|
||||
let d = clauseDiversity(m)
|
||||
check "positive clauses: {0,1},{0,1},{2,3} mean Jaccard = 1/3",
|
||||
abs(d.jaccardPos - 1.0 / 3.0) < 1e-12
|
||||
check "positive diversity is valid", d.posValid
|
||||
check "empty negative clauses are skipped", not d.negValid
|
||||
check "overall pools the same 3 pairs", d.nPairs == 3
|
||||
check "overall Jaccard = 1/3",
|
||||
abs(d.jaccardOverall - 1.0 / 3.0) < 1e-12
|
||||
|
||||
# ── churn ────────────────────────────────────────────────────────────────────
|
||||
|
||||
proc testChurnHelpers() =
|
||||
check "binMeans bins [1,1,3,3,5] with window 2 -> [1,3,5]",
|
||||
binMeans(@[1.0, 1.0, 3.0, 3.0, 5.0], 2) == @[1.0, 3.0, 5.0]
|
||||
let fl = firstLastMeans(@[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0])
|
||||
check "firstLastMeans first ~0", fl.first < 1e-9
|
||||
check "firstLastMeans last ~1", abs(fl.last - 1.0) < 1e-9
|
||||
check "churn trend falling",
|
||||
classifyChurnTrend(@[0.9, 0.8, 0.7, 0.6, 0.5, 0.1, 0.05, 0.01]) == "falling"
|
||||
check "churn trend flat",
|
||||
classifyChurnTrend(@[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]) == "flat"
|
||||
check "churn trend rising",
|
||||
classifyChurnTrend(@[0.01, 0.02, 0.03, 0.1, 0.5, 0.6, 0.7, 0.9]) == "rising"
|
||||
check "churn trend frozen",
|
||||
classifyChurnTrend(@[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) == "frozen"
|
||||
|
||||
proc testChurnTrace() =
|
||||
var samples: seq[DiagSample]
|
||||
for i in 0..<120:
|
||||
var raw = newSeq[int](49)
|
||||
for b in 0..<49: raw[b] = (i + b) mod 2
|
||||
raw[0] = 1
|
||||
raw[45] = if i mod 2 == 0: 1 else: 0
|
||||
samples.add makeSample(49, raw, if raw[45] == 1: 2 else: 1, i)
|
||||
let tmpl = newMachine(49, 3, 40, 64, 3.0, 1)
|
||||
let c = churnTrace(tmpl, samples, epochs = 1, seed = 5, window = 30)
|
||||
check "churn nAutomata = classes*clauses*literals",
|
||||
c.nAutomata == 3 * 40 * 98
|
||||
check "churn nClauses = classes*clauses", c.nClauses == 3 * 40
|
||||
check "churn rate is non-negative", c.flipRate >= 0.0
|
||||
check "churn per100 = rate*100",
|
||||
abs(c.flipRatePer100 - c.flipRate * 100.0) < 1e-12
|
||||
check "churn windows are non-empty", c.flipWindows.len > 0
|
||||
check "churn trained machine has the right shape",
|
||||
c.trained.teams.len == 3 and c.trained.teams[0].len == 40 * 98
|
||||
|
||||
# ── vote disagreement ────────────────────────────────────────────────────────
|
||||
|
||||
proc testVoteDisagreement() =
|
||||
# all-agree machine: only positive clauses fire -> disagreement 0
|
||||
var agree = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||
agree.teams[0][0 * 8 + 0] = 1
|
||||
agree.teams[0][1 * 8 + 1] = 1
|
||||
let allOnes = @[makeSample(4, @[1, 1, 1, 1], 0)]
|
||||
let da = voteDisagreement(agree, allOnes)
|
||||
check "all-positive firing clauses -> disagreement 0",
|
||||
abs(da.overall - 0.0) < 1e-12
|
||||
check "one class observed", da.perClassSamples[0] == 1
|
||||
|
||||
# split machine: one positive and one negative clause fire -> tied vote -> 1
|
||||
var split = newMachine(4, 2, 4, 64, 3.0, 1)
|
||||
split.teams[0][0 * 8 + 0] = 1
|
||||
split.teams[0][2 * 8 + 2] = 1
|
||||
let ds = voteDisagreement(split, allOnes)
|
||||
check "a tied vote counts every firing clause as disagreeing",
|
||||
abs(ds.overall - 1.0) < 1e-12
|
||||
|
||||
# ── health line / verdict / report ───────────────────────────────────────────
|
||||
|
||||
proc testHealthLine() =
|
||||
let spec = draftTMSpec()
|
||||
var samples: seq[DiagSample]
|
||||
for i in 0..<200:
|
||||
var raw = newSeq[int](49)
|
||||
for b in 0..<49: raw[b] = if (i * 7 + b) mod 3 == 0: 1 else: 0
|
||||
raw[0] = if i mod 2 == 0: 1 else: 0
|
||||
raw[45] = if i mod 3 == 0: 1 else: 0
|
||||
let label = if raw[0] == 1 and raw[45] == 1: 2 elif raw[0] == 1: 1 else: 0
|
||||
samples.add makeSample(49, raw, label, i)
|
||||
let tmpl = newMachine(49, 3, 40, 64, 3.0, 1)
|
||||
let d = automataDiagnostics(tmpl, samples, spec, epochs = 3, seed = 7,
|
||||
settleThreshold = 0.5, nHistBins = 9)
|
||||
check "health line is non-empty and mentions every metric",
|
||||
d.summary.len > 0 and "settledness" in d.summary and
|
||||
"diversity" in d.summary and "churn" in d.summary and
|
||||
"disagreement" in d.summary
|
||||
check "automataVerdict is non-empty", automataVerdict(d).len > 0
|
||||
check "formatAutomataReport contains the histogram and table",
|
||||
"state histogram" in formatAutomataReport(d) and
|
||||
"per-input confidence" in formatAutomataReport(d)
|
||||
check "disagreement per-class array is the right size",
|
||||
d.disagreement.perClass.len == 3
|
||||
check "input confidence covers all bits", d.inputConfidence.len == 49
|
||||
|
||||
when isMainModule:
|
||||
testStateConventions()
|
||||
testSettledness()
|
||||
testHistogram()
|
||||
testInputConfidence()
|
||||
testJaccard()
|
||||
testDiversity()
|
||||
testChurnHelpers()
|
||||
testChurnTrace()
|
||||
testVoteDisagreement()
|
||||
testHealthLine()
|
||||
echo ""
|
||||
if failures > 0:
|
||||
echo failures, " / ", checks, " check(s) FAILED"
|
||||
quit(1)
|
||||
echo "All ", checks, " automata-diag unit checks passed."
|
||||
Reference in New Issue
Block a user