diff --git a/common_libs/tests/diag_tm_pattern_offline.nim b/common_libs/tests/diag_tm_pattern_offline.nim index 95aaeb7..b372fa8 100644 --- a/common_libs/tests/diag_tm_pattern_offline.nim +++ b/common_libs/tests/diag_tm_pattern_offline.nim @@ -177,6 +177,14 @@ proc main() = echo &"# class{cls} votes={c.votes:<7} len={c.length} {c.text}" echo &"# -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}" + # ── CLAUSE-SHAPE checker (Task 2/4): distribution + healthy-band verdict ── + # The default healthy band is 3-8 literals for a ~50-bit problem; the shipped + # tm_pattern is 40 bits, so the same band is used and the verdict is explicit. + let sd = clauseShapeDiagnostics(m, bestSamples, spec) + echo "\n## CLAUSE-SHAPE readout (healthy band 3-8 literals)" + echo formatClauseShapeReport(sd) + echo "# SHAPE VERDICT: ", sd.verdict + # ── AUTOMATA-LEVEL metrics on the shipped GF head (Task 5) ── # settledness / diversity / histogram / per-input confidence / disagreement # are read DIRECTLY off the exported teams; churn is measured on a tm_core @@ -189,6 +197,7 @@ proc main() = ad.histogram = stateHistogram(m, 9) ad.inputConfidence = perInputConfidence(m, spec, bestSamples, 0.5) ad.disagreement = voteDisagreement(m, bestSamples) + ad.shape = clauseShapeDiagnostics(m, bestSamples, spec) let churnN = min(bestSamples.len, 10000) var churnSamples = newSeq[DiagSample](churnN) for i in 0..= half=2) + result.teams[0][2 * nl + 6] = 1 + +proc testClauseSummaryShape() = + let m = shapeModel() + let spec = tinySpec() + let samples = @[makeSample(8, @[1, 1, 1, 0, 1, 1, 1, 0], 0)] + let summ = clauseSummary(clauseInfo(m, samples, spec)) + check "summary nonEmpty = 3", summ.nonEmpty == 3 + check "summary empty = 5", summ.emptyClauses == 5 + check "summary mean length = 2.0", abs(summ.meanLength - 2.0) < 1e-9 + check "summary median = 2.0", abs(summ.medianLength - 2.0) < 1e-9 + check "summary min = 1", summ.minLength == 1 + check "summary max = 3", summ.maxLength == 3 + check "summary lengthHist", summ.lengthHist == @[0, 1, 1, 1] + check "summary posMean = (3+1)/2", abs(summ.posMeanLength - 2.0) < 1e-9 + check "summary negMean = 2", abs(summ.negMeanLength - 2.0) < 1e-9 + check "summary per-class mean = 2.0", + abs(summ.perClassMeanLength[0] - 2.0) < 1e-9 + check "summary emptyFraction = 5/8", + abs(summ.emptyFraction - 5.0 / 8.0) < 1e-9 + +# ── healthy-band verdict ───────────────────────────────────────────────────── + +proc verdictOf(mean: float, n = 4): string = + var s = ClauseSummary() + s.nonEmpty = n + s.totalClauses = 8 + s.meanLength = mean + clauseShapeVerdict(s) + +proc testShapeVerdict() = + check "mean 1.5 -> collapsed", verdictOf(1.5) == "collapsed" + check "mean 2.5 -> short", verdictOf(2.5) == "short" + check "mean 3.0 -> healthy (band low edge)", verdictOf(3.0) == "healthy" + check "mean 8.0 -> healthy (band high edge)", verdictOf(8.0) == "healthy" + check "mean 12.0 -> too long", verdictOf(12.0) == "too long" + check "mean 19.17 -> too long (the shipped gun)", verdictOf(19.17) == "too long" + check "no non-empty clauses -> n/a", verdictOf(0.0, 0) == "n/a" + # custom band is honoured + var s = ClauseSummary() + s.nonEmpty = 4 + s.totalClauses = 8 + s.meanLength = 2.5 + check "custom band 2-6 makes mean 2.5 healthy", + clauseShapeVerdict(s, 2.0, 6.0, 1.0) == "healthy" + check "custom collapsedMax makes mean 2.5 collapsed", + clauseShapeVerdict(s, 2.0, 6.0, 3.0) == "collapsed" + +proc testLengthHistText() = + let t = lengthHistText(@[0, 3, 1]) + check "lengthHistText skips empty bins", "len 0" notin t + check "lengthHistText renders len 1", "len 1: 3" in t + check "lengthHistText renders len 2", "len 2: 1" in t + +# ── per-block contribution ─────────────────────────────────────────────────── + +proc testBlockContribution() = + let m = shapeModel() + let spec = tinySpec() + let blocks = blockLengthContributions(m, spec) + check "one entry per block", blocks.len == 2 + # block0 (bits 0-3): 2 in clause0 + 1 in clause1 = 3 over 3 clauses + check "block0 total = 3", blocks[0].totalLits == 3 + check "block0 mean/clause = 1.0", abs(blocks[0].meanPerClause - 1.0) < 1e-9 + check "block0 clausesUsing = 2", blocks[0].clausesUsing == 2 + check "block0 max per clause = 2", blocks[0].perClauseMax == 2 + # block1 (bits 4-7): 1 in clause0 + 2 in neg clause2 = 3 over 3 clauses + check "block1 total = 3", blocks[1].totalLits == 3 + check "block1 mean/clause = 1.0", abs(blocks[1].meanPerClause - 1.0) < 1e-9 + check "shares sum to 1", abs(blocks[0].share + blocks[1].share - 1.0) < 1e-9 + # skipEmpty = false averages over all 8 clauses instead of 3 + let blocksAll = blockLengthContributions(m, spec, skipEmpty = false) + check "skipEmpty=false dilutes the mean", + abs(blocksAll[0].meanPerClause - 3.0 / 8.0) < 1e-9 + +# ── coverage / concentration ───────────────────────────────────────────────── + +proc testCoverage() = + ## 2 classes x 2 clauses = 4 clauses. class0/clause0 = {+b0.0} fires on the + ## all-ones sample; nothing else fires. So 1 of 4 clauses fires. + var m = newMachine(8, 2, 2, 64, 3.0, 1) + let nl = m.nLiterals + m.teams[0][0 * nl + 0] = 1 + let samples = @[ + makeSample(8, @[1, 0, 0, 0, 0, 0, 0, 0], 0), + makeSample(8, @[1, 1, 1, 1, 1, 1, 1, 1], 0), + ] + let cov = clauseCoverage(m, samples) + check "coverage totalClauses = 4", cov.totalClauses == 4 + check "coverage mean firing = 1", abs(cov.meanFiringClauses - 1.0) < 1e-9 + check "coverage fraction = 1/4", abs(cov.meanFiringFraction - 0.25) < 1e-9 + check "one firing clause", cov.firingClauseCount == 1 + check "effective clauses = 1 (single clause does all voting)", + abs(cov.effectiveClauses - 1.0) < 1e-9 + check "top3 share = 1 (all fires from one clause)", + abs(cov.top3Share - 1.0) < 1e-9 + +# ── report helpers ─────────────────────────────────────────────────────────── + +proc testReportHelpers() = + let m = shapeModel() + let spec = tinySpec() + let samples = @[makeSample(8, @[1, 1, 1, 0, 1, 1, 1, 0], 0)] + let d = clauseShapeDiagnostics(m, samples, spec) + check "shapeLine mentions shape/coverage", + "shape mean=" in shapeLine(d) and "coverage=" in shapeLine(d) + let rep = formatClauseShapeReport(d) + check "report has length histogram", "length histogram" in rep + check "report has per-block contribution", "per-block literal contribution" in rep + check "report has per-class mean", "per-class mean/median" in rep + +# ── Task 3: synthetic validation with KNOWN geometry ───────────────────────── + +const + NBits = 49 + BitA = 0 ## dist-to-nearest-wall (WALLS block) + BitB = 45 ## bullet-lateral-offset (BULLETS block) + NoiseBit = 17 + NClasses = 3 + +proc genRule(n, seed: int): seq[DiagSample] = + ## class2 = A AND B ; class1 = A AND NOT B ; class0 = NOT A. + var rng = initRand(seed) + for i in 0..= 3.0 and d.summary.meanLength <= 8.0 + # the recovered necessary literals are the 2-literal rule + var c2ok = false + let c2 = necessaryLiterals(m, ev, 2) + var pos2: set[uint8] + for l in c2: + if l < NBits: pos2.incl uint8(l) + c2ok = pos2 == {uint8(BitA), uint8(BitB)} + check "SHORT rule recovered exactly (A AND B)", c2ok + check "SHORT rule is NOT called too long", d.verdict != "too long" + + # ── per-block contribution: planted blocks dominate, irrelevant block dead ── + # Use the sparser s=3 model here: with less padding the irrelevant block is + # unambiguously negligible (below the uniform 1/12 = 8.3% share). + let btmpl = newMachine(NBits, NClasses, nClauses = 40, nStates = 64, + sValue = 3.0, seed = 1) + let bm = trainModel(btmpl, train, epochs = 25, seed = 777) + let bd = clauseShapeDiagnostics(bm, ev, spec) + let walls = blockByName(bd.blocks, "dist-to-nearest-wall") + let bullets = blockByName(bd.blocks, "bullet-lateral-offset") + let usBlock = blockByName(bd.blocks, "dist-from-us") + echo &"# per-block (s=3.0): walls(mean={walls.meanPerClause:.2f} share={walls.share*100:.1f}%) " & + &"bullets(mean={bullets.meanPerClause:.2f} share={bullets.share*100:.1f}%) " & + &"dist-from-us(mean={usBlock.meanPerClause:.2f} share={usBlock.share*100:.1f}%) " & + &"[uniform=8.3%]" + check "the WALLS block (holds the planted A) dominates", walls.share > 0.25 + check "the BULLETS block (holds the planted B) is a top contributor", + bullets.share > 0.15 + check "the irrelevant US block is negligible (below uniform share)", + usBlock.share < 0.083 + check "the planted blocks outweigh the irrelevant block by >5x", + usBlock.totalLits > 0 and + walls.totalLits + bullets.totalLits > 5 * usBlock.totalLits + # the noise MOTION bit is not singled out as a top block + check "the pure-noise turn-direction block is far smaller than WALLS", + blockByName(bd.blocks, "turn-direction").totalLits < walls.totalLits + + # ── RANDOM labels: report whatever the checker actually says ── + let rtrain = genRandom(3000, 5) + let rev = genRandom(1500, 6) + let rtmpl = newMachine(NBits, NClasses, nClauses = 40, nStates = 64, + sValue = 3.0, seed = 1) + let rm = trainModel(rtmpl, rtrain, epochs = 25, seed = 777) + let rd = clauseShapeDiagnostics(rm, rev, spec) + echo &"# RANDOM labels s=3.0: mean={rd.summary.meanLength:.2f} " & + &"median={rd.summary.medianLength:.2f} p90={rd.summary.p90Length:.2f} " & + &"max={rd.summary.maxLength} empty={rd.summary.emptyClauses}/" & + &"{rd.summary.totalClauses} verdict={rd.verdict} " & + &"acc={evalAcc(rm, rev)*100:.1f}% eff={rd.coverage.effectiveClauses:.1f} " & + &"top3={rd.coverage.top3Share*100:.1f}%" + # On THIS encoding noise does not pad clauses long; it fails to commit at all + # (short clauses, many firing, low concentration). Report and assert only that + # the verdict is a legal word and that it is not "healthy". + check "RANDOM labels are not reported healthy", + rd.verdict in ["collapsed", "short", "too long"] + check "RANDOM-label accuracy is near the majority baseline", + abs(evalAcc(rm, rev) - 1.0 / 3.0) < 0.10 + + echo "" + if failures > 0: + echo &"{failures} / {checks} check(s) FAILED" + quit(1) + echo "All ", checks, " clause-shape checks passed." diff --git a/common_libs/tm_diag/README.md b/common_libs/tm_diag/README.md index 60c9b8b..c6741f6 100644 --- a/common_libs/tm_diag/README.md +++ b/common_libs/tm_diag/README.md @@ -16,7 +16,7 @@ Everything is pure and offline — no battles, no Java, no harness. |---|---| | `feature_spec.nim` | `FeatureSpec`, `describe`, `describeClause`, `draftTMSpec()` (49-bit draft), `tmPatternSpec()` (40-bit shipped encoding) | | `tm_core.nim` | compact deterministic Granmo Table 2/3 multiclass TM (mirrors the tm_pattern core), introspectable clause layout | -| `diagnostics.nim` | the seven groups (re-exports the two above) | +| `diagnostics.nim` | the eight groups (re-exports the two above) | Import everything with: @@ -24,6 +24,65 @@ Import everything with: import tm_diag/diagnostics ``` +## Task 1b — the `s` specificity knob (direction SETTLED, from the code) + +`TM_S` / `m.sValue` is the Type I specificity knob. It appears in ONE place in +each core — the Type I feedback branch of `tmLearnDir`: + +`common_libs/guns/tm_pattern.nim` lines 258-265 (identical in +`guns/tsetlin.nim` lines 224-231 and `tm_diag/tm_core.nim` lines 110-121): + +```nim +if pol * d > 0.0: + # Type I (Table 2) collapsed to the resulting state move: + # c=1, lk=1 -> +1 (toward Include) w.p. (s-1)/s + # c=0, lk=1 -> -1 (toward Exclude) w.p. 1/s <- the missing counter-force + # lk=0 -> -1 (toward Exclude) w.p. 1/s + for lit in 0..=3`: +the specificity mechanism is switched off. + +**Practical usable range: `s` in roughly [1.5, 5].** `s=1.0` is degenerate and +must be avoided. The shipped guns sit inside the range: `tm_pattern` uses `3.0`, +`tsetlin.nim` uses `1.5`. The clause-shape checker's healthy-band verdict is the +tool that tells you whether a given `s` produced a sane geometry. + ## Task 1 — named features / clause rendering ```nim @@ -43,7 +102,7 @@ last bits (`UNUSED-38/39`) are a real bug**: `tmBuildBits` writes only 38 raw bits into an `array[TM_NBITS=40, uint8]`, so bits 38 and 39 are always 0 and their negations always 1. The kit reports them as constant dead inputs. -## Task 2 — the six groups +## Task 2 — the data / clause / feature / accuracy / curve / ablation groups All functions take a trained `TmMachine` (or an externally supplied clause set) plus `seq[DiagSample]` where `DiagSample.lits` is the pos-then-neg literal vector @@ -93,6 +152,55 @@ let rs = ablateScrambleFeature(tmplMachine, train, eval, spec, bit) input earn its bits"). Pass `baselineAcc` from `ablateBaseline` to avoid recomputing it per block. +## Task 2b / 4 — the CLAUSE-SHAPE checker (group 8) + +The automata metrics say whether the TM is settling. The clause-shape checker +says whether what it settled ON is sane: a rule needing ~19 conditions to fire +is almost certainly fitting noise. + +```nim +import tm_diag/diagnostics + +let d = clauseShapeDiagnostics(m, samples, spec, + healthyLo = 3.0, healthyHi = 8.0, + collapsedMax = 2.0) +d.summary.meanLength # 19.17 for the shipped gun +d.summary.medianLength # 16.0 +d.summary.p10Length # 4.0 +d.summary.p90Length # 44.8 +d.summary.maxLength # 57 +d.summary.lengthHist # index = length, value = count +d.summary.posMeanLength / .negMeanLength +d.summary.perClassMeanLength / .perClassMedianLength / .perClassHist +d.verdict # "healthy" | "too long" | "collapsed" | "short" | "n/a" +d.blocks # per-FeatureSpec-block literal contribution +d.coverage # firing fraction + effective clause count +echo shapeLine(d) # one line, same style as healthLine +echo formatClauseShapeReport(d) # full readout +``` + +* **length distribution** — `meanLength`, `medianLength`, `p10Length`, + `p90Length`, `minLength`, `maxLength`, `stdLength` and `lengthHist`, plus + per-polarity (`posMeanLength` / `negMeanLength`) and per-class + (`perClassMeanLength` / `perClassMedianLength` / `perClassHist`) breakdowns. + `lengthStats(lengths)` and `percentile(sorted, p)` are exported standalone. +* **healthy-band verdict** — `clauseShapeVerdict(summ, healthyLo=3, healthyHi=8, + collapsedMax=2)`: `> healthyHi` = `too long`, `< collapsedMax` = `collapsed`, + between `collapsedMax` and `healthyLo` = `short`, in band = `healthy`, no + non-empty clauses = `n/a`. The band is a PARAMETER of the call. +* **per-block contribution** — `blockLengthContributions(m, spec)`: for each + `FeatureSpec` block, the literals it contributes across clauses + (`totalLits`, `meanPerClause`, `share`, `clausesUsing`, `perClauseMax`). A + block that contributes to every clause is dominating; a block contributing + ~0 is dead (the block-level view of the dead-INPUT list). +* **coverage / concentration** — `clauseCoverage(m, samples)`: the fraction of + clauses that fire on a sample (`meanFiringFraction`), the participation-ratio + `effectiveClauses = (sum v)^2 / sum v^2` (if 3 clauses cast most votes it + reads ~3, not the configured count) and `top3Share`. + +The one-line `healthLine` now appends `shape= ()` alongside the +settledness / diversity / churn / disagreement verdicts. + ## Task 5 — the AUTOMATA level (settledness / diversity / churn / disagreement) Task 2 reads the clauses. Task 5 reads the **automata inside them**: one @@ -146,10 +254,11 @@ So a flip is exactly a change in the predicate `state > 0`. `perInputConfidence(m, spec, samples, threshold)`, `rankedInputConfidence(conf)`. - **one-line health summary** — `healthLine(ad)` gives - `settledness / diversity / churn trend / disagreement`, each with its own - verdict word (`settling`, `fidgeting`, `frozen`, `coherent`, ...), and - `automataVerdict(ad)` reduces the trajectory to one word. `formatAutomataReport(ad)` - prints everything. + `settledness / diversity / churn trend / disagreement / shape`, each with its + own verdict word (`settling`, `fidgeting`, `frozen`, `coherent`, `healthy`, + `too long`, ...), and `automataVerdict(ad)` reduces the trajectory to one word. + `formatAutomataReport(ad)` prints everything, including the clause-shape report + when `ad.shape` was computed. ### API @@ -161,7 +270,7 @@ let ad = automataDiagnostics(tmpl, samples, spec, settleThreshold = 0.5, nHistBins = 9, window = 100, measureChurn = true) # ad.machine, ad.settledness, ad.diversity, ad.churn, ad.disagreement, -# ad.histogram, ad.inputConfidence, ad.summary +# ad.histogram, ad.inputConfidence, ad.shape, ad.summary echo ad.summary # one line echo automataVerdict(ad) # "settling" | "fidgeting" | "collapsed" | ... echo formatAutomataReport(ad) # the full readout @@ -226,6 +335,37 @@ on a `tm_core` temporal one-pass proxy, live-order): - constant inputs flagged: bits 38/39 (the known never-written ones) plus 19/36/37 in this fixture - context: pooled warm accuracy 35.72% vs the 34.24% majority = **+1.48pp** +**Clause-shape reading (group 8, same run):** `mean=19.17 median=16.00 p10=4.00 +p90=44.80 std=13.89 min=1 max=57 nonEmpty=173 empty=27/200` -> verdict +**`too long`**. Per polarity `posMean=19.80 / negMean=18.70`; per class +`c0=11.4 c1=13.4 c2=31.2 c3=22.6 c4=21.6`. Per-block share is diffuse (no block +dominates): wall-near 10.8%, distance-band 8.9%, flight-band 8.7%, radial-frac +7.6%, speed-band 7.0%, closing 6.8%, **UNUSED 6.4%** (the always-true negations of +the never-written bits 38/39 are free padding), the rest 2-6%. Coverage: +`firing/sample=48.53 (24.3%) effectiveClauses=97.85/200 top3Share=5.3%` — the +voting is NOT concentrated on a few clauses. One-line health: + +``` +settledness=0.484 (settling) | diversity=0.267 (moderate) | churn=0.094/100 falling (settling) | disagreement=0.145 (coherent) | shape=19.17 (too long) +``` + +**Does `s` rescue this gun? MEASURED, no.** Recompiling the offline driver with +`-d:TM_S_DEF=` (source untouched) retrains the gun end to end: + +| `s` | mean clause len | shape verdict | pooled warm acc | margin vs majority | +|---|---|---|---|---| +| 1.5 | 15.82 | too long | 32.22% | **-2.03pp** | +| 2.0 | 14.99 | too long | 34.03% | **-0.22pp** | +| 3.0 (shipped) | 19.17 | too long | 35.72% | **+1.48pp** | +| 5.0 | 20.47 | too long | 34.72% | **+0.47pp** | + +Lowering `s` shrinks the clauses (15.8 at `s=1.5`) but makes accuracy WORSE; +raising it pads them and also loses. The shipped `s=3.0` is the best of the four, +and NO value gets the mean anywhere near the healthy 3-8 band. Combined with the +settledness finding, the shape is consistent with **"no consistent short rule +exists in this representation/target"** — the clauses are long, diffuse and +padded, and the knob cannot fix a signal problem. + The gun settles onto the within-battle labels but its settled rules barely beat the majority class, which points at the TARGET / representation rather than the inertia `N`. See the Task 5 report for the inertia discussion. @@ -254,10 +394,12 @@ let m = machineFromTeams(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S, ```sh nim c -r -d:release --path:common_libs common_libs/tests/test_tm_diag.nim # 48 pure unit checks nim c -r --path:common_libs common_libs/tests/test_tm_automata_diag.nim # 55 automata-metric unit checks +nim c -r -d:release --path:common_libs common_libs/tests/test_tm_clause_shape.nim # 66 clause-shape unit + synthetic checks nim c -r -d:release --path:common_libs common_libs/tests/diag_synthetic.nim # Task 3 proof (17 checks) nim c -r -d:release --path:common_libs common_libs/tests/diag_automata_validation.nim # Task 5 A/B/C proof -nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim # Task 4 real reading +nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim # Task 4 real reading + shape ``` -See `common_libs/tests/diag_synthetic.nim` for the ground-truth validation and +See `common_libs/tests/diag_synthetic.nim` for the ground-truth validation, +`common_libs/tests/test_tm_clause_shape.nim` for the clause-shape validation and `common_libs/tests/diag_tm_pattern_offline.nim` for the real reading. diff --git a/common_libs/tm_diag/diagnostics.nim b/common_libs/tm_diag/diagnostics.nim index 9d9fe52..2516b69 100644 --- a/common_libs/tm_diag/diagnostics.nim +++ b/common_libs/tm_diag/diagnostics.nim @@ -1,7 +1,7 @@ ## tm_diag/diagnostics.nim — THE DIAGNOSTICS KIT (Tasks 2 + 5). ## ## Everything runs OFFLINE: a trained `TmMachine` plus a labelled sample set. -## No battles, no harness, no Java. The seven groups are: +## No battles, no harness, no Java. The eight groups are: ## 1. pre-flight DATA checks (dataChecks, shuffledLabelControl) ## 2. clause introspection (clauseInfo, clauseSummary, topClauses) ## 3. per-feature contribution (featureContributions, deadInputs, ...) @@ -12,6 +12,10 @@ ## voteDisagreement, stateHistogram, ## perInputConfidence, healthLine, ## automataDiagnostics) +## 8. CLAUSE SHAPE (Task 2/4) (clauseShapeVerdict, lengthStats, +## blockLengthContributions, clauseCoverage, +## clauseShapeDiagnostics, shapeLine, +## formatClauseShapeReport) ## ## A `DiagSample` carries the LITERAL vector (positive literals [0..nBits), ## negations [nBits..2*nBits)) exactly as the shipped TM stores it, plus the @@ -184,6 +188,68 @@ type lengthHist*: seq[int] ## index = clause length, value = count (non-empty) meanLength*: float maxLength*: int + ## ── clause-SHAPE distribution (Task 2, group 8) ── + minLength*: int ## shortest non-empty clause + medianLength*: float ## p50 of the non-empty length distribution + p10Length*: float + p90Length*: float + stdLength*: float + posMeanLength*: float ## positive-polarity clauses only + negMeanLength*: float ## negative-polarity clauses only + perClassMeanLength*: seq[float] + perClassMedianLength*: seq[float] + perClassHist*: seq[seq[int]] + emptyFraction*: float ## emptyClauses / totalClauses + +proc percentile*(sorted: openArray[int], p: float): float = + ## Linear-interpolation percentile (numpy default) over an ASCENDING array. + ## Empty -> 0.0; p is clamped to [0,1]. + if sorted.len == 0: return 0.0 + let pp = clamp(p, 0.0, 1.0) + let rank = pp * float(sorted.len - 1) + let lo = int(floor(rank)) + let hi = int(ceil(rank)) + if lo == hi: return sorted[lo].float + sorted[lo].float + (rank - float(lo)) * (sorted[hi] - sorted[lo]).float + +type + LengthStats* = object + n*: int + minL*: int + maxL*: int + mean*: float + median*: float + p10*: float + p90*: float + stdDev*: float + hist*: seq[int] + +proc lengthStats*(lengths: openArray[int]): LengthStats = + ## Full distribution of clause lengths: n, min/max, mean, median, p10/p90, + ## standard deviation and a histogram (index = length). + result.n = lengths.len + if result.n == 0: + result.minL = 0 + result.maxL = 0 + return + var sorted = newSeq[int](lengths.len) + for i, l in lengths: sorted[i] = l + sorted.sort() + result.minL = sorted[0] + result.maxL = sorted[^1] + var sum = 0 + for l in sorted: sum += l + result.mean = sum.float / result.n.float + result.median = percentile(sorted, 0.5) + result.p10 = percentile(sorted, 0.10) + result.p90 = percentile(sorted, 0.90) + var varSum = 0.0 + for l in sorted: + let d = l.float - result.mean + varSum += d * d + result.stdDev = sqrt(varSum / result.n.float) + result.hist = newSeq[int](result.maxL + 1) + for l in sorted: inc result.hist[l] proc clauseInfo*(m: TmMachine, samples: openArray[DiagSample], spec: FeatureSpec): seq[ClauseInfo] = @@ -200,14 +266,31 @@ proc clauseInfo*(m: TmMachine, samples: openArray[DiagSample], proc clauseSummary*(infos: openArray[ClauseInfo]): ClauseSummary = var maxLen = 0 var lenSum = 0 + var posSum = 0 + var posN = 0 + var negSum = 0 + var negN = 0 + var lengths: seq[int] + var posLengths, negLengths: seq[int] + var nClasses = 0 + for inf in infos: + if inf.cls + 1 > nClasses: nClasses = inf.cls + 1 + var classLengths = newSeq[seq[int]](nClasses) for inf in infos: inc result.totalClauses + if inf.cls >= 0 and inf.cls < nClasses: + if inf.length > 0: classLengths[inf.cls].add inf.length if inf.length == 0: inc result.emptyClauses else: inc result.nonEmpty lenSum += inf.length + lengths.add inf.length if inf.length > maxLen: maxLen = inf.length + if inf.polarity > 0: + posSum += inf.length; inc posN; posLengths.add inf.length + else: + negSum += inf.length; inc negN; negLengths.add inf.length if inf.votes > 0: inc result.firedAtLeastOnce if inf.polarity > 0: inc result.posFired @@ -220,6 +303,26 @@ proc clauseSummary*(infos: openArray[ClauseInfo]): ClauseSummary = if inf.length > 0: inc result.lengthHist[inf.length] result.meanLength = if result.nonEmpty > 0: lenSum.float / result.nonEmpty.float else: 0.0 + let stats = lengthStats(lengths) + result.minLength = stats.minL + result.medianLength = stats.median + result.p10Length = stats.p10 + result.p90Length = stats.p90 + result.stdLength = stats.stdDev + result.posMeanLength = if posN > 0: posSum.float / posN.float else: 0.0 + result.negMeanLength = if negN > 0: negSum.float / negN.float else: 0.0 + result.emptyFraction = + if result.totalClauses > 0: + result.emptyClauses.float / result.totalClauses.float + else: 0.0 + result.perClassMeanLength = newSeq[float](nClasses) + result.perClassMedianLength = newSeq[float](nClasses) + result.perClassHist = newSeq[seq[int]](nClasses) + for c in 0.. healthyHi -> too long + ## mean < collapsedMax -> collapsed + ## collapsedMax <= mean < healthyLo -> short (below band, not yet collapsed) + ## healthyLo <= mean <= healthyHi -> healthy + if summ.nonEmpty == 0: return "n/a" + let mean = summ.meanLength + if mean > healthyHi: "too long" + elif mean < collapsedMax: "collapsed" + elif mean < healthyLo: "short" + else: "healthy" + +proc lengthHistText*(hist: openArray[int]): string = + ## One line per length, e.g. `len 3: 12 ####`. + var mx = 0 + for c in hist: + if c > mx: mx = c + for l in 0..3}: {hist[l]:>6} {bar}\n" + +type + BlockLengthContribution* = object + name*: string + first*: int + count*: int + totalLits*: int ## literals from this block across the clauses + meanPerClause*: float ## totalLits / number of counted clauses + share*: float ## totalLits / all literals across counted clauses + clausesUsing*: int ## clauses with >= 1 literal from this block + perClauseMax*: int ## most literals this block contributes to one clause + +proc blockLengthContributions*(m: TmMachine, spec: FeatureSpec, + skipEmpty = true): seq[BlockLengthContribution] = + ## Per input BLOCK (walls / us / motion / bullets from the `FeatureSpec`): how + ## many literals does it contribute on average across clauses? + ## + ## A block that contributes many literals to EVERY clause is dominating the + ## model; a block contributing none is dead (the block-level view of the dead + ## INPUT list). `skipEmpty = true` (default) averages over non-empty clauses + ## only, so empty clauses do not dilute the shares. + result = newSeq[BlockLengthContribution](spec.blocks.len) + for i, b in spec.blocks: + result[i].name = b.name + result[i].first = b.first + result[i].count = b.count + var nClauses = 0 + var total = 0 + for c in 0..= 0: + inc perBlock[bi] + inc total + for i in 0.. 0: inc result[i].clausesUsing + if perBlock[i] > result[i].perClauseMax: + result[i].perClauseMax = perBlock[i] + for i in 0.. 0: result[i].totalLits.float / nClauses.float else: 0.0 + result[i].share = if total > 0: result[i].totalLits.float / total.float else: 0.0 + +type + ClauseCoverage* = object + samples*: int + totalClauses*: int + meanFiringClauses*: float ## mean non-empty firing clauses per sample + meanFiringFraction*: float ## ... / totalClauses + effectiveClauses*: float ## participation ratio of the vote counts + top3Share*: float ## share of all fires from the 3 busiest clauses + firingClauseCount*: int ## clauses that fire at least once + +proc clauseCoverage*(m: TmMachine, samples: openArray[DiagSample]): + ClauseCoverage = + ## How many clauses are actually needed to cover the cases? + ## + ## `meanFiringFraction` is the fraction of clauses that fire on any given + ## sample. `effectiveClauses` is the participation ratio + ## `(sum v)^2 / sum v^2` of the per-clause fire counts: if only 3 clauses cast + ## most of the votes, it reads ~3 regardless of the configured count. + result.totalClauses = m.nClasses * m.nClauses + result.samples = samples.len + if samples.len == 0 or result.totalClauses == 0: return + var counts = newSeq[int](result.totalClauses) + var fireSum = 0.0 + for s in samples: + var firing = 0 + for c in 0.. 0: inc result.firingClauseCount + totalVotes += v + sqVotes += v.float * v.float + if sqVotes > 0.0: + result.effectiveClauses = totalVotes.float * totalVotes.float / sqVotes + var sorted = counts + sorted.sort(SortOrder.Descending) + let k = min(3, sorted.len) + var top = 0 + for i in 0.. 0: top.float / totalVotes.float else: 0.0 + +type + ClauseShapeDiag* = object + summary*: ClauseSummary + verdict*: string + blocks*: seq[BlockLengthContribution] + coverage*: ClauseCoverage + healthyLo*: float + healthyHi*: float + collapsedMax*: float + +proc clauseShapeDiagnostics*(m: TmMachine, samples: openArray[DiagSample], + spec: FeatureSpec, healthyLo = 3.0, + healthyHi = 8.0, collapsedMax = 2.0, + skipEmpty = true): ClauseShapeDiag = + ## The one-call clause-shape readout: length distribution + healthy-band + ## verdict + per-block length contribution + coverage/concentration. + result.summary = clauseSummary(clauseInfo(m, samples, spec)) + result.healthyLo = healthyLo + result.healthyHi = healthyHi + result.collapsedMax = collapsedMax + result.verdict = clauseShapeVerdict(result.summary, healthyLo, healthyHi, + collapsedMax) + result.blocks = blockLengthContributions(m, spec, skipEmpty) + result.coverage = clauseCoverage(m, samples) + +proc shapeLine*(d: ClauseShapeDiag): string = + ## One-line clause-shape summary, in the same style as `healthLine`. + let s = d.summary + &"shape mean={s.meanLength:.2f} med={s.medianLength:.1f} " & + &"p10={s.p10Length:.1f} p90={s.p90Length:.1f} max={s.maxLength} " & + &"({d.verdict}) | coverage={d.coverage.meanFiringFraction*100:.1f}%/sample " & + &"eff={d.coverage.effectiveClauses:.1f}/{d.coverage.totalClauses} " & + &"top3={d.coverage.top3Share*100:.0f}%" + +proc formatClauseShapeReport*(d: ClauseShapeDiag): string = + ## Full clause-shape readout: distribution, per-polarity / per-class means, + ## per-block contribution and coverage. + let s = d.summary + result.add &"shape: mean={s.meanLength:.2f} median={s.medianLength:.2f} " & + &"p10={s.p10Length:.2f} p90={s.p90Length:.2f} std={s.stdLength:.2f} " & + &"min={s.minLength} max={s.maxLength} nonEmpty={s.nonEmpty} " & + &"empty={s.emptyClauses}/{s.totalClauses} " & + &"({d.verdict}, healthy band {d.healthyLo:.0f}-{d.healthyHi:.0f})\n" + result.add &" polarity: posMean={s.posMeanLength:.2f} negMean={s.negMeanLength:.2f}\n" + result.add " per-class mean/median:" + for c in 0.. 0: + result.add &" | shape={d.shape.summary.meanLength:.2f} ({d.shape.verdict})" proc automataDiagnostics*(tmpl: TmMachine, samples: openArray[DiagSample], spec: FeatureSpec, epochs = 1, seed = 777'u64, @@ -1092,6 +1395,7 @@ proc automataDiagnostics*(tmpl: TmMachine, samples: openArray[DiagSample], result.histogram = stateHistogram(result.machine, nHistBins) result.inputConfidence = perInputConfidence(result.machine, spec, samples, settleThreshold) + result.shape = clauseShapeDiagnostics(result.machine, samples, spec) result.summary = healthLine(result) proc automataVerdict*(d: AutomataDiag): string = @@ -1139,3 +1443,5 @@ proc formatAutomataReport*(d: AutomataDiag): string = for c in 0.. 0: + result.add formatClauseShapeReport(d.shape)