SirRoboGarage/common_libs/tests/diag_tm_pattern_offline.nim

## TASK 4 — the tm_diag kit against the EXISTING guns/tm_pattern.nim GF head.
##
## Replays the committed DrussGT fixtures through the offline gun range with a
## FRESH, diagCapture-enabled TM per fixture (the live semantics: cold every
## battle, overfit within the battle), then reports:
##   * pooled label balance + majority baseline vs the gun's own warm accuracy;
##   * the DEAD-INPUT LIST for the actual 40-bit tm_pattern feature set;
##   * the top firing positive clauses and the recovered necessary literals.
##
## Offline only. Fixtures are READ-ONLY.
## Run: nim c -r -d:release --path:common_libs common_libs/tests/diag_tm_pattern_offline.nim
##   [--fixtures=a,b,c] [--maxsamples=N]

import std/[os, strformat, strutils, algorithm, random]
import gun_harness/offline_range
import guns/tm_pattern
import tm_diag/diagnostics

const repoRoot = currentSourcePath().parentDir.parentDir.parentDir
const fixturesDir = repoRoot / "tools" / "fixtures"

proc toDiag(s: TmDiagSample): DiagSample =
  result.lits = newSeq[uint8](TM_NLITS)
  for i in 0..<TM_NLITS: result.lits[i] = s.lits[i]
  result.label = s.label
  result.order = s.order

proc printPooledConfusion(cm: array[TM_CLASSES, array[TM_CLASSES, int]]) =
  var total = 0
  var correct = 0
  var rowSum, colSum: array[TM_CLASSES, int]
  for c in 0..<TM_CLASSES:
    for p in 0..<TM_CLASSES:
      total += cm[c][p]
      if c == p: correct += cm[c][p]
      rowSum[c] += cm[c][p]
      colSum[p] += cm[c][p]
  var maj = 0
  for c in 1..<TM_CLASSES:
    if rowSum[c] > rowSum[maj]: maj = c
  let majShare = if total > 0: rowSum[maj].float / total.float else: 0.0
  let acc = if total > 0: correct.float / total.float else: 0.0
  echo &"# pooled warm accuracy = {correct}/{total} = {acc*100:.2f}%"
  echo &"# majority class = {maj}  share/baseline = {rowSum[maj]}/{total} = {majShare*100:.2f}%"
  echo &"# margin = {(acc-majShare)*100:+.2f}pp   pred-majority share = " &
       &"{colSum[maj].float/max(1,total).float*100:.2f}%"
  for c in 0..<TM_CLASSES:
    let rec = if rowSum[c] > 0: cm[c][c].float / rowSum[c].float else: 0.0
    let prec = if colSum[c] > 0: cm[c][c].float / colSum[c].float else: 0.0
    echo &"#   class{c}: trueN={rowSum[c]:<6} predN={colSum[c]:<6} TP={cm[c][c]:<6} " &
         &"recall={rec*100:5.1f}% precision={prec*100:5.1f}%"

proc main() =
  var names = @["drussgt_vs_crazy", "drussgt_vs_spinbot", "drussgt_vs_drussgt",
                "tr_drussgt_vs_crazy", "tr_drussgt_vs_spinbot",
                "tr_drussgt_vs_modularbot"]
  var maxSamples = 40000
  for i in 1..paramCount():
    let a = paramStr(i)
    if a.startsWith("--fixtures="):
      names = a[11..^1].split(',')
    elif a.startsWith("--maxsamples="):
      maxSamples = parseInt(a[13..^1])

  let spec = tmPatternSpec()
  echo &"# tm_pattern GF head over DrussGT fixtures  (spec nBits={spec.nBits}, " &
       &"TM_CLASSES={TM_CLASSES}, TM_NCLAUSES={TM_NCLAUSES})"
  echo "# fixture,samples,captured,classTotal,labelHist"

  var pooledLabels: seq[int]
  var pooledConfusion: array[TM_CLASSES, array[TM_CLASSES, int]]
  var pooledClassCorrect, pooledClassTotal = 0
  var pooledLabelHist: array[TM_CLASSES, int]
  var bestGun: TmPatternGun
  var bestSamples: seq[DiagSample]
  var bestName = ""
  var bestClassTotal = -1

  for name in names:
    let path = fixturesDir / (name & ".jsonl")
    if not fileExists(path):
      echo &"# SKIP missing fixture {path}"
      continue
    let fx = loadFixture(path)
    let g = new(TmPatternGun)
    g[] = initTmPatternGun()
    g[].targetMode = tmGF
    g[].diagCapture = true
    randomize(1234)
    let drv = GunDriver(
      name: "TMPatGF",
      predictCb: proc(state: WorldState, bs: float): GunPrediction = g[].predict(state, bs),
      resultCb: proc(e: FeedbackEvent) = g[].onResult(e),
      readyCb: proc(): bool = true)
    discard replayFixture(fx, @[drv], fx.enemyId)

    var hs: string
    for c in 0..<TM_CLASSES:
      inc pooledLabelHist[c], g[].labelHist[c]
      hs.add $g[].labelHist[c] & ","
      for p in 0..<TM_CLASSES:
        pooledConfusion[c][p] += g[].confusion[c][p]
    pooledClassCorrect += g[].classCorrect
    pooledClassTotal += g[].classTotal

    echo &"# {name},{fx.states.len},{g[].diagSamples.len},{g[].classTotal},[{hs}]"

    # Keep the fixture with the most warm labelled samples for clause
    # introspection (the model is fresh per fixture, so we cannot pool teams).
    var thisSamples: seq[DiagSample]
    let cap = min(g[].diagSamples.len, maxSamples)
    thisSamples = newSeq[DiagSample](cap)
    for i in 0..<cap: thisSamples[i] = toDiag(g[].diagSamples[i])
    if g[].classTotal > bestClassTotal and thisSamples.len > 0:
      bestClassTotal = g[].classTotal
      bestGun = g[]
      bestSamples = thisSamples
      bestName = name

  # ── pooled label balance + majority baseline vs real accuracy ──
  for c in 0..<TM_CLASSES:
    for _ in 0..<pooledLabelHist[c]: pooledLabels.add c
  let dc = dataChecks(pooledLabels, TM_CLASSES)
  echo "\n## POOLED label balance (all resolved bullets, warm+cold)"
  echo &"# n={dc.n} counts={dc.classCounts}"
  var sh = ""
  for c in 0..<TM_CLASSES: sh.add &"c{c}={dc.classShares[c]*100:.1f}% "
  echo "# shares: ", sh
  echo &"# majority = class{dc.majorityClass} @ {dc.majorityShare*100:.2f}%  overThreshold={dc.overThreshold}"
  for f in dc.flags: echo "# ", f

  echo "\n## POOLED accuracy (gun's own warm confusion)"
  echo &"# cross-check: gun classCorrect/classTotal = {pooledClassCorrect}/{pooledClassTotal}"
  printPooledConfusion(pooledConfusion)

  if bestSamples.len == 0:
    echo "\n# no captured samples; aborting introspection"
    return

  echo &"\n## CLAUSE / INPUT INTROSPECTION on fixture '{bestName}' " &
       &"(bestGun: {bestSamples.len} captured samples, {bestClassTotal} warm)"

  let m = machineFromTeams(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S,
                           bestGun.exportTeams())
  let infos = clauseInfo(m, bestSamples, spec)
  let summ = clauseSummary(infos)
  echo &"# clauses: total={summ.totalClauses} empty={summ.emptyClauses} " &
       &"nonEmpty={summ.nonEmpty} fired>=1={summ.firedAtLeastOnce} " &
       &"neverFired={summ.neverFired} posFired={summ.posFired} negFired={summ.negFired}"
  echo &"# clause length: mean={summ.meanLength:.2f} max={summ.maxLength} hist={summ.lengthHist}"
  for cb in clauseBalanceByClass(infos, TM_CLASSES):
    echo &"#   class{cb.cls}: nonEmpty={cb.nonEmpty} empty={cb.empty} " &
         &"posFired={cb.posFired} negFired={cb.negFired} neverFired={cb.neverFired}"

  let contribs = featureContributions(m, bestSamples, spec)
  let ranked = rankedInputs(contribs)
  let dead = deadInputs(contribs)
  let never = neverUsedInputs(contribs)
  let consts = constantInputs(bestSamples, TM_NBITS)
  echo "\n## INPUT VALUE RANKING (top 15 by weighted vote-share)"
  for i in 0..<min(15, ranked.len):
    echo &"#   {ranked[i].bit:>2} {ranked[i].name:<26} appearances={ranked[i].appearances:<8} weighted={ranked[i].weighted:.1f}"
  echo "\n## DEAD-INPUT LIST (weighted < 5% of top, or never in a voting clause):"
  echo "#   ", dead
  echo "## never-used (strict appearances==0): ", never
  echo "## constant inputs (zero variance in this fixture): ", consts

  echo "\n## TOP FIRING POSITIVE CLAUSES per class"
  for cls in 0..<TM_CLASSES:
    var clsInfos: seq[ClauseInfo]
    for c in infos:
      if c.cls == cls: clsInfos.add c
    let tc = topClausesByPolarity(clsInfos, 1, 4)
    if tc.len == 0:
      echo &"#   class{cls}: (no firing positive clauses)"
    for c in tc:
      echo &"#   class{cls} votes={c.votes:<7} len={c.length}  {c.text}"
    echo &"#     -> necessary literals: {spec.describeClause(necessaryLiterals(m, bestSamples, cls), cls)}"

  # ── AUTOMATA-LEVEL metrics on the shipped GF head (Task 5) ──
  # settledness / diversity / histogram / per-input confidence / disagreement
  # are read DIRECTLY off the exported teams; churn is measured on a tm_core
  # retrain (same algorithm) over the captured samples, because the live gun
  # does not expose a per-sample state trace.
  var ad = AutomataDiag()
  ad.machine = m
  ad.settledness = settledness(m, 0.5)
  ad.diversity = clauseDiversity(m)
  ad.histogram = stateHistogram(m, 9)
  ad.inputConfidence = perInputConfidence(m, spec, bestSamples, 0.5)
  ad.disagreement = voteDisagreement(m, bestSamples)
  let churnN = min(bestSamples.len, 10000)
  var churnSamples = newSeq[DiagSample](churnN)
  for i in 0..<churnN: churnSamples[i] = bestSamples[i]
  let tmpl = newMachine(TM_NBITS, TM_CLASSES, TM_NCLAUSES, TM_NSTATES, TM_S,
                        seed = 1)
  # Mirror the live gun: one TEMPORAL pass (no shuffle) as bullets resolve.
  ad.churn = churnTrace(tmpl, churnSamples, epochs = 1, seed = 777,
                        window = 200, shuffle = false)
  ad.summary = healthLine(ad)
  echo "\n## AUTOMATA-LEVEL metrics (shipped tm_pattern GF head)"
  echo "# churn measured on a tm_core retrain over ", churnN,
       " captured samples, ONE temporal pass (live-order proxy)"
  echo formatAutomataReport(ad)
  echo "# VERDICT: ", automataVerdict(ad)

when isMainModule:
  main()