## OFFLINE re-adaptation measurement for the horizon TM (tm_horizon). ## ## The user's live observation: "once the pattern is learnt enough we hit ## DrussGT, but as soon as it adapts we are not fast enough to re-adapt". This ## harness makes that failure mode measurable WITHOUT a Java battle, using the ## committed DrussGT fixtures (READ-ONLY) and the same fact-label pipeline as ## `measure_tm_miss_shrink.nim` / `measure_tm_hit_optimal.nim`. ## ## It answers three questions: ## 1. Does the rolling SIDE accuracy decay as the battle progresses? ## 2. Does a candidate fix (sliding window / change-detection re-learn / ## lower inertia) improve the LATE half, where re-adaptation matters? ## 3. Do the fixes only "help" with SHUFFLED labels? (mandatory control) ## ## Metrics are PREQUENTIAL: at each streamed sample the model predicts the ## side label BEFORE it is updated with it (exactly what the live gun's ## `warm` accuracy counters record). Early = first half of the eval stream, ## Late = second half. Decay = Late - Early (negative = it is losing the enemy). ## ## Protocols: ## * within — train on the first `warmFrac` of a round, then stream the rest ## of the SAME round. The nearest no-Java proxy for "train early, ## adapt later". On real open-loop fixtures there may be little ## enemy change, so a flat curve here is a NEGATIVE result to report. ## * cross — warm on round R, then stream round R+1. A real distribution ## shift (new start geometry). Directly tests "the enemy changed, ## re-adapt now", which is what the retain-across-rounds gun faces. ## ## Run: nim c -r -d:release --path:common_libs \ ## common_libs/tests/measure_tm_readapt.nim ## [--fixtures=a,b] [--window=150] [--resetdrop=5.0] [--warmfrac=0.4] ## [--retrain=50] [--clauses=40] [--states=64] [--epochs=5] [--protocol=both] import std/[json, os, strformat, strutils, math, tables] import tm_diag/tm_core # ── configuration ──────────────────────────────────────────────────────────── const repoRoot* = currentSourcePath().parentDir.parentDir.parentDir const fixturesDir* = repoRoot / "tools" / "fixtures" const metaDir* = fixturesDir / "drussgt_meta" const HORIZONS = [15, 20, 25, 30] const NH = 4 const N_BASE = 49 const N_BITS = N_BASE + 4 const MIN_I = 12 const NRBIN = 10 ## relative-eval-position bins for the curve const ACC_CAP = 512 var cfgClauses = 40 cfgStates = 64 cfgS = 3.0 cfgWarmEpochs = 5 cfgWindow = 150 cfgResetDrop = 5.0 cfgRetrainEvery = 50 cfgWarmFrac = 0.40 cfgResetWindow = 200 const PRIMARY = ["tr_drussgt_vs_modularbot.jsonl", "tr_drussgt_vs_modularbot_shield.jsonl"] # ── small helpers ──────────────────────────────────────────────────────────── proc wrap180(a: float): float {.inline.} = var r = a while r > 180.0: r -= 360.0 while r <= -180.0: r += 360.0 r proc signf(x: float): int {.inline.} = if x > 1e-9: 1 elif x < -1e-9: -1 else: 0 proc jf(d: JsonNode, k: string): float = let n = d[k] case n.kind of JFloat: n.getFloat of JInt: float(n.getInt) else: parseFloat(n.getStr) proc pct(c, t: int): float = if t <= 0: return NaN 100.0 * float(c) / float(t) # ── data model ─────────────────────────────────────────────────────────────── type Tick* = object tick*: int ex*, ey*, eh*, es*, ee*: float sx*, sy*, sh*, ss*, se*: float Rnd* = object roundNo*: int st*: seq[Tick] BulletSeries* = object tta*: seq[int] lat*: seq[float] Sample = object lits: seq[uint8] side: int ArmKind = enum akFrozen, akAccum, akWindow, akResetDrop, akRehearse ArmState = object m: TmMachine caches: seq[seq[uint8]] predCache: seq[uint8] buf: seq[seq[uint8]] buflab: seq[int] bufCap, bufCount: int sinceRetrain: int accRing: array[ACC_CAP, uint8] accPos, accCount: int accPeak: float sinceResetDrop: int resets: int ArmResult = object earlyCor, earlyTot, lateCor, lateTot, totCor, totTot: int binCor, binTot: array[NRBIN, int] resets: int rollVals: seq[float] ## fine rolling curve (this run only) const ArmNames: array[ArmKind, string] = ["frozen", "accum", "window", "resetdrop", "rehearse-all"] # ── fixture loading ────────────────────────────────────────────────────────── proc loadTicks(path: string): seq[Tick] = for line in lines(path): let ln = line.strip() if ln.len == 0: continue let d = parseJson(ln) if not d.hasKey("tick"): continue result.add Tick(tick: d["tick"].getInt, ex: jf(d, "ex"), ey: jf(d, "ey"), eh: jf(d, "eh"), es: jf(d, "es"), ee: jf(d, "ee"), sx: jf(d, "sx"), sy: jf(d, "sy"), sh: jf(d, "sh"), ss: jf(d, "ss"), se: jf(d, "se")) proc loadRounds(path: string, ticks: seq[Tick]): seq[Rnd] = let rp = metaDir / (extractFilename(path) & ".rounds.json") var spans: seq[(int, int)] if fileExists(rp): let j = parseFile(rp) for r in j["rounds"]: spans.add (r["startTick"].getInt, r["count"].getInt) elif ticks.len > 0: spans.add (ticks[0].tick, ticks.len) var idxByTick = initTable[int, int]() for i, t in ticks: idxByTick[t.tick] = i for sp in spans: let (s0, c) = sp if not idxByTick.hasKey(s0): continue let i0 = idxByTick[s0] var st: seq[Tick] for k in 0.. 0: result.add Rnd(roundNo: result.len + 1, st: st) proc buildBulletSeries(r: Rnd): BulletSeries = let L = r.st.len result.tta = newSeq[int](L) for i in 0.. 3.1: continue var power = drop if power < 0.1: power = 0.1 if power > 3.0: power = 3.0 let speed = 20.0 - 3.0 * power let rng = hypot(r.st[t0].ex - r.st[t0].sx, r.st[t0].ey - r.st[t0].sy) let flight = int(ceil(rng / speed)) let dx = r.st[t0].ex - r.st[t0].sx let dy = r.st[t0].ey - r.st[t0].sy let nrm = max(1e-6, hypot(dx, dy)) let ux = dx / nrm let uy = dy / nrm for k in 0..flight: let t = t0 + k if t >= L: break let ta = t0 + flight - t if result.tta[t] < 0 or ta < result.tta[t]: result.tta[t] = ta let vx = r.st[t].ex - r.st[t0].sx let vy = r.st[t].ey - r.st[t0].sy result.lat[t] = ux * vy - uy * vx # ── 49 draft bits (causal, at tick i) — identical to the measure_tm_* pipeline ─ proc buildBase(r: Rnd, bi: BulletSeries, i: int, sinceRev: seq[int]): array[N_BASE, int] = let s = r.st let cur = s[i] let dL = cur.ex let dR = 800.0 - cur.ex let dT = 600.0 - cur.ey let dBottom = cur.ey let dmin = min(min(dL, dR), min(dT, dBottom)) var wallBin = 3 if dmin < 50.0: wallBin = 0 elif dmin < 100.0: wallBin = 1 elif dmin < 200.0: wallBin = 2 result[wallBin] = 1 var wb = 0 let walls = [dL, dR, dT, dBottom] for w in 1..3: if walls[w] < walls[wb]: wb = w result[4 + wb] = 1 let rng = hypot(cur.ex - cur.sx, cur.ey - cur.sy) var ub = 5 if rng < 100.0: ub = 0 elif rng < 200.0: ub = 1 elif rng < 300.0: ub = 2 elif rng < 400.0: ub = 3 elif rng < 600.0: ub = 4 result[8 + ub] = 1 let lane = arctan2(cur.sy - cur.ey, cur.sx - cur.ex) let hdg = cur.eh * PI / 180.0 let perp = abs(sin(hdg - lane)) var hb = 1 if perp < 0.5: hb = 2 elif perp > 0.866: hb = 0 result[14 + hb] = 1 for k in 0..2: if i - 1 - k >= 0: let d = wrap180(s[i - k].eh - s[i - 1 - k].eh) if d > 1e-6: result[17 + k] = 1 var rb = 4 let sr = sinceRev[i] if sr < 5: rb = 0 elif sr < 10: rb = 1 elif sr < 20: rb = 2 elif sr < 40: rb = 3 result[20 + rb] = 1 var pos = 0 var neg = 0 for k in 0..9: if i - 1 - k < 0: break let d = wrap180(s[i - k].eh - s[i - 1 - k].eh) if d > 1e-6: inc pos elif d < -1e-6: inc neg let tot = pos + neg let cons = if tot > 0: max(pos, neg).float / tot.float else: 0.0 var cb = 0 if cons > 0.8: cb = 2 elif cons >= 0.5: cb = 1 result[25 + cb] = 1 let j0 = max(0, i - 10) let dm = hypot(cur.ex - s[j0].ex, cur.ey - s[j0].ey) var mb = 1 if dm < 20.0: mb = 0 elif dm > 50.0: mb = 2 result[28 + mb] = 1 let st10 = abs(s[max(0, i - 10)].es) let spdDiff = abs(cur.es) - st10 var sb = 1 if spdDiff < -0.5: sb = 0 elif spdDiff > 0.5: sb = 2 result[31 + sb] = 1 var r1 = 0.0 var n1 = 0 for k in 0..4: if i - 1 - k >= 0: r1 += abs(wrap180(s[i - k].eh - s[i - 1 - k].eh)); inc n1 var r2 = 0.0 var n2 = 0 for k in 5..9: if i - 1 - k >= 0: r2 += abs(wrap180(s[i - k].eh - s[i - 1 - k].eh)); inc n2 let m1 = if n1 > 0: r1 / float(n1) else: 0.0 let m2 = if n2 > 0: r2 / float(n2) else: 0.0 let dtr = m1 - m2 var tb = 1 if dtr < -0.3: tb = 0 elif dtr > 0.3: tb = 2 result[34 + tb] = 1 let tta = bi.tta[i] var b1 = 0 if tta >= 0: if tta < 5: b1 = 1 elif tta < 10: b1 = 2 elif tta < 20: b1 = 3 else: b1 = 4 result[37 + b1] = 1 let lat = bi.lat[i] var lb = 3 if lat < -72.0: lb = 0 elif lat < -36.0: lb = 1 elif lat < -18.0: lb = 2 elif lat <= 18.0: lb = 3 elif lat <= 36.0: lb = 4 elif lat <= 72.0: lb = 5 else: lb = 6 result[42 + lb] = 1 proc toLits(base: array[N_BASE, int], h: int): seq[uint8] = var raw: array[N_BITS, int] for i in 0..= L: continue let cur = s[i] let gx = cur.ex + cur.es * cos(cur.eh * PI / 180.0) * float(h) let gy = cur.ey + cur.es * sin(cur.eh * PI / 180.0) * float(h) let ba = arctan2(s[j].ey - cur.sy, s[j].ex - cur.sx) let bg = arctan2(gy - cur.sy, gx - cur.sx) let err = radToDeg(arctan2(sin(ba - bg), cos(ba - bg))) if abs(err) < 1e-9: continue result.add Sample(lits: toLits(base[i], h), side: (if err > 0.0: 1 else: 0)) proc buildRoundSamples(r: Rnd): seq[seq[Sample]] = ## samples[hidx] for one round. let bi = buildBulletSeries(r) let L = r.st.len var base = newSeq[array[N_BASE, int]](L) let sr = sinceRevSeries(r) for i in 0.. v0: 1 else: 0 proc newArm(cap: int): ArmState = result.m = newMachine(N_BITS, 2, cfgClauses, cfgStates, cfgS, seed = 12345) result.caches = newSeq[seq[uint8]](2) for c in 0..1: result.caches[c] = newSeq[uint8](cfgClauses) result.predCache = newSeq[uint8](cfgClauses) result.bufCap = max(1, cap) result.buf = newSeq[seq[uint8]](result.bufCap) result.buflab = newSeq[int](result.bufCap) proc trainEpochs(arm: var ArmState, litsList: seq[seq[uint8]], lab: seq[int], lo, hi, epochs: int) = if hi <= lo: return var order = newSeq[int](hi - lo) for e in 0.. arm.accPeak: arm.accPeak = a100 inc arm.sinceResetDrop if resetDrop > 0.0 and arm.accCount >= 100 and (arm.accPeak - a100) > resetDrop and arm.sinceResetDrop >= 100: arm.accPeak = a100 arm.sinceResetDrop = 0 inc arm.resets return true false proc runArm(kind: ArmKind, warmLits: seq[seq[uint8]], warmLab: seq[int], streamLits: seq[seq[uint8]], streamLab: seq[int], emitCurve: bool): ArmResult = let cap = case kind of akRehearse: 1400 of akWindow: max(1, cfgWindow) of akResetDrop: max(cfgWindow, cfgResetWindow) else: 1 var arm = newArm(cap) # ── warmup ── case kind of akWindow: let lo = max(0, warmLits.len - cfgWindow) for i in lo.. 0: min(NRBIN - 1, (k * NRBIN) div n) else: 0 inc result.binTot[bi] if correct: inc result.binCor[bi] if kind != akFrozen: if trigger and kind == akResetDrop: let rn = if cfgWindow > 0: cfgWindow else: cfgResetWindow arm.rebuild(rn) trainOne(arm.m, lits, lab, arm.caches) arm.pushBuffer(lits, lab) if kind == akWindow: inc arm.sinceRetrain if arm.sinceRetrain >= cfgRetrainEvery: arm.sinceRetrain = 0 arm.rebuild(cfgWindow) elif kind == akRehearse: inc arm.sinceRetrain if arm.sinceRetrain >= cfgRetrainEvery: arm.sinceRetrain = 0 arm.rebuild(arm.bufCount) # ALL buffered samples: no forgetting if emitCurve and arm.accCount >= 100 and arm.accCount mod 50 == 0: result.rollVals.add armRolling(arm, 100) * 100.0 result.resets = arm.resets proc addRes(dst: var ArmResult, src: ArmResult) = dst.earlyCor += src.earlyCor; dst.earlyTot += src.earlyTot dst.lateCor += src.lateCor; dst.lateTot += src.lateTot dst.totCor += src.totCor; dst.totTot += src.totTot dst.resets += src.resets for b in 0..5.0f}" echo &"{HORIZONS[h]:>3} {name:<16} {r.earlyTot + r.lateTot:>6} " & &"{early:>7.1f} {late:>7.1f} {late - early:>+7.1f} {tot:>7.1f} [{curve}] resets={r.resets}" # ── main ───────────────────────────────────────────────────────────────────── proc main() = var names = @PRIMARY for i in 1..paramCount(): let a = paramStr(i) if a.startsWith("--fixtures="): names = a[11..^1].split(',') elif a.startsWith("--window="): cfgWindow = parseInt(a[9..^1]) elif a.startsWith("--resetdrop="): cfgResetDrop = parseFloat(a[12..^1]) elif a.startsWith("--retrain="): cfgRetrainEvery = parseInt(a[10..^1]) elif a.startsWith("--warmfrac="): cfgWarmFrac = parseFloat(a[11..^1]) elif a.startsWith("--clauses="): cfgClauses = parseInt(a[10..^1]) elif a.startsWith("--states="): cfgStates = parseInt(a[9..^1]) elif a.startsWith("--epochs="): cfgWarmEpochs = parseInt(a[9..^1]) elif a.startsWith("--resetwindow="): cfgResetWindow = parseInt(a[14..^1]) echo "=" .repeat(100) echo "TM HORIZON RE-ADAPTATION (offline, prequential side accuracy)" echo "=" .repeat(100) echo &"fixtures : {names.join(\", \")}" echo &"TM : {cfgClauses} clauses, {cfgStates} states, s={cfgS}, warmEpochs={cfgWarmEpochs}" echo &"arms : frozen(none) | accum(keep all) | window(N={cfgWindow}, retrain {cfgRetrainEvery}) | " & &"resetdrop(drop<{cfgResetDrop:.1f}pp -> retrain last {cfgWindow})" echo &"protocol : within-round warmFrac={cfgWarmFrac:.2f}; cross-round (warm R, stream R+1)" echo &"metric : prequential; early=1st half of eval, late=2nd half; decay=late-early" echo &"control : labels shuffled within each set (mandatory integrity check)" echo "=" .repeat(100) for name in names: let path = if name.endsWith(".jsonl"): fixturesDir / name else: fixturesDir / (name & ".jsonl") if not fileExists(path): echo &"# SKIP missing fixture {path}" continue let ticks = loadTicks(path) let rounds = loadRounds(path, ticks) var rs: seq[seq[seq[Sample]]] for r in rounds: rs.add buildRoundSamples(r) echo &"\n## FIXTURE {name}: {rounds.len} rounds" for protocol in ["within", "cross"]: var pooled: array[ArmKind, ArmResult] var pooledShuf: array[ArmKind, ArmResult] var examples: seq[ArmResult] var exH = 0 for hidx in 0..= n: continue var warmLits: seq[seq[uint8]] var warmLab: seq[int] var streamLits: seq[seq[uint8]] var streamLab: seq[int] for i in 0..= n: continue var wl: seq[seq[uint8]] var wla: seq[int] var sl: seq[seq[uint8]] var sla: seq[int] for i in 0..= n: continue var wl: seq[seq[uint8]] var wla: seq[int] var sl: seq[seq[uint8]] var sla: seq[int] for i in 0..5} {ArmNames[kind]:<12} {e:>6.1f}/{l:>6.1f}" cfgStates = 64 echo "\n" & "=".repeat(100) echo "## INTEGRITY: shuffled-label controls should hover at chance (~50%)" echo " If a fix only improves the shuffled rows, it is NOT learning the enemy." echo " rehearse-all = same periodic full retrain as window but WITHOUT forgetting" echo " (isolates the effect of the sliding buffer from the effect of retraining)." echo &"## VERDICT INPUTS: late-half accuracy, TRUE vs shuffled (pp)" echo "=" .repeat(100) when isMainModule: main()