## Adaptation-speed sweep harness for the Tsetlin gun. ## ## Measures how quickly the TM becomes USEFUL within a round, starting from a ## cold net (no persistence). For every fixture round it records every resolved ## virtual bullet by its resolution tick RELATIVE to the round start, and reports ## the virtual hit rate over: ## * the first 100 ticks of each round, ## * the first 300 ticks of each round, ## * the whole round (for context). ## Clause sparsity (mean included literals of active clauses) is reported too: ## a config that never fires looks identical to a config that fires badly in the ## hit-rate columns. ## ## Tsetlin is stochastic (tmLearnOne calls rand()), so each fixture is replayed ## over multiple seeds and results are pooled across seeds AND rounds. ## ## The compiled config is printed from the exported `TM_*` constants, so the ## harness always labels what was actually built. ## ## Usage: ## nim c -r --path:common_libs -d:release common_libs/tests/sweep_tsetlin.nim \ ## --set=real --seeds=3 ## Flags: ## --set=synthetic|real|range fixture set (default synthetic) ## --seeds=N independent TM RNG seeds, pooled (default 3) ## --maxrounds=N cap the rounds replayed per fixture ## --linear replay the deterministic Linear gun instead ## --shuffle randomised-feedback control: identical TM, but ## every training target is a random arena point ## --point use the bmPoint metric instead of the shipped ## bmPath ## ## The TM hyper-parameters are compile-time constants; override them without ## editing the gun, e.g. ## nim c -r ... -d:TM_WINDOW_SIZE=3 -d:TM_S_DEF=1.1 -d:TM_N_STATES=100 \ ## -d:TM_N_CLAUSES=100 -d:TM_T_DEF=50 sweep_tsetlin.nim --set=real ## ## Emits CSV to stdout (one row per fixture×seed) plus SUM and # summary lines. import std/[os, strformat, strutils, json, tables, math, random] import gun_harness/offline_range import range_guns import guns/tsetlin import guns/linear const repoRoot = currentSourcePath().parentDir.parentDir.parentDir const fixturesDir = repoRoot / "tools" / "fixtures" type AdaptResult = object h100, n100: int ## resolved within the first 100 ticks of the round h300, n300: int ## resolved within the first 300 ticks of the round hall, nall: int ## all resolutions in the round f100, m100: int ## FIRED within the first 100 ticks (robustness check) f300, m300: int ## FIRED within the first 300 ticks rounds: int RoundSpan = tuple[start, count: int] proc configLabel(): string = &"cl{TM_N_CLAUSES}_st{TM_N_STATES}_s{TM_S}_T{TM_T}_w{TM_WINDOW_SIZE}" proc loadRounds(path: string): seq[RoundSpan] = ## The round-boundary sidecar lives next to the classic/tr-bridge fixtures in ## a `drussgt_meta/` subdirectory. Synthetic fixtures have none -> one round. let dir = path.parentDir let base = path.extractFilename var side = dir / "drussgt_meta" / (base & ".rounds.json") if not fileExists(side): side = dir / (base & ".rounds.json") if not fileExists(side): return @[] let node = parseJson(readFile(side)) if not node.hasKey("rounds"): return @[] for r in node["rounds"]: result.add (r["startTick"].getInt(), r["count"].getInt()) proc addAdapt(dst: var AdaptResult, src: AdaptResult) = inc dst.rounds, src.rounds dst.h100 += src.h100; dst.n100 += src.n100 dst.h300 += src.h300; dst.n300 += src.n300 dst.hall += src.hall; dst.nall += src.nall dst.f100 += src.f100; dst.m100 += src.m100 dst.f300 += src.f300; dst.m300 += src.m300 proc replayAdaptRound(states: seq[WorldState], lastSeen: seq[int], enemyId, baseTick: int, driver: GunDriver, metric: BulletMetric): AdaptResult = ## One round, cold net, mirroring `replayFixture`'s tick order (spawn all 4 ## power bins, then `tickBullets` with the current state). Resolutions are ## bucketed by `localTick = state.tick - baseTick`. var tracker = initTracker(1, metric) var res: AdaptResult inc res.rounds for si in 0..= 0: lst = lastSeen[si] if state.enemies.len > 0: for e in state.enemies: enemyPositions[e.id] = (x: e.x, y: e.y, lastSeenTick: lst, alive: true) else: enemyPositions[enemyId] = (x: state.enemyX, y: state.enemyY, lastSeenTick: lst, alive: true) let localTick = state.tick - baseTick tracker.tickBullets(state, enemyPositions, proc(gunId: GunId, binIdx: int, e: FeedbackEvent) = inc res.nall if e.hit: inc res.hall if localTick < 100: inc res.n100 if e.hit: inc res.h100 if localTick < 300: inc res.n300 if e.hit: inc res.h300 let fireTick = e.fireTick - baseTick if fireTick < 100: inc res.m100 if e.hit: inc res.f100 if fireTick < 300: inc res.m300 if e.hit: inc res.f300 driver.resultCb(e)) res proc replayAdaptFixture(fx: Fixture, path: string, driver: GunDriver, metric: BulletMetric, maxRounds = 0): AdaptResult = ## Split a fixture into its rounds (one cold replay each) and pool. var spans = if fx.meta.source == "synthetic": @[(start: 0, count: fx.states.len)] else: loadRounds(path) if maxRounds > 0 and spans.len > maxRounds: spans.setLen(maxRounds) if spans.len == 0: return replayAdaptRound(fx.states, fx.lastSeen, fx.enemyId, 0, driver, metric) for sp in spans: var st: seq[WorldState] var ls: seq[int] for i in 0..= sp.start and t < sp.start + sp.count: st.add fx.states[i] ls.add(if i < fx.lastSeen.len: fx.lastSeen[i] else: -1) if st.len == 0: continue addAdapt(result, replayAdaptRound(st, ls, fx.enemyId, sp.start, driver, metric)) proc rate(h, n: int): string = if n == 0: " n/a " else: &"{h.float / n.float * 100.0:5.1f}%" proc fixtureSet(name: string): seq[string] = case name of "synthetic", "": for n in SyntheticFixtureNames: result.add n of "real": for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "drussgt_vs_drussgt", "tr_drussgt_vs_crazy", "tr_drussgt_vs_spinbot", "tr_drussgt_vs_modularbot"]: result.add(fixturesDir / (n & ".jsonl")) of "range": for n in SyntheticFixtureNames: result.add n for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "tr_drussgt_vs_crazy"]: result.add(fixturesDir / (n & ".jsonl")) else: discard proc resolveFixture(name: string): tuple[fx: Fixture, path: string] = let p = if fileExists(name): name else: fixturesDir / (name & ".jsonl") (loadFixture(p), p) proc makeShuffledTsetlinDriver(seed = -1): tuple[driver: GunDriver, gun: ref TsetlinGun] = ## Randomised-feedback CONTROL. Identical TM, identical predict/onResult call ## cadence and encoding, but every training target is a uniform random point in ## a 1000x1000 box instead of the real enemy position. If the early hit rate ## of this control matches the real-feedback TM, then the measured skill is not ## coming from informative feedback (it is luck / the linear baseline / the ## always-nonzero correction). let g = new(TsetlinGun) g[] = initTsetlinGun() if seed >= 0: randomize(seed) result.gun = g result.driver = GunDriver( name: "ShuffledTM", predictCb: proc(state: WorldState, bulletSpeed: float): GunPrediction = g[].predict(state, bulletSpeed), resultCb: proc(e: FeedbackEvent) = var e2 = e e2.actualX = rand(1000.0) e2.actualY = rand(1000.0) g[].onResult(e2), readyCb: proc(): bool = g[].isWarmedUp(), ) proc main() = var nSeeds = 3 var set = "synthetic" var maxRounds = 0 var useLinear = false var usePoint = false var useShuffle = false for i in 1..paramCount(): let a = paramStr(i) if a.startsWith("--seeds="): nSeeds = parseInt(a[8..^1]) elif a.startsWith("--set="): set = a[6..^1] elif a.startsWith("--maxrounds="): maxRounds = parseInt(a[12..^1]) elif a == "--linear": useLinear = true elif a == "--point": usePoint = true elif a == "--shuffle": useShuffle = true let metric = if usePoint: bmPoint else: bmPath let names = fixtureSet(set) if names.len == 0: echo "no fixtures for set: ", set quit(1) let label = if useLinear: "Linear" elif useShuffle: "ShuffledTM" else: configLabel() echo "# config ", label, " seeds=", nSeeds, " set=", set echo "config,fixture,seed,h100,n100,h300,n300,hall,nall,meanInc,meanIncAll,nActive,nClauses" for name in names: let (fx, path) = resolveFixture(name) let fxName = path.extractFilename.replace(".jsonl", "") var pooled: AdaptResult var meanIncSum = 0.0 var meanIncAllSum = 0.0 var activeSum = 0 var nClauses = 0 let nIter = if useLinear: 1 else: nSeeds for seed in 1..nIter: var drv: GunDriver var gun: ref TsetlinGun if useLinear: drv = makeDriver("Linear", LinearGun()) elif useShuffle: let pair = makeShuffledTsetlinDriver(seed = seed) drv = pair.driver gun = pair.gun else: let pair = makeTsetlinDriver(seed = seed) drv = pair.driver gun = pair.gun let r = replayAdaptFixture(fx, path, drv, metric, maxRounds) var meanInc = -1.0 var meanIncAll = -1.0 var nActive = -1 if not useLinear: let st = gun[].tmClauseStats() meanInc = st.meanIncluded meanIncAll = st.meanIncludedAll nActive = st.nActive nClauses = st.nClauses echo &"{label},{fxName},{seed},{r.h100},{r.n100},{r.h300},{r.n300},{r.hall},{r.nall}," & &"{meanInc:.1f},{meanIncAll:.1f},{nActive},{nClauses}" addAdapt(pooled, r) meanIncSum += meanInc meanIncAllSum += meanIncAll activeSum += nActive echo &"SUM,{label},{fxName},{pooled.h100},{pooled.n100},{pooled.h300},{pooled.n300}," & &"{pooled.hall},{pooled.nall},{pooled.f100},{pooled.m100},{pooled.f300},{pooled.m300}," & &"{meanIncSum / nIter.float:.1f},{meanIncAllSum / nIter.float:.1f}," & &"{activeSum.float / nIter.float:.0f},{nClauses}" echo &"# {label} {fxName}: first100 {rate(pooled.h100, pooled.n100)} " & &"first300 {rate(pooled.h300, pooled.n300)} whole {rate(pooled.hall, pooled.nall)} " & &"({pooled.hall}/{pooled.nall}, rounds={pooled.rounds}) " & &"sparsity={meanIncSum / nIter.float:.1f} meanAll={meanIncAllSum / nIter.float:.1f} " & &"active={activeSum.float / nIter.float:.0f}/{nClauses}" when isMainModule: main()