Tsetlin gun: NO configuration adapts faster than random feedback
The user's goal was "a TM gun that can learn fast and generalize better". Swept offline over the real DrussGT fixtures (no live battles) by coordinate descent, one lever at a time, with a SHUFFLED-FEEDBACK CONTROL - a TM trained on randomised targets. That control is what settles the question. Final confirmation, 4 seeds each (~74,600 first-100-tick bullets per config): config EARLY(first 100) OVERALL Shuf_w3 (RANDOM feedback) 23.9% 20.0% win3_s1.1 (best real TM found) 23.7% 20.2% Shuf_w10 (RANDOM feedback) 23.1% 20.0% win3_st100 (prior job's edit) 23.0% 20.1% win3_off (TM correction ~= 0) 22.7% 20.2% def_w10 (shipped default) 22.1% 20.3% Linear (deterministic reference) 34.0% 24.3% The best real config beats the default early (23.7% vs 22.1%, non-overlapping per-seed ranges, z=+7.34, p=2e-13) - but its own SHUFFLED control scores 23.9%, i.e. HIGHER, z=-0.91, p=0.37. Random targets do at least as well. So the early gain is not learning. Per-lever screens were flat: TM_N_CLAUSES 25/50/100/200 all 23.0% early, completely flat; TM_N_STATES 4/32/100 all ~22-23% (unstable across seeds); TM_S mildly monotonic (lower better early); TM_T flat; TM_WINDOW_SIZE 2/3/10 all within noise of each other and of the shuffled control. Two further findings: - The TM-off ablation (correction ~= 0) scores 22.7%/20.2%, essentially the same as TM-on. The TM's correction is near-zero-mean noise; the gun's one-shot internal linear baseline accounts for its accuracy. - The TM gun is 10.3 pp behind Linear early and 4.1 pp behind overall. That deficit is in the BASELINE MODEL (LinearGun iterates flight time; this gun does not), not in the TM hyper-parameters. Tuning knobs cannot close it. Conclusion: do not tune TM hyper-parameters further. Either the input representation or the prediction target is what needs to change - the shuffled control shows the TM is not extracting target information beyond its baseline. Defaults left UNCHANGED (window=10/states=32/S=1.5/T=25/clauses=50); an uncommitted prior edit (window=3/states=100) was reverted as unsupported. Hyper-parameters are now compile-time overridable (-d:TM_WINDOW_SIZE=3 etc.) so future sweeps need no gun edit. NOT MEASURED: real hit rate vs DrussGT (offline only by design). The repo's own docs/gun_rack_analysis.md 2 reports the virtual metric is a sign-unstable ranker of real hit rate, so the comparison against "Linear 10.7% real" is not direct - whether the TM is competitive live is INFERRED-unknown, not measured. Guards: test_gun_harness 39/39, test_vbullet_metric, test_power_selection, test_tsetlin_gun, test_tm_pattern_learning all green.
This commit is contained in:
@@ -0,0 +1,277 @@
|
||||
## Adaptation-speed sweep harness for the Tsetlin gun.
|
||||
##
|
||||
## Measures how quickly the TM becomes USEFUL within a round, starting from a
|
||||
## cold net (no persistence). For every fixture round it records every resolved
|
||||
## virtual bullet by its resolution tick RELATIVE to the round start, and reports
|
||||
## the virtual hit rate over:
|
||||
## * the first 100 ticks of each round,
|
||||
## * the first 300 ticks of each round,
|
||||
## * the whole round (for context).
|
||||
## Clause sparsity (mean included literals of active clauses) is reported too:
|
||||
## a config that never fires looks identical to a config that fires badly in the
|
||||
## hit-rate columns.
|
||||
##
|
||||
## Tsetlin is stochastic (tmLearnOne calls rand()), so each fixture is replayed
|
||||
## over multiple seeds and results are pooled across seeds AND rounds.
|
||||
##
|
||||
## The compiled config is printed from the exported `TM_*` constants, so the
|
||||
## harness always labels what was actually built.
|
||||
##
|
||||
## Usage:
|
||||
## nim c -r --path:common_libs -d:release common_libs/tests/sweep_tsetlin.nim \
|
||||
## --set=real --seeds=3
|
||||
## Flags:
|
||||
## --set=synthetic|real|range fixture set (default synthetic)
|
||||
## --seeds=N independent TM RNG seeds, pooled (default 3)
|
||||
## --maxrounds=N cap the rounds replayed per fixture
|
||||
## --linear replay the deterministic Linear gun instead
|
||||
## --shuffle randomised-feedback control: identical TM, but
|
||||
## every training target is a random arena point
|
||||
## --point use the bmPoint metric instead of the shipped
|
||||
## bmPath
|
||||
##
|
||||
## The TM hyper-parameters are compile-time constants; override them without
|
||||
## editing the gun, e.g.
|
||||
## nim c -r ... -d:TM_WINDOW_SIZE=3 -d:TM_S_DEF=1.1 -d:TM_N_STATES=100 \
|
||||
## -d:TM_N_CLAUSES=100 -d:TM_T_DEF=50 sweep_tsetlin.nim --set=real
|
||||
##
|
||||
## Emits CSV to stdout (one row per fixture×seed) plus SUM and # summary lines.
|
||||
|
||||
import std/[os, strformat, strutils, json, tables, math, random]
|
||||
import gun_harness/offline_range
|
||||
import range_guns
|
||||
import guns/tsetlin
|
||||
import guns/linear
|
||||
|
||||
const repoRoot = currentSourcePath().parentDir.parentDir.parentDir
|
||||
const fixturesDir = repoRoot / "tools" / "fixtures"
|
||||
|
||||
type
|
||||
AdaptResult = object
|
||||
h100, n100: int ## resolved within the first 100 ticks of the round
|
||||
h300, n300: int ## resolved within the first 300 ticks of the round
|
||||
hall, nall: int ## all resolutions in the round
|
||||
f100, m100: int ## FIRED within the first 100 ticks (robustness check)
|
||||
f300, m300: int ## FIRED within the first 300 ticks
|
||||
rounds: int
|
||||
|
||||
RoundSpan = tuple[start, count: int]
|
||||
|
||||
proc configLabel(): string =
|
||||
&"cl{TM_N_CLAUSES}_st{TM_N_STATES}_s{TM_S}_T{TM_T}_w{TM_WINDOW_SIZE}"
|
||||
|
||||
proc loadRounds(path: string): seq[RoundSpan] =
|
||||
## The round-boundary sidecar lives next to the classic/tr-bridge fixtures in
|
||||
## a `drussgt_meta/` subdirectory. Synthetic fixtures have none -> one round.
|
||||
let dir = path.parentDir
|
||||
let base = path.extractFilename
|
||||
var side = dir / "drussgt_meta" / (base & ".rounds.json")
|
||||
if not fileExists(side): side = dir / (base & ".rounds.json")
|
||||
if not fileExists(side): return @[]
|
||||
let node = parseJson(readFile(side))
|
||||
if not node.hasKey("rounds"): return @[]
|
||||
for r in node["rounds"]:
|
||||
result.add (r["startTick"].getInt(), r["count"].getInt())
|
||||
|
||||
proc addAdapt(dst: var AdaptResult, src: AdaptResult) =
|
||||
inc dst.rounds, src.rounds
|
||||
dst.h100 += src.h100; dst.n100 += src.n100
|
||||
dst.h300 += src.h300; dst.n300 += src.n300
|
||||
dst.hall += src.hall; dst.nall += src.nall
|
||||
dst.f100 += src.f100; dst.m100 += src.m100
|
||||
dst.f300 += src.f300; dst.m300 += src.m300
|
||||
|
||||
proc replayAdaptRound(states: seq[WorldState], lastSeen: seq[int],
|
||||
enemyId, baseTick: int, driver: GunDriver,
|
||||
metric: BulletMetric): AdaptResult =
|
||||
## One round, cold net, mirroring `replayFixture`'s tick order (spawn all 4
|
||||
## power bins, then `tickBullets` with the current state). Resolutions are
|
||||
## bucketed by `localTick = state.tick - baseTick`.
|
||||
var tracker = initTracker(1, metric)
|
||||
var res: AdaptResult
|
||||
inc res.rounds
|
||||
for si in 0..<states.len:
|
||||
let state = states[si]
|
||||
var preds: array[len(PowerBins), GunPrediction]
|
||||
for i in 0..<len(PowerBins):
|
||||
preds[i] = driver.predictCb(state, bulletSpeed(PowerBins[i]))
|
||||
let ready = if driver.readyCb == nil: true else: driver.readyCb()
|
||||
if ready:
|
||||
tracker.spawnBullets(0, preds, state, enemyId)
|
||||
|
||||
var enemyPositions: Table[int, tuple[x, y: float, lastSeenTick: int, alive: bool]]
|
||||
var lst = state.tick
|
||||
if si < lastSeen.len and lastSeen[si] >= 0: lst = lastSeen[si]
|
||||
if state.enemies.len > 0:
|
||||
for e in state.enemies:
|
||||
enemyPositions[e.id] = (x: e.x, y: e.y, lastSeenTick: lst, alive: true)
|
||||
else:
|
||||
enemyPositions[enemyId] = (x: state.enemyX, y: state.enemyY,
|
||||
lastSeenTick: lst, alive: true)
|
||||
|
||||
let localTick = state.tick - baseTick
|
||||
tracker.tickBullets(state, enemyPositions,
|
||||
proc(gunId: GunId, binIdx: int, e: FeedbackEvent) =
|
||||
inc res.nall
|
||||
if e.hit: inc res.hall
|
||||
if localTick < 100:
|
||||
inc res.n100
|
||||
if e.hit: inc res.h100
|
||||
if localTick < 300:
|
||||
inc res.n300
|
||||
if e.hit: inc res.h300
|
||||
let fireTick = e.fireTick - baseTick
|
||||
if fireTick < 100:
|
||||
inc res.m100
|
||||
if e.hit: inc res.f100
|
||||
if fireTick < 300:
|
||||
inc res.m300
|
||||
if e.hit: inc res.f300
|
||||
driver.resultCb(e))
|
||||
res
|
||||
|
||||
proc replayAdaptFixture(fx: Fixture, path: string, driver: GunDriver,
|
||||
metric: BulletMetric, maxRounds = 0): AdaptResult =
|
||||
## Split a fixture into its rounds (one cold replay each) and pool.
|
||||
var spans =
|
||||
if fx.meta.source == "synthetic": @[(start: 0, count: fx.states.len)]
|
||||
else: loadRounds(path)
|
||||
if maxRounds > 0 and spans.len > maxRounds:
|
||||
spans.setLen(maxRounds)
|
||||
if spans.len == 0:
|
||||
return replayAdaptRound(fx.states, fx.lastSeen, fx.enemyId, 0, driver, metric)
|
||||
for sp in spans:
|
||||
var st: seq[WorldState]
|
||||
var ls: seq[int]
|
||||
for i in 0..<fx.states.len:
|
||||
let t = fx.states[i].tick
|
||||
if t >= sp.start and t < sp.start + sp.count:
|
||||
st.add fx.states[i]
|
||||
ls.add(if i < fx.lastSeen.len: fx.lastSeen[i] else: -1)
|
||||
if st.len == 0: continue
|
||||
addAdapt(result, replayAdaptRound(st, ls, fx.enemyId, sp.start, driver, metric))
|
||||
|
||||
proc rate(h, n: int): string =
|
||||
if n == 0: " n/a " else: &"{h.float / n.float * 100.0:5.1f}%"
|
||||
|
||||
proc fixtureSet(name: string): seq[string] =
|
||||
case name
|
||||
of "synthetic", "":
|
||||
for n in SyntheticFixtureNames: result.add n
|
||||
of "real":
|
||||
for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "drussgt_vs_drussgt",
|
||||
"tr_drussgt_vs_crazy", "tr_drussgt_vs_spinbot",
|
||||
"tr_drussgt_vs_modularbot"]:
|
||||
result.add(fixturesDir / (n & ".jsonl"))
|
||||
of "range":
|
||||
for n in SyntheticFixtureNames: result.add n
|
||||
for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "tr_drussgt_vs_crazy"]:
|
||||
result.add(fixturesDir / (n & ".jsonl"))
|
||||
else:
|
||||
discard
|
||||
|
||||
proc resolveFixture(name: string): tuple[fx: Fixture, path: string] =
|
||||
let p = if fileExists(name): name else: fixturesDir / (name & ".jsonl")
|
||||
(loadFixture(p), p)
|
||||
|
||||
proc makeShuffledTsetlinDriver(seed = -1): tuple[driver: GunDriver, gun: ref TsetlinGun] =
|
||||
## Randomised-feedback CONTROL. Identical TM, identical predict/onResult call
|
||||
## cadence and encoding, but every training target is a uniform random point in
|
||||
## a 1000x1000 box instead of the real enemy position. If the early hit rate
|
||||
## of this control matches the real-feedback TM, then the measured skill is not
|
||||
## coming from informative feedback (it is luck / the linear baseline / the
|
||||
## always-nonzero correction).
|
||||
let g = new(TsetlinGun)
|
||||
g[] = initTsetlinGun()
|
||||
if seed >= 0:
|
||||
randomize(seed)
|
||||
result.gun = g
|
||||
result.driver = GunDriver(
|
||||
name: "ShuffledTM",
|
||||
predictCb: proc(state: WorldState, bulletSpeed: float): GunPrediction =
|
||||
g[].predict(state, bulletSpeed),
|
||||
resultCb: proc(e: FeedbackEvent) =
|
||||
var e2 = e
|
||||
e2.actualX = rand(1000.0)
|
||||
e2.actualY = rand(1000.0)
|
||||
g[].onResult(e2),
|
||||
readyCb: proc(): bool = g[].isWarmedUp(),
|
||||
)
|
||||
|
||||
proc main() =
|
||||
var nSeeds = 3
|
||||
var set = "synthetic"
|
||||
var maxRounds = 0
|
||||
var useLinear = false
|
||||
var usePoint = false
|
||||
var useShuffle = false
|
||||
for i in 1..paramCount():
|
||||
let a = paramStr(i)
|
||||
if a.startsWith("--seeds="): nSeeds = parseInt(a[8..^1])
|
||||
elif a.startsWith("--set="): set = a[6..^1]
|
||||
elif a.startsWith("--maxrounds="): maxRounds = parseInt(a[12..^1])
|
||||
elif a == "--linear": useLinear = true
|
||||
elif a == "--point": usePoint = true
|
||||
elif a == "--shuffle": useShuffle = true
|
||||
let metric = if usePoint: bmPoint else: bmPath
|
||||
let names = fixtureSet(set)
|
||||
if names.len == 0:
|
||||
echo "no fixtures for set: ", set
|
||||
quit(1)
|
||||
|
||||
let label = if useLinear: "Linear"
|
||||
elif useShuffle: "ShuffledTM"
|
||||
else: configLabel()
|
||||
echo "# config ", label, " seeds=", nSeeds, " set=", set
|
||||
echo "config,fixture,seed,h100,n100,h300,n300,hall,nall,meanInc,meanIncAll,nActive,nClauses"
|
||||
|
||||
for name in names:
|
||||
let (fx, path) = resolveFixture(name)
|
||||
let fxName = path.extractFilename.replace(".jsonl", "")
|
||||
var pooled: AdaptResult
|
||||
var meanIncSum = 0.0
|
||||
var meanIncAllSum = 0.0
|
||||
var activeSum = 0
|
||||
var nClauses = 0
|
||||
let nIter = if useLinear: 1 else: nSeeds
|
||||
for seed in 1..nIter:
|
||||
var drv: GunDriver
|
||||
var gun: ref TsetlinGun
|
||||
if useLinear:
|
||||
drv = makeDriver("Linear", LinearGun())
|
||||
elif useShuffle:
|
||||
let pair = makeShuffledTsetlinDriver(seed = seed)
|
||||
drv = pair.driver
|
||||
gun = pair.gun
|
||||
else:
|
||||
let pair = makeTsetlinDriver(seed = seed)
|
||||
drv = pair.driver
|
||||
gun = pair.gun
|
||||
let r = replayAdaptFixture(fx, path, drv, metric, maxRounds)
|
||||
var meanInc = -1.0
|
||||
var meanIncAll = -1.0
|
||||
var nActive = -1
|
||||
if not useLinear:
|
||||
let st = gun[].tmClauseStats()
|
||||
meanInc = st.meanIncluded
|
||||
meanIncAll = st.meanIncludedAll
|
||||
nActive = st.nActive
|
||||
nClauses = st.nClauses
|
||||
echo &"{label},{fxName},{seed},{r.h100},{r.n100},{r.h300},{r.n300},{r.hall},{r.nall}," &
|
||||
&"{meanInc:.1f},{meanIncAll:.1f},{nActive},{nClauses}"
|
||||
addAdapt(pooled, r)
|
||||
meanIncSum += meanInc
|
||||
meanIncAllSum += meanIncAll
|
||||
activeSum += nActive
|
||||
echo &"SUM,{label},{fxName},{pooled.h100},{pooled.n100},{pooled.h300},{pooled.n300}," &
|
||||
&"{pooled.hall},{pooled.nall},{pooled.f100},{pooled.m100},{pooled.f300},{pooled.m300}," &
|
||||
&"{meanIncSum / nIter.float:.1f},{meanIncAllSum / nIter.float:.1f}," &
|
||||
&"{activeSum.float / nIter.float:.0f},{nClauses}"
|
||||
echo &"# {label} {fxName}: first100 {rate(pooled.h100, pooled.n100)} " &
|
||||
&"first300 {rate(pooled.h300, pooled.n300)} whole {rate(pooled.hall, pooled.nall)} " &
|
||||
&"({pooled.hall}/{pooled.nall}, rounds={pooled.rounds}) " &
|
||||
&"sparsity={meanIncSum / nIter.float:.1f} meanAll={meanIncAllSum / nIter.float:.1f} " &
|
||||
&"active={activeSum.float / nIter.float:.0f}/{nClauses}"
|
||||
|
||||
when isMainModule:
|
||||
main()
|
||||
Reference in New Issue
Block a user