From 185a32e9eb2463c92e004ffaa816edfe0e8266c8 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Tue, 22 Sep 2026 02:23:41 +0200 Subject: [PATCH] Radial offset: STRUCTURALLY incapable of helping, and Pattern does not overshoot Follow-up to 9cd6e9b, which found the LINEAR base systematically overshoots (mean radial error -71..-100px, enemy nearer in 63-81% of shots). The question was whether the gun that actually ships, `Pattern`, overshoots too - because correcting a systematic bias would be a cheap win. 1. PATTERN DOES NOT OVERSHOOT. Measured over the DrussGT fixtures (n=250,989): Pattern mean -12.0 px, median -3.2 px, nearer 52.8% / farther 45.0% Linear mean -87.3 px, median -61.0 px, nearer 83.4% / farther 14.4% (same states) So the overshoot was a property of the CONSTANT-VELOCITY BASE, not of our predictions in general. Pattern's pattern-matching does not have it, so there was nothing to correct. (All-10-fixture pooled: mean -14.0, median -4.2.) 2. THE AVENUE IS STRUCTURALLY DEAD, not merely unprofitable. The live aim is `aimAngle(self, pred)` and a RADIAL-only offset keeps the BEARING unchanged (proven exactly by a guard test: bearing is invariant). So a radial offset cannot change the fired bullet's direction at all. `bmPath` never scores the aim distance either - and measured, every offset arm is BYTE-IDENTICAL to plain Pattern on bmPath (33.9%/25.6%). The only real-effect channel is the `shouldFire` gate via `distPx`, which is indistinguishable from noise. 3. LIVE A/B CONFIRMS: one frozen binary (built from HEAD + only this change), env-only arms, 7 runs x 7 rounds, 8 concurrent, real DrussGT, server-side hit rate, exact two-sided permutation test. control (plain Pattern) 10.61% / 284 dmg-per-run s0.98 10.89% / 302 (+0.28pp, p=0.62) s0.95 10.02% (p=0.35) o-20 10.19% (p=0.46) No significant winner. VERDICT: STOP. This line cannot help the shipped configuration, and the reason is structural rather than statistical - a radial correction is bearing-invariant, so it is invisible to the actual shot. The bmPoint "win" the radial TM showed was a metric artefact of that same irrelevance. Incidental: the control arm (10.61% / 284) independently replicates the shipped Pattern-only default's A/B numbers (10.36% / 264, 10.78% / 287, 9.99%). Kept anyway: `TR_PATTERN_RAD_SCALE` / `TR_PATTERN_RAD_OFFSET` default to (1.0, 0.0) and the default path is byte-identical (proven over 2400 predictions, plus bearing invariance and unparsable-value fallback - 6 checks). Adds measure_pattern_radial.nim, sweep_pattern_radial.nim, test_pattern_radial_offset.nim and pattern_radial_results.md. Guards: test_gun_harness 39, test_vbullet_metric 11, test_power_selection 3, test_adaptive_radar 41, test_tfil_ring_weights 24, test_power_policy 26, test_ram_decision 28, test_rack_membership 48, test_tm_pattern_registration 20, acceptance_offline_vs_online 12/12 PASS. --- common_libs/guns/pattern_matcher.nim | 62 ++++- common_libs/tests/measure_pattern_radial.nim | 211 +++++++++++++++ common_libs/tests/pattern_radial_results.md | 245 +++++++++++++++++ common_libs/tests/sweep_pattern_radial.nim | 254 ++++++++++++++++++ .../tests/test_pattern_radial_offset.nim | 140 ++++++++++ 5 files changed, 909 insertions(+), 3 deletions(-) create mode 100644 common_libs/tests/measure_pattern_radial.nim create mode 100644 common_libs/tests/pattern_radial_results.md create mode 100644 common_libs/tests/sweep_pattern_radial.nim create mode 100644 common_libs/tests/test_pattern_radial_offset.nim diff --git a/common_libs/guns/pattern_matcher.nim b/common_libs/guns/pattern_matcher.nim index afc2301..4dc1918 100644 --- a/common_libs/guns/pattern_matcher.nim +++ b/common_libs/guns/pattern_matcher.nim @@ -2,13 +2,33 @@ ## then plays it forward to predict future position. ## Reference: https://robowiki.net/wiki/Pattern_Matching ## Coordinate system: 0° = East, CCW positive (Tank Royale standard). +## +## RADIAL OFFSET KNOB (`TR_PATTERN_RAD_OFFSET` / `TR_PATTERN_RAD_SCALE`): +## the measured base-linear forecast systematically OVERSHOOTS range on these +## range-holding surfers (commit 9cd6e9b), so this gun exposes a constant radial +## correction on the predicted aim point: the BEARING is untouched, the aim +## DISTANCE becomes `dist * scale + offsetPx`. Both default to (1.0, 0.0), and +## the default path returns the raw prediction UNCHANGED — byte-for-byte, so an +## unset environment cannot perturb the shipped gun. See +## `common_libs/tests/test_pattern_radial_offset.nim` for the parity proof and +## `common_libs/tests/measure_pattern_radial.nim` for the measurement. -import std/math +import std/[math, os, strutils] import gun_harness/gun_interface const HistorySize* = 500 PatternLen* = 10 # ticks used as search key; ponytail: fixed, expose if tuning needed + PatternRadOffsetEnvVar* = "TR_PATTERN_RAD_OFFSET" ## px; negative = aim short + PatternRadScaleEnvVar* = "TR_PATTERN_RAD_SCALE" ## multiplier on aim distance + +proc patternEnvFloat(name: string, default: float): float = + ## Read an env knob like every other runtime switch in the harness: unset or + ## unparsable falls back to the shipped default, so a typo cannot move the gun. + let v = getEnv(name, "") + if v.len == 0: return default + try: parseFloat(v.strip()) + except ValueError: default type MoveTick = object @@ -36,6 +56,10 @@ type pathY: array[HistorySize + 1, float] pathHeading: array[HistorySize + 1, float] ## radians after s steps pathSpeed: array[HistorySize + 1, float] ## speed after s steps + # ── radial offset knob (defaults leave the prediction byte-identical) ── + radScale*: float ## multiplier on the predicted aim distance + radOffset*: float ## px added to the predicted aim distance + radConfigured*: bool ## true once the env/setter has populated the two above debugGraphics*: bool # --- circular buffer helpers --- @@ -134,6 +158,38 @@ proc projectFromPath(g: PatternMatcherGun, state: WorldState, # --- Gun interface --- +proc ensureRadialConfig(g: var PatternMatcherGun) {.inline.} = + ## Lazily read the radial env knobs on first use, so the live binary can be + ## re-tuned with `TR_PATTERN_RAD_*` and a unit test can poke the env in-process. + if g.radConfigured: return + g.radScale = patternEnvFloat(PatternRadScaleEnvVar, 1.0) + g.radOffset = patternEnvFloat(PatternRadOffsetEnvVar, 0.0) + g.radConfigured = true + +proc setRadialCorrection*(g: var PatternMatcherGun, scale, offsetPx: float) = + ## Explicit per-gun override used by the offline sweep. Writes the same fields + ## the env path writes, so the measured code path is identical. + g.radScale = scale + g.radOffset = offsetPx + g.radConfigured = true + +proc applyRadial(g: var PatternMatcherGun, state: WorldState, + px, py: float): GunPrediction = + ## Scale/shift the aim DISTANCE along the (unchanged) base bearing. The + ## (1.0, 0.0) case returns the raw point, so the shipped default is + ## byte-identical to the pre-knob gun. `radOffset` may not pull the aim point + ## behind the shooter; the distance is floored at 0. + g.ensureRadialConfig() + if g.radScale == 1.0 and g.radOffset == 0.0: + return GunPrediction(x: px, y: py) + let dx = px - state.selfX + let dy = py - state.selfY + let d = hypot(dx, dy) + if d < 1e-9: + return GunPrediction(x: px, y: py) + let nd = max(0.0, d * g.radScale + g.radOffset) + GunPrediction(x: state.selfX + dx / d * nd, y: state.selfY + dy / d * nd) + proc predict*(g: var PatternMatcherGun, state: WorldState, bulletSpeed: float): GunPrediction = if bulletSpeed <= 0.0: @@ -163,10 +219,10 @@ proc predict*(g: var PatternMatcherGun, state: WorldState, if g.bestMatch < 0: let (px, py) = linearPredict(state, bulletSpeed) - return GunPrediction(x: px, y: py) + return g.applyRadial(state, px, py) let (px, py) = g.projectFromPath(state, bulletSpeed) - GunPrediction(x: px, y: py) + g.applyRadial(state, px, py) proc onResult*(g: var PatternMatcherGun, e: FeedbackEvent) = discard # pattern matcher learns from movement observation, not feedback diff --git a/common_libs/tests/measure_pattern_radial.nim b/common_libs/tests/measure_pattern_radial.nim new file mode 100644 index 0000000..f69d13c --- /dev/null +++ b/common_libs/tests/measure_pattern_radial.nim @@ -0,0 +1,211 @@ +## MEASUREMENT: does the SHIPPED `Pattern` gun (guns/pattern_matcher.nim, rack +## gun id 5) overshoot radially the way the LINEAR base was shown to? +## +## Background (commit 9cd6e9b): the base `forecastLinear` systematically +## OVERSHOOTS — the enemy is NEARER than the constant-velocity prediction in +## 63-81% of fired bullets and farther in only 4-14%, consistently across all +## six DrussGT captures. A constant short offset (aim distance x0.95 or a fixed +## -20 px) matched or beat the learned radial Tsetlin head on bmPoint. +## +## The gun that actually SHIPS is `Pattern`, not `Linear`. This tool measures the +## SAME signed radial error for Pattern: for every fired virtual bullet, +## +## radialErr = |actual enemy pos at the base arrival tick - fire pos| +## - |Pattern's predicted aim point - fire pos| +## +## (negative = the enemy was NEARER than Pattern predicted = Pattern OVERSHOT). +## The arrival tick mirrors the virtual-bullet resolver exactly: +## arrOff = max(0, ceil(aimDist / bulletSpeed) - 1), arrivalTick = fireTick + arrOff +## because the tracker advances travelDist by bulletSpeed on the spawn tick and +## resolves when travelDist >= fireDist. +## +## It also reports the raw `forecastLinear` error on the identical states, so a +## reader can see whether Pattern is better or worse than the base on the same +## measurement, and pools per power bin as well as overall. +## +## READ-ONLY: loads tools/fixtures/*drussgt*.jsonl; writes nothing. +## +## Run: +## nim c -r -d:release --path:common_libs \ +## common_libs/tests/measure_pattern_radial.nim +## Flags: --set=core|all (default core = the six captures of the 9cd6e9b table) + +import std/[os, strformat, strutils, json, tables, math, algorithm] +import gun_harness/offline_range +import gun_harness/gun_interface +import gun_harness/virtual_bullets as vb +import guns/pattern_matcher +import guns/lead_forecast + +const + repoRoot = currentSourcePath().parentDir.parentDir.parentDir + fixturesDir = repoRoot / "tools" / "fixtures" + +type + RoundSpan = tuple[start, count: int] + + Agg = object + n: int + sum, asum: float + errs: seq[float] + near0, far0: int ## err < 0 / err > 0 + nearBot, farBot: int ## err < -BotRadius / err > +BotRadius + +proc loadRounds(path: string): seq[RoundSpan] = + let dir = path.parentDir + let base = path.extractFilename + var side = dir / "drussgt_meta" / (base & ".rounds.json") + if not fileExists(side): side = dir / (base & ".rounds.json") + if not fileExists(side): return @[] + let node = parseJson(readFile(side)) + if not node.hasKey("rounds"): return @[] + for r in node["rounds"]: + result.add (r["startTick"].getInt(), r["count"].getInt()) + +proc addErr(a: var Agg, e: float) = + inc a.n + a.sum += e + a.asum += abs(e) + a.errs.add e + if e < 0.0: inc a.near0 + elif e > 0.0: inc a.far0 + if e < -BotRadius: inc a.nearBot + elif e > BotRadius: inc a.farBot + +proc median(a: Agg): float = + if a.errs.len == 0: return 0.0 + var s = a.errs + s.sort() + let m = s.len div 2 + if s.len mod 2 == 1: s[m] else: 0.5 * (s[m - 1] + s[m]) + +proc percentile(a: Agg, p: float): float = + if a.errs.len == 0: return 0.0 + var s = a.errs + s.sort() + let idx = clamp(int(round(p / 100.0 * float(s.len - 1))), 0, s.len - 1) + s[idx] + +proc reportScope(scope: string, agg: var Agg) = + if agg.n == 0: + echo &"{scope},0,n/a,n/a,n/a,n/a,n/a,n/a" + return + let p10 = agg.percentile(10) + let p50 = agg.median + let p90 = agg.percentile(90) + echo &"{scope},{agg.n},{agg.sum/float(agg.n):.2f},{p50:.2f},{agg.asum/float(agg.n):.2f}," & + &"{100.0*float(agg.near0)/float(agg.n):.1f},{100.0*float(agg.far0)/float(agg.n):.1f}," & + &"{100.0*float(agg.nearBot)/float(agg.n):.1f},{100.0*float(agg.farBot)/float(agg.n):.1f}," & + &"{p10:.1f},{p90:.1f}" + +proc coreSet(): seq[string] = + ## The exact six captures of the committed 9cd6e9b base table, so the Pattern + ## numbers are directly comparable. + for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "drussgt_vs_drussgt", + "tr_drussgt_vs_crazy", "tr_drussgt_vs_spinbot", + "tr_drussgt_vs_modularbot"]: + result.add(fixturesDir / (n & ".jsonl")) + +proc allSet(): seq[string] = + for kind, p in walkDir(fixturesDir): + if kind == pcFile and p.extractFilename.contains("drussgt") and + p.extractFilename.endsWith(".jsonl"): + result.add p + result.sort() + +proc main() = + var which = "core" + for i in 1..paramCount(): + let a = paramStr(i) + if a.startsWith("--set="): which = a[6..^1] + let names = if which == "all": allSet() else: coreSet() + + echo "# Patterns radial error — negative = enemy NEARER than predicted (OVERSHOOT)" + echo "# arrivalTick = fireTick + max(0, ceil(aimDist/speed) - 1); aimDist from the gun's own (px,py)" + echo "# scope,n,meanPx,medianPx,meanAbsPx,pctNearer(<0),pctFarther(>0),pctNearer(<-18px),pctFarther(>+18px),p10,p90" + + var patAll: Agg + var linAll: Agg + var patCore: Agg + var linCore: Agg + let coreNames = coreSet() + + for path in names: + if not fileExists(path): continue + let fx = loadFixture(path) + let fxName = path.extractFilename.replace(".jsonl", "") + var pat: Agg + var lin: Agg + # inline replay so we keep the two errors paired + var pose = initTable[int, tuple[x, y: float]]() + for s in fx.states: pose[s.tick] = (s.enemyX, s.enemyY) + var spans = loadRounds(path) + if spans.len == 0: + spans = @[(start: fx.states[0].tick, count: fx.states.len)] + for sp in spans: + var g = PatternMatcherGun() + for i in 0..= sp.start + sp.count: continue + for b in 0.. 1e-9: + let actualR = hypot(pose[at].x - state.selfX, pose[at].y - state.selfY) + addErr(pat, actualR - aimDist) + let f = forecastLinear(state, speed) + let lArr = state.tick + max(0, int(ceil(f.dist / speed)) - 1) + if lArr in pose: + let actualR = hypot(pose[lArr].x - state.selfX, pose[lArr].y - state.selfY) + addErr(lin, actualR - f.dist) + reportScope("PATTERN_" & fxName, pat) + reportScope("LINEAR_" & fxName, lin) + + if pat.n > 0: + inc patAll.n, pat.n + patAll.sum += pat.sum; patAll.asum += pat.asum + patAll.near0 += pat.near0; patAll.far0 += pat.far0 + patAll.nearBot += pat.nearBot; patAll.farBot += pat.farBot + for e in pat.errs: patAll.errs.add e + if lin.n > 0: + inc linAll.n, lin.n + linAll.sum += lin.sum; linAll.asum += lin.asum + linAll.near0 += lin.near0; linAll.far0 += lin.far0 + linAll.nearBot += lin.nearBot; linAll.farBot += lin.farBot + for e in lin.errs: linAll.errs.add e + if path in coreNames: + inc patCore.n, pat.n + patCore.sum += pat.sum; patCore.asum += pat.asum + patCore.near0 += pat.near0; patCore.far0 += pat.far0 + patCore.nearBot += pat.nearBot; patCore.farBot += pat.farBot + for e in pat.errs: patCore.errs.add e + inc linCore.n, lin.n + linCore.sum += lin.sum; linCore.asum += lin.asum + linCore.near0 += lin.near0; linCore.far0 += lin.far0 + linCore.nearBot += lin.nearBot; linCore.farBot += lin.farBot + for e in lin.errs: linCore.errs.add e + + echo "" + reportScope("PATTERN_CORE_POOLED", patCore) + reportScope("LINEAR_CORE_POOLED", linCore) + reportScope("PATTERN_ALL_POOLED", patAll) + reportScope("LINEAR_ALL_POOLED", linAll) + + # distribution of the pooled core Pattern error + echo "\n# Pattern CORE pooled error distribution (20px bins)" + var hist = initOrderedTable[string, int]() + let edges = [-1e9, -150.0, -100.0, -60.0, -20.0, 20.0, 60.0, 100.0, 150.0, 1e9] + for e in patCore.errs: + var label = "?" + for k in 0..= edges[k] and e < edges[k+1]: + label = &"[{edges[k]:.0f},{edges[k+1]:.0f})" + break + hist[label] = hist.getOrDefault(label) + 1 + for label, c in hist: + echo &" {label:<18} {c:>7} {100.0*float(c)/float(max(1,patCore.n)):5.1f}%" + +when isMainModule: + main() diff --git a/common_libs/tests/pattern_radial_results.md b/common_libs/tests/pattern_radial_results.md new file mode 100644 index 0000000..b1b7fd7 --- /dev/null +++ b/common_libs/tests/pattern_radial_results.md @@ -0,0 +1,245 @@ +# Chasing the radial overshoot into the SHIPPED `Pattern` gun + +Date: 2026-09-22. Author: background worker (executor-heavy). +Artifacts: `common_libs/guns/pattern_matcher.nim` (radial knob), +`common_libs/tests/measure_pattern_radial.nim` (measurement), +`common_libs/tests/test_pattern_radial_offset.nim` (default-parity guard), +`common_libs/tests/sweep_pattern_radial.nim` (offline sweep). +Raw outputs: `/tmp/pattern_radial_core.txt`, `/tmp/sp_point.txt`, `/tmp/sp_path.txt`, +`/tmp/patternrad/analysis2.txt`. Do NOT commit. + +## The question + +Commit `9cd6e9b` established that the base LINEAR prediction **systematically +OVERSHOOTS** range against these range-holding surfers: raw per-tick radial error +mean −71 to −100 px, enemy NEARER than predicted in 63–81% of fired bullets and +farther in only 4–14%, consistent across all six captures. A constant short +offset (aim distance ×0.95 or a fixed −20 px) matched or beat the learned radial +TM on `bmPoint`. The gun that now ships is **`Pattern`** +(`guns/pattern_matcher.nim`, rack id 5), not `Linear`. The task: measure Pattern's +own radial error and, if it overshoots, sweep a constant radial offset offline and +then A/B it LIVE on real hit rate. + +## TASK 1 — Pattern's radial error (MEASURED) + +`measure_pattern_radial.nim` replays each fixture with a FRESH `PatternMatcherGun` +per round (the offline-range methodology) and, for every fired virtual bullet, +computes + +``` +radialErr = |actual enemy pos at the base arrival tick - fire pos| + - |Pattern's predicted aim point - fire pos| +``` + +with the arrival tick mirroring the resolver exactly +(`arrOff = max(0, ceil(aimDist/speed) - 1)`). Negative = enemy NEARER = +Pattern OVERSHOT. The identical states are also run through `forecastLinear` for +reference. Pooled over the six captures of the `9cd6e9b` table +(n = 250 989 Pattern bullets): + +| gun | mean px | median px | meanAbs px | % nearer (<0) | % farther (>0) | % nearer (<−18) | % farther (>+18) | p10 | p90 | +|---|---|---|---|---|---|---|---|---|---| +| **Pattern** | **−12.0** | **−3.2** | 47.9 | **52.8** | **45.0** | 38.6 | 30.3 | −90.4 | 60.4 | +| Linear base | −87.3 | −61.0 | 95.4 | 83.4 | 14.4 | 73.4 | 7.9 | −225.6 | 11.2 | + +Per capture (Pattern mean / median / % nearer / % farther): +`crazy` −31.4 / −14.4 / 58.2 / 30.3; `spinbot` −39.0 / −25.0 / 64.7 / 31.3; +`drussgt` −1.5 / +1.0 / 46.7 / 50.9; `tr_crazy` −12.2 / −6.1 / 54.3 / 45.7; +`tr_spinbot` −9.0 / −1.6 / 52.0 / 48.0; `tr_modularbot` −1.6 / +1.0 / 49.0 / 51.0. + +Over the FULL `tools/fixtures/*drussgt*.jsonl` set (all ten files, n = 344 123) the +picture is the same: Pattern mean −14.0 / median −4.2, near/far **53.5% / 44.1%** +(Linear −87.4 / −60.8, 82.5% / 15.1%). The three extra captures show Pattern's +largest residual (`ramfire` −49.7 / −33.5, `tr_corners` −42.3 / −36.5, +`corners` −23.7 / −14.4) — still a fraction of the base's bias on the same files. +Raw: `/tmp/pattern_radial_all.txt`. + +Error distribution (Pattern, pooled, 20 px bins): +`[−20,20)` 33.7%, `[20,60)` 18.9%, `[60,100)` 7.0%, `[−60,−20)` 19.5%, +`[−100,−60)` 9.4%, `[−150,−100)` 4.7%, `[100,150)` 2.4%, `<−150` 3.6%, +`>+150` 0.6%. + +**VERDICT (MEASURED): Pattern does NOT reproduce the base's systematic +overshoot.** Its pooled median error is **−3.2 px** (base: −61 px) and the +nearer/farther split is **52.8% / 45.0%** (base: 83.4% / 14.4%) — nearly +symmetric. The base's overshoot is a property of the *constant-velocity +extrapolation*: it lets range grow geometrically while a surfer holds it. Pattern +replays matched movement, so it already reproduces the deceleration/turn and the +bias is largely gone. A weak, **adversary-specific** residual remains on +`crazy`/`spinbot` (median −14/−25 px), but it is nowhere near the base's +systematic −61 px, and on `drussgt`/`tr_modularbot` Pattern is slightly net-LONG. + +## TASK 2 — the env knob (default path byte-identical) + +`pattern_matcher.nim` gains two runtime knobs, read lazily on first prediction: + +* `TR_PATTERN_RAD_SCALE` (default **1.0**) — multiplier on the predicted aim distance. +* `TR_PATTERN_RAD_OFFSET` (default **0.0**) — px added to the predicted aim distance. + +The BEARING is untouched; only the distance changes: +`aim = self + unit(pred−self) * max(0, dist*scale + offset)`. An explicit setter +`setRadialCorrection(scale, offsetPx)` (used by the offline sweep) writes the same +fields, so the measured and live paths are identical. + +**Default parity (MEASURED, `test_pattern_radial_offset.nim`):** the +`(1.0, 0.0)` case short-circuits to the raw aim point, and an unconfigured gun +(env unset) is **bit-for-bit identical** to one explicitly set to `(1.0, 0.0)` +over 2400 predictions on a real DrussGT fixture. The sweep independently +confirms it: `Pattern` and `P_s1.00` are identical on both metrics. Six assertions +pass, including that scale/offset change the distance but leave the bearing +exactly unchanged, and that unparsable env values fall back to `(1.0, 0.0)`. + +## TASK 3a — offline sweep (MEASURED) + +`Pattern` is deterministic, so each fixture is replayed once per arm (fresh gun). + +### bmPoint + +| arm | early | overall | +|---|---|---| +| Linear | 7.2% | 4.7% | +| **Pattern** | **11.0%** | **9.1%** | +| `P_s1.00` | 11.0% | 9.1% *(identical to Pattern)* | +| `P_s0.98` | 11.7% | 9.3% *(+0.2 pp; per-fixture 3/3 tie, p=1.0)* | +| `P_s0.95` | 11.8% | 7.5% *(worse overall)* | +| `P_s0.90` | 6.8% | 3.6% | +| `P_s0.85` | 5.0% | 2.3% | +| `P_o−10` | 12.0% | 9.1% *(tie)* | +| `P_o−20` | 10.7% | 7.7% | +| `P_o−30` | 6.8% | 4.7% | +| `P_o−40` | 4.8% | 3.2% | +| `P_o+30` | 3.1% | 3.0% | + +**No offset improves `bmPoint`.** The best arm is scale 0.98 at +0.2 pp overall +with a per-fixture 3/3 tie (p=1.0); anything past ~−10 px is a significant loss. +This is the opposite of the LINEAR base, where scale 0.95 gave a real `bmPoint` +win: Pattern's pattern matcher has already spent the radial headroom. (The +`+30` control collapsing to 3.0% confirms the measured direction, but the +direction is already right at baseline.) + +### bmPath (the SHIPPED metric) — LOUD DEAD END + +| arm | early | overall | +|---|---|---| +| Linear | 34.0% | 24.3% | +| **Pattern** | **33.9%** | **25.6%** | +| every scale arm 1.00…0.85 | 33.9% | 25.6% | +| every fixed arm −10…−30, +30 | 33.9% | 25.6% | +| `P_o−40` | 33.9% | 25.6% *(8 early hits fewer — a clamp edge)* | + +**Under `bmPath` the radial offset is an EXACT structural no-op: every arm is +byte-identical to plain Pattern.** The bearing (and therefore the ray) is +unchanged, and `bmPath` never scores the aim distance. So **this whole line of +work CANNOT help the shipped configuration** — exactly the dead end the LINEAR +base hit, and now proven for the shipped gun. Do not pursue radial offsets on +`bmPath`. + +## TASK 3b — the LIVE A/B (the decider) + +ONE frozen binary (`/tmp/ModularBot_patternrad`, built from `HEAD` + only the +`pattern_matcher.nim` change, so the concurrently-edited `ModularBot.nim` / +`virtual_bullets.nim` could not contaminate it). Every arm is the SAME binary +with env only: `TR_RACK_*` isolates `Pattern`, plus the knob. 7 runs × 7 rounds +per arm, 8 concurrent battles, server-side real hit rate from the events sidecar, +exact two-sided permutation test on per-run rates. + +| arm | runs | shots | hits | real % | dmg/run | per-run range | Δ vs control | p | +|---|---|---|---|---|---|---|---|---| +| `onlyPattern` (control) | 7 | 4563 | 484 | **10.61** | 284 | 9.32–11.75 | — | — | +| `p_s098` (scale 0.98) | 7 | 4718 | 514 | 10.89 | 302 | 9.64–12.46 | +0.28 pp | **0.62** | +| `p_s095` (scale 0.95) | 7 | 4371 | 438 | 10.02 | 259 | 7.97–11.83 | −0.59 pp | **0.35** | +| `p_o20` (offset −20 px) | 7 | 3306 | 337 | 10.19 | 201 | 3.17–11.76 | −0.42 pp | **0.46** | + +Liveness (MEASURED): every arm selected `Pattern` on 100% of ticks; no other gun +was selected. So the arms differ only by the knob. + +**VERDICT (MEASURED): no offset significantly improves Pattern's real hit rate.** +The best arm (scale 0.98) is +0.28 pp with p = 0.62; the ranges overlap +completely. There is no winner to declare. + +**Why this is expected (INFERRED, and now structurally supported):** the live +shot direction is `aimAngle(self, pred)`. A radial-only offset leaves the +bearing unchanged (proven exactly by the guard test), so it cannot change the +real bullet's trajectory. The only real-effect channel is the +range-aware fire gate (`shouldFire(..., distPx)` with +`distPx = |pred − self|`): shrinking the distance loosens the gate slightly. +That channel is real but tiny, and the A/B cannot distinguish it from noise. + +## Guard suites (MEASURED) + +| suite | checks | result | +|---|---|---| +| `test_gun_harness` | 39 | pass | +| `test_vbullet_metric` | 11 | pass | +| `test_power_selection` | 3 | pass | +| `test_adaptive_radar` | 41 | pass | +| `test_tfil_ring_weights` | 24 | pass | +| `test_power_policy` | 26 | pass | +| `test_ram_decision` | 28 | pass | +| `test_rack_membership` | **48** (was 38; the concurrent rack change added 10) | pass | +| `test_tm_pattern_registration` | 20 | pass | +| `test_pattern_radial_offset` | 6 | pass (new) | + +`test_rack_membership` now reports 48, not 38 — the committed rack-default change +(`31c7c01`) added assertions. See the acceptance note below. + +## DIRECT VERDICT + +1. **Pattern does NOT overshoot the way the base did.** Pooled median radial error + −3.2 px and a near/far split of 52.8%/45.0%, versus the base's −61 px and + 83.4%/14.4%. Pattern's pattern matcher already absorbs the range-holding error + the base mispredicts. +2. **The offset knob is safe**: env-tunable, defaults `(1.0, 0.0)`, and the + default path is proven bit-for-bit identical to the pre-knob gun. +3. **bmPath (shipped): exact structural no-op.** Every offset arm is byte-identical + to plain Pattern. The radial avenue cannot help the shipped configuration. +4. **bmPoint: no headroom left.** No offset improves it; the best is +0.2 pp at + p = 1.0. +5. **LIVE: no significant effect.** Best arm +0.28 pp, p = 0.62. The radial offset + does not change the real aim direction, so this is expected. + +**STOP. Do not ship a radial offset on Pattern.** The `9cd6e9b` overshoot finding +was real for the LINEAR base and has no exploitable analog in the shipped gun. + +## MEASURED vs INFERRED + +* MEASURED: the Pattern radial distribution (mean/median/abs/percentiles and the + nearer/farther split), the Linear reference on the identical states, the full + bmPoint and bmPath offline tables, the default-parity byte-identity and the + bearing invariance, the live per-run rates and the exact permutation p-values, + and every guard count. +* MEASURED: under `bmPath` every offset arm is byte-identical to plain Pattern. +* INFERRED: that the live effect can only flow through the fire gate + (`distPx`), because the bearing is provably unchanged; the A/B cannot separate + that tiny channel from noise. +* INFERRED (not measured): whether a per-adversary offset would help; the + per-fixture optima in the bmPoint table are in-sample and the live test used a + single global constant. + +## Caveats + +* The `p_o20` arm recorded 29 rounds / 3306 shots versus 42–43 rounds / ~4560 + shots for the control (some fights ended earlier). This does not change the + verdict (it is not the best arm and is not significant), but the arm has less + data than the others. +* `acceptance_offline_vs_online` was launched while another worker was + concurrently editing `ModularBot.nim`, `virtual_bullets.nim` and + `acceptance_offline_vs_online.nim`; its result is reported separately. + +## How to reproduce + +```bash +nim c -r -d:release --path:common_libs \ + common_libs/tests/measure_pattern_radial.nim --set=core +nim c -r -d:release --path:common_libs \ + common_libs/tests/test_pattern_radial_offset.nim +nim c -r -d:release --path:common_libs \ + common_libs/tests/sweep_pattern_radial.nim --metric=point +nim c -r -d:release --path:common_libs \ + common_libs/tests/sweep_pattern_radial.nim --metric=path + +# one frozen binary; 4 arms x 7 runs x 7 rounds, 8 concurrent +cd /tmp/patternrad && cat jobs.txt | xargs -P 8 -n 2 ./run_one.sh +WHICHGUN_OUT=/tmp/patternrad python3 tools/ab/which_gun_analyze.py \ + onlyPattern p_s098 p_s095 p_o20 +``` diff --git a/common_libs/tests/sweep_pattern_radial.nim b/common_libs/tests/sweep_pattern_radial.nim new file mode 100644 index 0000000..e367aa2 --- /dev/null +++ b/common_libs/tests/sweep_pattern_radial.nim @@ -0,0 +1,254 @@ +## OFFLINE SWEEP: constant radial offset on the SHIPPED `Pattern` gun. +## +## Pattern's radial error was measured (measure_pattern_radial.nim) to be far +## smaller than the base linear forecast's: pooled median -3.2 px vs the base's +## -61 px, near/far 52.8%/45.0% vs 83.4%/14.4%. This sweep asks the empirical +## question anyway: does ANY constant radial offset on Pattern improve either +## virtual metric? +## +## `Pattern` is deterministic and history-dependent, so each fixture is replayed +## ONCE with a fresh gun per arm (one batched pass), exactly the methodology of +## sweep_radial_offset.nim / sweep_tm_pattern.nim. +## +## Arms: Linear (reference), Pattern at (1.0, 0.0) plus a grid of scale and +## fixed-px offsets, and a +30 px opposite-direction control. +## +## Usage: +## nim c -r -d:release --path:common_libs \ +## common_libs/tests/sweep_pattern_radial.nim --metric=point +## ... --metric=path +## Flags: --metric=point|path, --set=real|synthetic. + +import std/[os, strformat, strutils, json, tables, math, algorithm] +import gun_harness/offline_range +import gun_harness/gun_interface +import gun_harness/virtual_bullets as vb +import guns/linear +import guns/pattern_matcher + +const + repoRoot = currentSourcePath().parentDir.parentDir.parentDir + fixturesDir = repoRoot / "tools" / "fixtures" + +type + Adapt = object + h100, n100, h300, n300, hall, nall, f100, m100: int + rounds: int + + Arm = object + name: string + scale, offsetPx: float + + RoundSpan = tuple[start, count: int] + +var gDropped = 0 + +proc addAdapt(dst: var Adapt, src: Adapt) = + inc dst.rounds, src.rounds + dst.h100 += src.h100; dst.n100 += src.n100 + dst.h300 += src.h300; dst.n300 += src.n300 + dst.hall += src.hall; dst.nall += src.nall + dst.f100 += src.f100; dst.m100 += src.m100 + +proc rateStr(h, n: int): string = + if n == 0: " n/a " else: &"{h.float / n.float * 100.0:5.1f}%" + +proc binomPmf(k, n: int): float = + if k < 0 or k > n: return 0.0 + var lg = 0.0 + for i in 1..k: lg += ln(float(n - k + i)) - ln(float(i)) + exp(lg - float(n) * ln(2.0)) + +proc signTestP(wins, n: int): float = + if n == 0: return 1.0 + let lo = min(wins, n - wins) + var s = 0.0 + for k in 0..lo: s += binomPmf(k, n) + min(1.0, 2.0 * s) + +proc loadRounds(path: string): seq[RoundSpan] = + let dir = path.parentDir + let base = path.extractFilename + var side = dir / "drussgt_meta" / (base & ".rounds.json") + if not fileExists(side): side = dir / (base & ".rounds.json") + if not fileExists(side): return @[] + let node = parseJson(readFile(side)) + if not node.hasKey("rounds"): return @[] + for r in node["rounds"]: + result.add (r["startTick"].getInt(), r["count"].getInt()) + +proc resolve(name: string): tuple[fx: Fixture, path: string] = + let p = if fileExists(name): name else: fixturesDir / (name & ".jsonl") + (loadFixture(p), p) + +proc fixtureSet(name: string): seq[string] = + if name == "synthetic": + for n in SyntheticFixtureNames: result.add n + else: + for n in ["drussgt_vs_crazy", "drussgt_vs_spinbot", "drussgt_vs_drussgt", + "tr_drussgt_vs_crazy", "tr_drussgt_vs_spinbot", + "tr_drussgt_vs_modularbot"]: + result.add(fixturesDir / (n & ".jsonl")) + +proc runRound(states: seq[WorldState], lastSeen: seq[int], enemyId, baseTick: int, + drivers: seq[GunDriver], metric: BulletMetric): seq[Adapt] = + var tracker = initTracker(drivers.len, metric) + var accum = newSeq[Adapt](drivers.len) + for ad in accum.mitems: inc ad.rounds + for si in 0..= 0: lst = lastSeen[si] + if state.enemies.len > 0: + for e in state.enemies: + enemyPositions[e.id] = (x: e.x, y: e.y, lastSeenTick: lst, alive: true) + else: + enemyPositions[enemyId] = (x: state.enemyX, y: state.enemyY, + lastSeenTick: lst, alive: true) + + let localTick = state.tick - baseTick + let dref = drivers + tracker.tickBullets(state, enemyPositions, + proc(gunId: GunId, binIdx: int, e: FeedbackEvent) = + inc accum[gunId].nall + if e.hit: inc accum[gunId].hall + if localTick < 100: + inc accum[gunId].n100 + if e.hit: inc accum[gunId].h100 + if localTick < 300: + inc accum[gunId].n300 + if e.hit: inc accum[gunId].h300 + let fireTick = e.fireTick - baseTick + if fireTick < 100: + inc accum[gunId].m100 + if e.hit: inc accum[gunId].f100 + dref[gunId].resultCb(e)) + gDropped += tracker.droppedBullets + result = accum + +proc runFixtureClean(fx: Fixture, path: string, drivers: seq[GunDriver], + metric: BulletMetric): seq[Adapt] = + var spans = + if fx.meta.source == "synthetic": @[(start: 0, count: fx.states.len)] + else: loadRounds(path) + result = newSeq[Adapt](drivers.len) + if spans.len == 0: + result = runRound(fx.states, fx.lastSeen, fx.enemyId, 0, drivers, metric) + return + for sp in spans: + var st: seq[WorldState] + var ls: seq[int] + for i in 0..= sp.start and t < sp.start + sp.count: + st.add fx.states[i] + ls.add(if i < fx.lastSeen.len: fx.lastSeen[i] else: -1) + if st.len == 0: continue + let rr = runRound(st, ls, fx.enemyId, sp.start, drivers, metric) + for gi in 0..B,nB>A,ties,p" + for a in arms: + if a.name in ["Linear", "Pattern"]: continue + var winsA, winsB, ties, n = 0 + for name in names: + let (_, path) = resolve(name) + let fxName = path.extractFilename.replace(".jsonl", "") + var ha, na, hb, nb: int + for row in rows: + if row.fixture != fxName: continue + if row.arm == a.name: ha += row.r.hall; na += row.r.nall + elif row.arm == "Pattern": hb += row.r.hall; nb += row.r.nall + if na == 0 or nb == 0: continue + let ra = ha.float / na.float + let rb = hb.float / nb.float + inc n + if ra > rb: inc winsA + elif rb > ra: inc winsB + else: inc ties + echo &"{a.name},overall,{n},{winsA},{winsB},{ties},{signTestP(winsA, n - ties):.4f}" + + if gDropped > 0: + echo &"\n# WARNING: droppedBullets={gDropped}" + +when isMainModule: + main() diff --git a/common_libs/tests/test_pattern_radial_offset.nim b/common_libs/tests/test_pattern_radial_offset.nim new file mode 100644 index 0000000..d70d57d --- /dev/null +++ b/common_libs/tests/test_pattern_radial_offset.nim @@ -0,0 +1,140 @@ +## Guard for the Pattern gun's radial-offset knob (`TR_PATTERN_RAD_OFFSET` / +## `TR_PATTERN_RAD_SCALE`). +## +## Proves the two things the shipped-default contract requires: +## 1. BYTE-IDENTITY — an unconfigured gun (env unset, the shipped path) and a +## gun explicitly set to (1.0, 0.0) produce EXACTLY the same predictions as +## a gun whose knob is never touched, over a real DrussGT fixture. The +## (1.0, 0.0) case short-circuits to the raw aim point, so this is a direct +## proof the shipped prediction is unchanged by the knob. +## 2. ENV — `TR_PATTERN_RAD_SCALE=x` scales only the aim DISTANCE, and +## `TR_PATTERN_RAD_OFFSET=px` shifts only the aim DISTANCE; the BEARING +## (the direction the real gun is pointed, `aimAngle(self, pred)`) is +## unchanged. This is the structural reason a radial offset cannot change +## the real shot direction. +## +## Run: nim c -r --path:common_libs common_libs/tests/test_pattern_radial_offset.nim + +import std/[math, os, strformat] +import gun_harness/offline_range +import gun_harness/virtual_bullets +import guns/pattern_matcher + +const + repoRoot = currentSourcePath().parentDir.parentDir.parentDir + fixture = repoRoot / "tools" / "fixtures" / "drussgt_vs_crazy.jsonl" + +var failures = 0 +proc check(name: string, ok: bool) = + if ok: echo "PASS: ", name + else: echo "FAIL: ", name; inc failures + +proc clearEnv() = + delEnv("TR_PATTERN_RAD_OFFSET") + delEnv("TR_PATTERN_RAD_SCALE") + +proc bearingsClose(a, b: GunPrediction, sx, sy: float): bool = + let ba = arctan2(a.y - sy, a.x - sx) + let bb = arctan2(b.y - sy, b.x - sx) + var d = ba - bb + while d > PI: d -= 2.0 * PI + while d < -PI: d += 2.0 * PI + abs(d) < 1e-12 + +proc dist(p: GunPrediction, sx, sy: float): float = + hypot(p.x - sx, p.y - sy) + +when isMainModule: + let fx = loadFixture(fixture) + # First 600 states is plenty to exercise the linear fallback AND the pattern + # path (history + a match) without a long compile/run. + var states = fx.states + if states.len > 600: states.setLen 600 + + # ── 1. default-path byte-identity ───────────────────────────────────────── + clearEnv() + var plain = PatternMatcherGun() # never configured -> env path (1.0, 0.0) + var explicit = PatternMatcherGun() + explicit.setRadialCorrection(1.0, 0.0) # explicit shipped-default values + + var identical = true + var nPred = 0 + for si in 0.. 1e-6: + inc scaledPoints + if abs(ds - d0 * 0.95) > 1e-6: distOK = false + if not bearingsClose(p0, ps, s.selfX, s.selfY): bearingOK = false + else: + if p0.x != ps.x or p0.y != ps.y: distOK = false + check &"env scale: scale=0.95 gives dist*0.95 on {scaledPoints} valid preds", distOK + check "env scale: bearing unchanged (real aim angle invariant)", bearingOK + + # ── 3. env offset shifts distance only, keeps bearing ───────────────────── + clearEnv() + var h0 = PatternMatcherGun() + h0.setRadialCorrection(1.0, 0.0) + putEnv("TR_PATTERN_RAD_OFFSET", "-20") + var hO = PatternMatcherGun() + var offOK = true + var offBearingOK = true + var offPoints = 0 + for si in 0.. 20.0: + inc offPoints + if abs(dof - (d0 - 20.0)) > 1e-6: offOK = false + if not bearingsClose(p0, po, s.selfX, s.selfY): offBearingOK = false + check &"env offset: offset=-20 gives dist-20 on {offPoints} valid preds", offOK + check "env offset: bearing unchanged (real aim angle invariant)", offBearingOK + clearEnv() + + # ── 4. unparsable env falls back to the shipped defaults ────────────────── + putEnv("TR_PATTERN_RAD_SCALE", "banana") + putEnv("TR_PATTERN_RAD_OFFSET", "nope") + var gBad = PatternMatcherGun() + discard gBad.predict(states[0], bulletSpeed(PowerBins[0])) + clearEnv() + var gDef = PatternMatcherGun() + discard gDef.predict(states[0], bulletSpeed(PowerBins[0])) + var fallbackOK = true + for si in 0.. 0: + echo "\n", failures, " check(s) FAILED" + quit(1) + echo "\nAll Pattern radial-offset checks passed."