From 140fe2519a639d32a34cfcb1db9220ecf38f465b Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Thu, 24 Sep 2026 23:52:49 +0200 Subject: [PATCH] HeadOn (no-lead) vs Pattern LIVE at long range: clean negative, offline ruler killed 2 arms x 15 runs x 7 rounds, one frozen binary from HEAD a82c864, real DrussGT, server-side events sidecar. Shipped rack is onlyPattern, so control=Pattern-only and headon=HeadOn-only (TR_RACK_PATTERN=off TR_RACK_HEADON=both). arm dmg/run dmgtk/run round wins shots/run control 279 211 48/105 785 headon 14 228 0/105 580 Round wins and dmg/run both separate at p<0.0001 (MC permutation, se 0.0000), ~7x the damage MDE (35.8). Per range band (pooled, 15 runs): 300-450: Pattern 12.3% (4590 shots) vs HeadOn 0.6% (3701) p<0.0001, MDE 2.0pp 450+ : Pattern 9.2% (6671) vs HeadOn 0.4% (4177) p<0.0001, MDE 1.1pp HeadOn loses EVERY long-range band by 20-23x, so the whole-battle loss is not a close-range artefact. The offline ruler (prediction_quality_results.txt) predicted the opposite: HeadOn meanAbs 14.61 vs Pattern 17.53 at 300-450 and 12.33 vs 16.19 at 450+, hitProxy .105/.104 and .098/.077 (+27%). That is an open-loop replay of a FIXED enemy track, so it cannot see that a different bullet makes the surfer dodge differently; live, the static gun does not lead at all. TR_PATTERN_RAD_SCALE arms were skipped: applyRadial scales aim DISTANCE along an unchanged bearing, so it cannot express 'less lead' (bearing is what firing uses). HeadOn confirmed to ignore bulletSpeed (head_on.nim:9), liveness OK 15/15. Adds the range-band analyzer tools/ab/ab_range_bands.py (reuses the lead-capture Run alignment) and the captured fixtures. Does not touch bitbrain_gun.nim / bitbrain_campaign.md (job-100). --- .../fixtures/headon_longrange_ab_report.txt | 45 ++++ .../tests/fixtures/headon_longrange_bands.txt | 42 ++++ .../headon_longrange_live_results.json | 199 ++++++++++++++++++ docs/headon_longrange_live.md | 109 ++++++++++ tools/ab/ab_range_bands.py | 182 ++++++++++++++++ tools/ab/arms_headon.txt | 6 + 6 files changed, 583 insertions(+) create mode 100644 common_libs/tests/fixtures/headon_longrange_ab_report.txt create mode 100644 common_libs/tests/fixtures/headon_longrange_bands.txt create mode 100644 common_libs/tests/fixtures/headon_longrange_live_results.json create mode 100644 docs/headon_longrange_live.md create mode 100755 tools/ab/ab_range_bands.py create mode 100644 tools/ab/arms_headon.txt diff --git a/common_libs/tests/fixtures/headon_longrange_ab_report.txt b/common_libs/tests/fixtures/headon_longrange_ab_report.txt new file mode 100644 index 0000000..9eadde4 --- /dev/null +++ b/common_libs/tests/fixtures/headon_longrange_ab_report.txt @@ -0,0 +1,45 @@ +# session /tmp/ab_headon +# commit=a82c864c6075d8325589681bae23b371e3c4f1b5 binary_sha256=9d20f530d633982b30735d5ece38da09b707737ae1c57238ee41abe928276601 rounds=7 runs=15 conc=8 ts=2026-09-24T23:45:01+02:00 + +ARM SUMMARY +arm runs dmg/run dmgtk/run wins win% shots/run hitstk/run +-------------------------------------------------------------------------- +control 15 279 211 48/105 45.7 785 92.9 +headon 15 14 228 0/105 0.0 580 72.1 + +PER-RUN (never just the mean) + control dmg: r1=263 r2=307 r3=294 r4=221 r5=290 r6=235 r7=327 r8=262 r9=330 r10=312 r11=274 r12=292 r13=288 r14=281 r15=217 + wins: r1=2/7 r2=3/7 r3=4/7 r4=2/7 r5=4/7 r6=3/7 r7=3/7 r8=2/7 r9=6/7 r10=5/7 r11=2/7 r12=4/7 r13=2/7 r14=4/7 r15=2/7 + headon dmg: r1=4 r2=8 r3=4 r4=17 r5=14 r6=10 r7=28 r8=18 r9=0 r10=36 r11=17 r12=8 r13=14 r14=14 r15=15 + wins: r1=0/7 r2=0/7 r3=0/7 r4=0/7 r5=0/7 r6=0/7 r7=0/7 r8=0/7 r9=0/7 r10=0/7 r11=0/7 r12=0/7 r13=0/7 r14=0/7 r15=0/7 + +PAIRWISE PERMUTATION TEST (per-run values) + MANN-WHITNEY CROSS-CHECK +permutation: exact when C(n,na) <= 20,000,000; otherwise Monte-Carlo 1,000,000 draws, seed=0x5eed5eed, p = (cnt+1)/(B+1), se = sqrt(p(1-p)/(B+1)) +metric A B diff(A-B) perm p method MC se MW p MW U +------------------------------------------------------------------------------------------------- +dmg/run control headon +265.670 0.0000 MC/B=1,000,000 0.0000 0.0000 0.0 +round wins control headon +3.200 0.0000 MC/B=1,000,000 0.0000 0.0000 0.0 + +MINIMUM DETECTABLE EFFECT (two-sample, alpha=0.05 two-sided, 80% power; MDE = 2.8016*sd*sqrt(2/n)) +metric n/arm sd(control) MDE(abs) MDE vs control mean +---------------------------------------------------------------- +dmg/run 15 34.956 35.760 12.8% of 279.5 +round wins 15 1.265 1.294 40.4% of 3.2 + +ROUND-LEVEL TEST (pooled rounds, Fisher exact) vs `control` — ANTI-CONSERVATIVE: rounds cluster within runs +arm ref wins arm wins p +---------------------------------------------- +headon 48/105 0/105 0.0000 + +LIVENESS (arm env applied in the bot's own boot report) + control OK (15/15 runs: no arm env; report present) + headon OK (15/15 runs: TR_RACK_PATTERN=off TR_RACK_HEADON=both applied) + +[bb] APPLIED-SHIFT CHECK (from bot stdout; needs TR_BITBRAIN_LOG=1). A provably-zero placebo emits ZERO [bb] lines. + arm runs w/log lines min max zeros + control 0/15 0 - - - + headon 0/15 0 - - - + +ROUND-WIN ATTRIBUTION (events primary; score tie-break for mutual-kill / timeout rounds) + control wins==firstPlaces 15/15 runs OK; single-death rounds agree with score 102/102 (3 tie-broken) + headon wins==firstPlaces 15/15 runs OK; single-death rounds agree with score 105/105 (0 tie-broken) diff --git a/common_libs/tests/fixtures/headon_longrange_bands.txt b/common_libs/tests/fixtures/headon_longrange_bands.txt new file mode 100644 index 0000000..a650e29 --- /dev/null +++ b/common_libs/tests/fixtures/headon_longrange_bands.txt @@ -0,0 +1,42 @@ +==================================================================================================== +HIT RATE BY RANGE BAND (our shots; band = shooter->target distance px at the fire tick) +session: /tmp/ab_headon arms: control, headon reference: control +==================================================================================================== +band control headon +-------------------------------------------------------------- +0-100 3 2 66.7% 1 0 0.0% +100-200 28 4 14.3% 97 8 8.2% +200-300 211 26 12.3% 469 12 2.6% +300-450 4590 565 12.3% 3701 21 0.6% +450+ 6671 616 9.2% 4177 15 0.4% +ALL 11503 1213 10.5% 8445 56 0.7% + +PER-RUN BAND RATES (shows the spread behind the pooled numbers) + band 0-100 + control n= 2 50 100 + headon n= 1 0 + band 100-200 + control n=11 0 0 0 20 0 0 0 0 30 0 0 + headon n=15 0 30 0 0 0 0 0 0 0 14 11 0 17 25 0 + band 200-300 + control n=15 8 17 40 0 40 17 20 10 9 14 15 0 5 12 11 + headon n=15 3 4 6 0 4 0 4 3 0 4 4 3 3 0 0 + band 300-450 + control n=15 14 11 15 13 13 10 11 15 15 10 10 11 14 11 13 + headon n=15 0 0 1 1 0 0 0 0 0 1 0 1 1 1 0 + band 450+ + control n=15 9 11 7 8 9 10 9 10 10 8 11 9 10 9 9 + headon n=15 0 0 0 0 1 1 1 0 0 0 0 0 1 0 0 + band ALL + control n=15 10 11 9 10 11 10 10 12 12 9 11 10 12 10 11 + headon n=15 0 1 1 0 1 1 1 1 0 1 1 1 1 1 0 + +PER-BAND PERMUTATION TEST vs `control` (per-run rates, two-sided) +band arm d(pp) p method MCse MDE(pp) +------------------------------------------------------------------ +0-100 headon -75.00 0.6667 exact 0.0000 99.05 +100-200 headon +1.93 0.6609 exact 0.0000 12.37 +200-300 headon -12.12 0.0000 MC/B=200,000 0.0000 12.07 +300-450 headon -11.84 0.0000 MC/B=200,000 0.0000 1.96 +450+ headon -8.89 0.0000 MC/B=200,000 0.0000 1.14 +ALL headon -9.90 0.0000 MC/B=200,000 0.0000 1.02 diff --git a/common_libs/tests/fixtures/headon_longrange_live_results.json b/common_libs/tests/fixtures/headon_longrange_live_results.json new file mode 100644 index 0000000..32e0e8d --- /dev/null +++ b/common_libs/tests/fixtures/headon_longrange_live_results.json @@ -0,0 +1,199 @@ +{ + "arms": { + "control": { + "bands_pooled": { + "0-100": { + "hits": 2, + "shots": 3 + }, + "100-200": { + "hits": 4, + "shots": 28 + }, + "200-300": { + "hits": 26, + "shots": 211 + }, + "300-450": { + "hits": 565, + "shots": 4590 + }, + "450+": { + "hits": 616, + "shots": 6671 + }, + "ALL": { + "hits": 1213, + "shots": 11503 + } + }, + "runs": { + "run1.jsonl": { + "hits": 81, + "shots": 781 + }, + "run10.jsonl": { + "hits": 90, + "shots": 819 + }, + "run11.jsonl": { + "hits": 78, + "shots": 822 + }, + "run12.jsonl": { + "hits": 80, + "shots": 781 + }, + "run13.jsonl": { + "hits": 85, + "shots": 754 + }, + "run14.jsonl": { + "hits": 78, + "shots": 811 + }, + "run15.jsonl": { + "hits": 62, + "shots": 628 + }, + "run2.jsonl": { + "hits": 92, + "shots": 750 + }, + "run3.jsonl": { + "hits": 90, + "shots": 751 + }, + "run4.jsonl": { + "hits": 67, + "shots": 761 + }, + "run5.jsonl": { + "hits": 88, + "shots": 830 + }, + "run6.jsonl": { + "hits": 72, + "shots": 725 + }, + "run7.jsonl": { + "hits": 86, + "shots": 731 + }, + "run8.jsonl": { + "hits": 77, + "shots": 774 + }, + "run9.jsonl": { + "hits": 87, + "shots": 785 + } + } + }, + "headon": { + "bands_pooled": { + "0-100": { + "hits": 0, + "shots": 1 + }, + "100-200": { + "hits": 8, + "shots": 97 + }, + "200-300": { + "hits": 12, + "shots": 469 + }, + "300-450": { + "hits": 21, + "shots": 3701 + }, + "450+": { + "hits": 15, + "shots": 4177 + }, + "ALL": { + "hits": 56, + "shots": 8445 + } + }, + "runs": { + "run1.jsonl": { + "hits": 2, + "shots": 590 + }, + "run10.jsonl": { + "hits": 6, + "shots": 604 + }, + "run11.jsonl": { + "hits": 5, + "shots": 550 + }, + "run12.jsonl": { + "hits": 2, + "shots": 522 + }, + "run13.jsonl": { + "hits": 4, + "shots": 616 + }, + "run14.jsonl": { + "hits": 3, + "shots": 584 + }, + "run15.jsonl": { + "hits": 4, + "shots": 549 + }, + "run2.jsonl": { + "hits": 3, + "shots": 561 + }, + "run3.jsonl": { + "hits": 1, + "shots": 552 + }, + "run4.jsonl": { + "hits": 6, + "shots": 570 + }, + "run5.jsonl": { + "hits": 4, + "shots": 567 + }, + "run6.jsonl": { + "hits": 3, + "shots": 520 + }, + "run7.jsonl": { + "hits": 8, + "shots": 605 + }, + "run8.jsonl": { + "hits": 5, + "shots": 588 + }, + "run9.jsonl": { + "hits": 0, + "shots": 467 + } + } + } + }, + "band_labels": [ + "0-100", + "100-200", + "200-300", + "300-450", + "450+" + ], + "session": { + "binary_sha256": "9d20f530d633982b30735d5ece38da09b707737ae1c57238ee41abe928276601", + "commit": "a82c864c6075d8325589681bae23b371e3c4f1b5", + "conc": 8, + "rounds": 7, + "runs": 15, + "timestamp": "2026-09-24T23:45:01+02:00" + } +} \ No newline at end of file diff --git a/docs/headon_longrange_live.md b/docs/headon_longrange_live.md new file mode 100644 index 0000000..df9745e --- /dev/null +++ b/docs/headon_longrange_live.md @@ -0,0 +1,109 @@ +# HeadOn vs Pattern at long range — LIVE gate + +**Date:** 2026-09-24 +**Branch:** `research/lead-targeting` · **commit:** `a82c864` · **binary sha256:** `9d20f530d6…` (one frozen binary, `git archive HEAD`) +**Session:** `/tmp/ab_headon` · 2 arms × 15 runs × 7 rounds = 210 rounds, `conc=8` +**Fixtures committed:** `headon_longrange_ab_report.txt` (ab_analyze), `headon_longrange_bands.txt` +(range-band), `headon_longrange_live_results.json` (per-run + per-band counts). Arms: `tools/ab/arms_headon.txt`. + +## Direct answer + +**NO. The no-lead static gun does not beat Pattern live, overall or in any long-range band.** It is +not close: HeadOn won **0 of 105 rounds** and dealt **14 dmg/run vs 279** (20× less), and at +**450+ px it hits 0.4% vs Pattern's 9.2%** (23× less). The offline ruler's headline — HeadOn has +lower aim error and a *better* hit proxy than Pattern at 300+ px — is **killed**. The offline ruler +was the only thing that predicted the opposite; this live A/B is the only thing that can confirm or +kill it, and it killed it. + +## The `TR_PATTERN_RAD_SCALE` arms were skipped (the knob cannot express the lead hypothesis) + +`common_libs/guns/pattern_matcher.nim` `applyRadial` moves the aim point **along the unchanged +bearing** `(selfX + dx/d*nd, selfY + dy/d*nd)` — the direction `dx/d, dy/d` is untouched, only the +distance changes. The fired bearing is `aimAngle(self, pred)` (`ModularBot.nim:1208`), so scaling the +aim distance cannot move the bearing. The knob is a **radial range correction**, not a lead +multiplier, and cannot express "less lead". Per the task rule, arms `rad075`/`rad050` were **skipped** +and the experiment ran as **2 arms × 15 runs**. + +## HeadOn really does ignore `bulletSpeed` + +`common_libs/guns/head_on.nim:9` returns `GunPrediction(x: state.enemyX, y: state.enemyY)`, never +reading its `bulletSpeed` argument. The rack calls it for the selected power bin and the fire block +takes the bearing straight to that point. So the premise holds: HeadOn applies **zero lead**, +independent of power. The `headon` arm's liveness check passed (its env reached the boot report in +15/15 runs), so this was HeadOn and only HeadOn. + +## Primary result — damage/run and round wins [MEASURED] + +| arm | runs | dmg/run | dmgtk/run | round wins | win% | shots/run | hits taken/run | +|---|---:|---:|---:|---:|---:|---:|---:| +| `control` (shipped Pattern-only) | 15 | **279** | 211 | **48/105** | 45.7 | 785 | 93 | +| `headon` (HeadOn-only) | 15 | **14** | 228 | **0/105** | 0.0 | 580 | 72 | + +Per-run damage (`control`): 263 307 294 221 290 235 327 262 330 312 274 292 288 281 217. +Per-run damage (`headon`): 4 8 4 17 14 10 28 18 0 36 17 8 14 14 15. + +| metric | diff (control−headon) | perm p | method | MDE (abs) | MDE vs control mean | +|---|---:|---:|---|---:|---:| +| dmg/run | **+265.7** | **<0.0001** | MC/B=1,000,000 (se 0.0000) | 35.8 | 12.8% of 279.5 | +| round wins | **+3.20** | **<0.0001** | MC/B=1,000,000 (se 0.0000) | 1.29 | 40.4% of 3.2 | + +Mann-Whitney U cross-check p = 0.0000 on both. Round-level Fisher p < 0.0001 (anti-conservative). +The effect is ~7× the damage MDE; this is not an under-powered null. + +## Hit rate by range band — the load-bearing breakdown [MEASURED] + +Pooled over each arm's 15 runs; band = shooter→target distance at the fire tick (same bands as the +offline ruler). HeadOn loses the long-range bands by a landslide — the whole-battle defeat is **not** +a close-range artefact. + +| band (px) | control shots / hits / rate | headon shots / hits / rate | +|---|---:|---:| +| 0–100 | 3 / 2 / 66.7% | 1 / 0 / 0.0% | +| 100–200 | 28 / 4 / 14.3% | 97 / 8 / 8.2% | +| 200–300 | 211 / 26 / 12.3% | 469 / 12 / 2.6% | +| **300–450** | **4590 / 565 / 12.3%** | **3701 / 21 / 0.6%** | +| **450+** | **6671 / 616 / 9.2%** | **4177 / 15 / 0.4%** | +| **ALL** | 11503 / 1213 / **10.5%** | 8445 / 56 / **0.7%** | + +Per-band permutation test on per-run band rates (`arm − control`): + +| band | diff (pp) | p | method | MDE (pp) | +|---|---:|---:|---|---:| +| 100–200 | +1.93 | 0.6609 | exact | 12.4 | +| 200–300 | −12.12 | <0.0001 | MC/B=200,000 | 12.1 | +| 300–450 | **−11.84** | **<0.0001** | MC/B=200,000 | 2.0 | +| 450+ | **−8.89** | **<0.0001** | MC/B=200,000 | 1.1 | +| ALL | −9.90 | <0.0001 | MC/B=200,000 | 1.0 | + +The two long-range bands separate at **4–8× their MDE**, overwhelmingly in HeadOn's *favour* being +false. Note the 98%+ of shots that land at 300+ px, exactly as the offline ruler reported. + +## Offline ruler vs live [MEASURED] + +| source | metric | Pattern 300–450 | HeadOn 300–450 | Pattern 450+ | HeadOn 450+ | +|---|---|---:|---:|---:|---:| +| offline `prediction_quality_results.txt` | meanAbs err (deg) | 17.53 | **14.61** | 16.19 | **12.33** | +| offline | hitProxy | 0.104 | **0.105** | 0.077 | **0.098 (+27%)** | +| **live (this run)** | **hit rate** | **12.3%** | **0.6%** | **9.2%** | **0.4%** | + +The ruler said HeadOn is equal-or-better at 300+; live it is 20×/23× worse. The ruler is a per-tick, +open-loop replay of a **fixed** enemy track: a no-lead gun's aim error is scored against the enemy's +true future position on that same pre-recorded dodge, which is not the distribution a *different* +bullet induces. Live, the bullet changes the dodge, and the static gun simply does not lead the +surfer. This is precisely the open-loop/live gap the ruler cannot see. **The offline aim-error ruler +is not a valid predictor of the live HeadOn-vs-Pattern question — do not use it to rank no-lead guns +at range.** + +## Caveats and what is inferred + +- **[MEASURED]** All numbers above, from this session's server-side events sidecar; one frozen + binary; liveness OK for both arms; `wins == firstPlaces` 15/15 both arms. +- **[INFERRED]** The rack-membership change also removes a gun's 4 virtual bullets/tick from the + shared order-sensitive ring (`docs/gun_rack_analysis.md` §1.1). Both arms are single-gun racks and + the effect size (0/105 rounds, ~7× MDE) is far too large for ring-order to explain. +- **[MEASURED]** One adversary (the real DrussGT jar). The 70-run offline corpus this ruler was built + on is the same live capture set, so the two disagree on the *same* adversary — the disagreement is + the open-loop/live gap, not a matchup difference. +- **Reproduce:** `tools/ab/ab_run.sh --arms tools/ab/arms_headon.txt --runs 15 --outdir /tmp/ab_headon --conc 8 --rounds 7`; + `python3 tools/ab/ab_analyze.py /tmp/ab_headon --reference control`; + `python3 tools/ab/ab_range_bands.py /tmp/ab_headon --reference control`. diff --git a/tools/ab/ab_range_bands.py b/tools/ab/ab_range_bands.py new file mode 100755 index 0000000..819e3dd --- /dev/null +++ b/tools/ab/ab_range_bands.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""ab_range_bands.py — hit rate BY RANGE BAND per arm, from an ab_run.sh session. + + python3 tools/ab/ab_range_bands.py [--reference ARM] + +Why this exists: ab_analyze.py judges damage/run and round wins over the whole +battle, but the HeadOn-vs-Pattern claim is specifically about LONG RANGE. A +whole-battle win that comes only from close range would NOT support it, so the +range breakdown is the load-bearing view. ab_analyze.py does not emit it. + +Method: reuse `Run` from common_libs/tests/analyze_lead_capture_by_range.py, +which aligns each of OUR fire events to the capture tick (owner side resolved +per run, round-startTick + local-tick search) and gives the fire-tick range +(shooter->target distance) plus the server-resolved outcome of that bullet. +A band's hit rate is then hits/shots over that band's shots, pooled across the +arm's runs. A two-sided permutation test on the PER-RUN band rates (only runs +with shots in the band) is printed alongside, so a pooled difference can be +told apart from run-to-run noise; the MDE for those n is printed too. + +Bands match the offline ruler (prediction_quality.nim) and the lead-capture +report: 0-100, 100-200, 200-300, 300-450, 450+ px. +""" +import argparse +import collections +import math +import os +import random +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(_HERE, "..", "..", "common_libs", "tests")) +from analyze_lead_capture_by_range import Run, BAND_LABELS # noqa: E402 + +Z_ALPHA_POWER = 1.959963984540054 + 0.8416212335729143 +MC_SEED = 0x5EED5EED +MC_DRAWS = 200_000 + + +def load_arm(session_dir, arm): + armdir = os.path.join(session_dir, arm) + runs = [] + for fn in sorted(os.listdir(armdir)): + if not fn.endswith(".jsonl") or fn.endswith(".events.jsonl"): + continue + cap = os.path.join(armdir, fn) + ev, rj = cap[:-6] + ".events.jsonl", cap + ".rounds.json" + if os.path.exists(ev) and os.path.exists(rj): + runs.append((fn, cap, ev, rj)) + out = {} + for fn, cap, ev, rj in runs: + try: + r = Run(cap, ev, rj) + except Exception as e: # noqa: BLE001 + print(f" WARN {arm}/{fn}: {e}", file=sys.stderr) + continue + shots = list(r.shots()) + out[fn] = shots + return out + + +def perm_test(xa, xb): + """Two-sided permutation on the difference of means (exact if small).""" + na, nb = len(xa), len(xb) + if na == 0 or nb == 0: + return None + import itertools + obs = abs(sum(xa) / na - sum(xb) / nb) + pooled = list(xa) + list(xb) + n = na + nb + total = sum(pooled) + ncomb = math.comb(n, na) + if ncomb <= 20_000_000: + cnt = 0 + for combo in itertools.combinations(range(n), na): + sa = sum(pooled[i] for i in combo) + if abs(sa / na - (total - sa) / nb) >= obs - 1e-9: + cnt += 1 + return obs, cnt / ncomb, "exact", 0.0 + rng = random.Random(MC_SEED) + B = MC_DRAWS + cnt = 0 + for _ in range(B): + sa = 0 + for i in rng.sample(range(n), na): + sa += pooled[i] + if abs(sa / na - (total - sa) / nb) >= obs - 1e-9: + cnt += 1 + p = (cnt + 1) / (B + 1) + return obs, p, f"MC/B={B:,}", math.sqrt(p * (1 - p) / (B + 1)) + + +def _sd(xs): + m = sum(xs) / len(xs) + return math.sqrt(sum((v - m) ** 2 for v in xs) / (len(xs) - 1)) + + +def banded(shots): + """{band: (shots, hits)}, plus overall.""" + c = collections.Counter() + h = collections.Counter() + for s in shots: + c[s["band"]] += 1 + h[s["band"]] += int(s["hit"]) + c["ALL"] += len(shots) + h["ALL"] += int(sum(s["hit"] for s in shots)) + return c, h + + +def per_run_band_rate(runs, band): + """Per-run hit rate for a band, only runs that fired in that band.""" + out = [] + for fn, shots in runs.items(): + sub = shots if band == "ALL" else [s for s in shots if s["band"] == band] + if sub: + out.append(sum(s["hit"] for s in sub) / len(sub)) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("session_dir") + ap.add_argument("--reference", default=None) + args = ap.parse_args() + + arms = [d for d in sorted(os.listdir(args.session_dir)) + if os.path.isdir(os.path.join(args.session_dir, d)) + and not d.startswith(".") and d != "frozen"] + arms = [a for a in arms if any( + f.endswith(".jsonl") and not f.endswith(".events.jsonl") + for f in os.listdir(os.path.join(args.session_dir, a)))] + ref = args.reference or arms[0] + data = {a: load_arm(args.session_dir, a) for a in arms} + + print("=" * 100) + print("HIT RATE BY RANGE BAND (our shots; band = shooter->target distance px at the fire tick)") + print(f"session: {args.session_dir} arms: {', '.join(arms)} reference: {ref}") + print("=" * 100) + labels = BAND_LABELS + ["ALL"] + hdr = f"{'band':<10}" + "".join(f"{a:>26}" for a in arms) + print(hdr) + print("-" * len(hdr)) + for band in labels: + cells = [] + for a in arms: + c, h = banded([s for shots in data[a].values() for s in shots]) + n = c[band] + cells.append(f"{n:>7} {h[band]:>5} {(100*h[band]/n if n else 0):>6.1f}%") + print(f"{band:<10}" + "".join(f"{c:>26}" for c in cells)) + + print("\nPER-RUN BAND RATES (shows the spread behind the pooled numbers)") + for band in labels: + print(f" band {band}") + for a in arms: + rates = per_run_band_rate(data[a], band) + txt = " ".join(f"{100*r:.0f}" for r in rates) + print(f" {a:<10} n={len(rates):>2} {txt}") + + print("\nPER-BAND PERMUTATION TEST vs `" + ref + "` (per-run rates, two-sided)") + print(f"{'band':<10} {'arm':<10} {'d(pp)':>7} {'p':>9} {'method':<12} {'MCse':>7} {'MDE(pp)':>8}") + print("-" * 66) + for band in labels: + xa = per_run_band_rate(data[ref], band) + if len(xa) < 2: + continue + sd = _sd(xa) + mde = Z_ALPHA_POWER * sd * math.sqrt(2.0 / len(xa)) * 100 + for a in arms: + if a == ref: + continue + xb = per_run_band_rate(data[a], band) + res = perm_test(xa, xb) + if res is None: + continue + obs, p, method, se = res + signed = (sum(xb) / len(xb) - sum(xa) / len(xa)) * 100 # arm - ref + print(f"{band:<10} {a:<10} {signed:>+7.2f} {p:>9.4f} {method:<12} " + f"{se:>7.4f} {mde:>8.2f}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ab/arms_headon.txt b/tools/ab/arms_headon.txt new file mode 100644 index 0000000..8519967 --- /dev/null +++ b/tools/ab/arms_headon.txt @@ -0,0 +1,6 @@ +# HeadOn-vs-Pattern long-range live gate (2 arms x 15 runs x 7 rounds). +# Shipped default is `onlyPattern` (Pattern is the only admitted rack gun), so: +# control = shipped Pattern-only rack, no env +# headon = Pattern off, HeadOn re-admitted -> HeadOn-only rack +control | +headon | TR_RACK_PATTERN=off TR_RACK_HEADON=both | static no-lead gun only