j124: spinner + fair-melee results doc; analyzer convergence tests

tools/ab/spinner_analyze.py

@@ -363,6 +363,30 @@ def main():
                 f"| {statistics.mean([e['d_hit'] for e in es]):+.2f} |")
     out()
 
+    # ── spinner-only aggregation (the sub-claim's own field) ────────────────
+    out("### MEASURED: spinner-only paired stats (the sub-claim's own field)")
+    out()
+    out("| arm | metric | mean Δ | spread (SD) | 95% CI | sign test | p(sign) | p(sign-flip) | MDE |")
+    out("|---|---|---:|---:|---|---:|---:|---:|---:|")
+    for a in arms:
+        if a == ref:
+            continue
+        for key, mkey in (("d_damage", "damage"), ("d_wins", "wins"),
+                          ("d_hit", "our hit rate (pp)")):
+            ds = [per_opp[a][o][key] for o in spinner_opps
+                  if per_opp[a][o] and not math.isnan(per_opp[a][o][key])]
+            if len(ds) < 2:
+                continue
+            desc = ga.describe(ds)
+            pos, neg, ties, p_sign = ga.sign_test(ds)
+            sf = ga.signflip_perm(ds)
+            out(f"| `{a}` | {mkey} | {desc['mean']:+.2f} | {desc['sd']:.2f} "
+                f"| [{desc['mean'] - 1.96 * desc['se']:+.2f}, "
+                f"{desc['mean'] + 1.96 * desc['se']:+.2f}] "
+                f"| {pos}/{pos + neg} | {p_sign:.4g} | {sf['p']:.4g} "
+                f"| {desc['mde']:.2f} |")
+    out()
+
     # ── CONVERGENCE ─────────────────────────────────────────────────────────
     out("### MEASURED: CONVERGENCE — our per-round hit rate")
     out()
@@ -447,6 +471,88 @@ def main():
             out(f"| `{a}` | n/a | n/a | n/a |")
     out()
 
+    # per-RUN convergence deltas + a two-sample permutation test
+    def conv_run_deltas(arm):
+        ds = []
+        for o in spinner_opps:
+            for r in data[o][arm]:
+                if 1 not in r["per_round"]:
+                    continue
+                h1 = r["per_round"][1]["mb_hits"]
+                f1 = r["per_round"][1]["mb_fired"]
+                hl = sum(d["mb_hits"] for i, d in r["per_round"].items() if i >= 2)
+                fl = sum(d["mb_fired"] for i, d in r["per_round"].items() if i >= 2)
+                if f1 and fl:
+                    ds.append(100.0 * hl / fl - 100.0 * h1 / f1)
+        return ds
+
+    def within_run_deltas(arm):
+        ds = []
+        for o in spinner_opps:
+            adir = os.path.join(session_dir, o, arm)
+            for run in discover_runs(adir):
+                evs = ga.parse_events(os.path.join(adir, f"run{run}.events.jsonl"))
+                rpath = os.path.join(adir, f"run{run}.jsonl.rounds.json")
+                starts = read_round_starts(rpath)
+                counters = ga.parse_counters("".join(ga.read_lines(
+                    os.path.join(adir, f"run{run}.battle.log"))))
+                if counters is None or not starts:
+                    continue
+                subj, _ = ga.attribute_subject(evs, counters)
+                if subj is None:
+                    continue
+                counts = {rr["round"]: rr["count"]
+                          for rr in json.load(open(rpath))["rounds"]}
+                e2 = [0, 0]
+                l2 = [0, 0]
+                for e in evs:
+                    rnd = e.get("round", 0)
+                    st = starts.get(rnd)
+                    if st is None:
+                        continue
+                    b = e2 if (e.get("tick", 0) - st) < counts.get(rnd, 0) / 2 else l2
+                    if e.get("type") == "fire" and e.get("owner") == subj:
+                        b[1] += 1
+                    elif e.get("type") == "hit" and e.get("owner") == subj:
+                        b[0] += 1
+                if e2[1] and l2[1]:
+                    ds.append(100.0 * l2[0] / l2[1] - 100.0 * e2[0] / e2[1])
+        return ds
+
+    out("Per-run adaptation deltas on the true spinners (the claim is about")
+    out("SPEED, so these are per-run deltas, not pooled rates):")
+    out()
+    out("* `conv` = our hit rate over rounds 2..R minus round 1 (cross-round)")
+    out("* `within` = our hit rate in the second half of a round minus the first")
+    out("  half (same round)")
+    out()
+    out("| arm | n | conv mean Δ (pp) | conv SD | within mean Δ (pp) | within SD |")
+    out("|---|---:|---:|---:|---:|---:|")
+    conv_lists = {}
+    within_lists = {}
+    for a in arms:
+        cv = conv_run_deltas(a)
+        wi = within_run_deltas(a)
+        conv_lists[a] = cv
+        within_lists[a] = wi
+        out(f"| `{a}` | {len(cv)} | {statistics.mean(cv):+.2f} | "
+            f"{statistics.pstdev(cv):.2f} | {statistics.mean(wi):+.2f} | "
+            f"{statistics.pstdev(wi):.2f} |")
+    out()
+    out("Two-sample permutation test (200,000 draws, seed 0x5eed5eed) of each")
+    out(f"arm's adaptation delta against `{ref}`:")
+    out()
+    out("| arm | conv Δ − ref Δ (pp) | p(conv) | within Δ − ref Δ (pp) | p(within) |")
+    out("|---|---:|---:|---:|---:|")
+    for a in arms:
+        if a == ref:
+            continue
+        dc = statistics.mean(conv_lists[a]) - statistics.mean(conv_lists[ref])
+        dw = statistics.mean(within_lists[a]) - statistics.mean(within_lists[ref])
+        out(f"| `{a}` | {dc:+.2f} | {perm_two_sample(conv_lists[a], conv_lists[ref]):.4g} "
+            f"| {dw:+.2f} | {perm_two_sample(within_lists[a], within_lists[ref]):.4g} |")
+    out()
+
     # ── verdict ─────────────────────────────────────────────────────────────
     out("### The pre-registered reading")
     out()
@@ -485,6 +591,22 @@ def main():
     return 0
 
 
+def perm_two_sample(xa, xb, draws=MC_DRAWS, seed=MC_SEED):
+    """Two-sided two-sample permutation test on the difference of means."""
+    if len(xa) < 2 or len(xb) < 2:
+        return float("nan")
+    obs = abs(statistics.mean(xa) - statistics.mean(xb))
+    pool = list(xa) + list(xb)
+    na = len(xa)
+    rng = random.Random(seed)
+    cnt = 0
+    for _ in range(draws):
+        rng.shuffle(pool)
+        if abs(statistics.mean(pool[:na]) - statistics.mean(pool[na:])) >= obs - 1e-12:
+            cnt += 1
+    return (cnt + 1) / (draws + 1)
+
+
 def wilcoxon_signed(deltas):
     """Two-sided Wilcoxon signed-rank normal approximation with tie correction."""
     nz = [d for d in deltas if d != 0.0]