#!/usr/bin/env python3 """Analyze the which-gun A/B from server-side events sidecars. Ground truth: real hit rate from the events sidecar (fire/hit, server-side). Per-run rates -> exact two-sided permutation test (enumerated when small). Also reads gun_stats.jsonl to prove each arm was LIVE (selected-gun mix). """ import json, glob, os, re, sys, collections, itertools, math OUT = os.environ.get("WHICHGUN_OUT", "/tmp/whichgun") ARMS = ["full", "onlyLinear", "onlyPattern", "onlyGF", "onlyKNN", "lean8", "lean6", "pairPC", "pairPK", "pairPL"] BASELINES = ["full", "onlyPattern"] MB_POWERS = {1.0, 1.5, 2.0, 3.0} def runs_for(arm): rs = set() for f in glob.glob(f"{OUT}/events_{arm}_r*.json"): m = re.search(rf"_{re.escape(arm)}_r(\d+)\.json$", f) if m: rs.add(int(m.group(1))) return sorted(rs) def parse_subject_fired(logpath): if not os.path.exists(logpath): return None for line in open(logpath, errors="replace"): m = re.search(r"subject event counts:.*bulletsFired=(\d+)", line) if m: return int(m.group(1)) return None def ev_stats(path, subject_fired): fires = collections.Counter() hits = collections.Counter() dmg = collections.Counter() powers = collections.defaultdict(set) for line in open(path): line = line.strip() if not line: continue try: o = json.loads(line) except json.JSONDecodeError: continue # partial line from an in-progress battle t = o.get("type") if t == "fire": fires[o["owner"]] += 1 powers[o["owner"]].add(round(o.get("power", -1), 3)) elif t == "hit": hits[o["owner"]] += 1 dmg[o["owner"]] += o.get("damage", 0.0) # ModularBot = owner whose power set is a subset of the discrete bins mb = None for own, pw in powers.items(): if fires[own] > 0 and pw <= MB_POWERS: mb = own if mb is None: # fallback: owner that is NOT the subject (identified by bulletsFired) subj = None if subject_fired is not None: for own, n in fires.items(): if n == subject_fired: subj = own cand = [o for o in fires if o != subj] if len(cand) == 1: mb = cand[0] if mb is None: return None return fires[mb], hits[mb], dmg[mb] def gun_mix(arm): """Sum selected/realShots/realHits per gun across all rounds of the arm.""" mix = collections.defaultdict(lambda: collections.Counter()) rounds = 0 for r in runs_for(arm): p = f"{OUT}/gun_stats_{arm}_r{r}.jsonl" if not os.path.exists(p): continue for line in open(p): line = line.strip() if not line: continue o = json.loads(line) rounds += 1 for g in o["guns"]: a = mix[g["name"]] a["selected"] += g["selected"] a["realShots"] += g["realShots"] a["realHits"] += g["realHits"] return mix, rounds def analyze(arm, verbose=False): per = [] totF = totH = 0 totD = 0.0 for r in runs_for(arm): ev = ev_stats(f"{OUT}/events_{arm}_r{r}.json", parse_subject_fired(f"{OUT}/battle_{arm}_r{r}.log")) if not ev: if verbose: print(f" r{r}: no events") continue f, h, d = ev rate = 100.0 * h / max(1, f) per.append({"run": r, "fires": f, "hits": h, "rate": rate, "dmg": d}) totF += f totH += h totD += d if verbose: print(f" r{r}: {h}/{f} = {rate:.2f}% dmg={d:.0f}") return dict(arm=arm, per=per, shots=totF, hits=totH, dmg=totD, rate=100.0 * totH / max(1, totF), n=len(per)) def perm_test_exact(a, b): """Exact two-sided permutation test on per-run rates (difference of means).""" xa = [p["rate"] for p in a["per"]] xb = [p["rate"] for p in b["per"]] na, nb = len(xa), len(xb) if na == 0 or nb == 0: return float("nan"), float("nan") obs = abs(sum(xa) / na - sum(xb) / nb) pooled = xa + xb n = na + nb cnt = 0 total = 0 # exact enumeration of C(n, na); fall back to sampling for large n if math.comb(n, na) <= 200000: for combo in itertools.combinations(range(n), na): s = set(combo) ma = sum(pooled[i] for i in combo) / na mb = sum(pooled[i] for i in range(n) if i not in s) / nb if abs(ma - mb) >= obs - 1e-12: cnt += 1 total += 1 return obs, cnt / total # sampled import random rng = random.Random(12345) B = 200000 for _ in range(B): idx = rng.sample(range(n), na) s = set(idx) ma = sum(pooled[i] for i in idx) / na mb = sum(pooled[i] for i in range(n) if i not in s) / nb if abs(ma - mb) >= obs - 1e-12: cnt += 1 return obs, (cnt + 1) / (B + 1) def fmt_range(per): if not per: return "n/a" rs = [p["rate"] for p in per] return f"{min(rs):.2f}-{max(rs):.2f}" def main(): arms = [a for a in ARMS if runs_for(a)] extra = sys.argv[1:] if extra: arms = extra res = {a: analyze(a, verbose=("-v" in sys.argv)) for a in arms} print("=" * 96) print("REAL HIT RATE (server-side events sidecar) — ModularBot vs real DrussGT") print("=" * 96) hdr = f"{'arm':<12} {'runs':>4} {'shots':>6} {'hits':>5} {'real %':>7} {'dmg/run':>8} {'per-run range':>14}" print(hdr) print("-" * len(hdr)) for a in arms: r = res[a] print(f"{a:<12} {r['n']:>4} {r['shots']:>6} {r['hits']:>5} {r['rate']:>7.2f} " f"{r['dmg']/max(1,r['n']):>8.0f} {fmt_range(r['per']):>14}") print("\nper-run rates:") for a in arms: pr = " ".join(f"r{p['run']}={p['rate']:.2f}" for p in res[a]["per"]) print(f" {a:<12} {pr}") for base in BASELINES: if base not in res or len(arms) < 2: continue print("\n" + "=" * 96) print(f"EXACT TWO-SIDED PERMUTATION TEST vs `{base}` (per-run rates)") print("=" * 96) for a in arms: if a == base: continue obs, p = perm_test_exact(res[base], res[a]) print(f" {base} vs {a:<12}: mean-rate diff = {obs:+.2f} pp, exact two-sided p = {p:.4f}") print("\n" + "=" * 96) print("LIVENESS — selected-gun mix (sum of per-tick `selected` over all rounds)") print("=" * 96) for a in arms: mix, rounds = gun_mix(a) tot_sel = sum(v["selected"] for v in mix.values()) parts = [] for name, v in sorted(mix.items(), key=lambda kv: -kv[1]["selected"]): if v["selected"] == 0: continue share = 100.0 * v["selected"] / max(1, tot_sel) parts.append(f"{name} {share:.1f}% (sel={v['selected']}, real={v['realHits']}/{v['realShots']})") print(f" {a:<12} rounds={rounds:>3} " + ("; ".join(parts) if parts else "NO SELECTION RECORDED")) if __name__ == "__main__": main()