diff --git a/common_libs/tests/fixtures/learned_surfer_gate_report.txt b/common_libs/tests/fixtures/learned_surfer_gate_report.txt index 98ccca6..9ee1186 100644 --- a/common_libs/tests/fixtures/learned_surfer_gate_report.txt +++ b/common_libs/tests/fixtures/learned_surfer_gate_report.txt @@ -75,6 +75,47 @@ turn : -0.142, 0.000, 0.105 cross-check with the canonical edges: log-loss(state) 4.9272, log-loss(global) 4.9739, Δ -0.0467, top-1 0.0393 +## F. is the DANGER MAP the module minimises the RIGHT one? (MEASURED) + +shots 54936, base hit rate 9.98% +corr( P(arrival bin) , P(hit | arrival bin) ) over the 31 bins = -0.342 +safest bin by the histogram MASS the mover minimises: bin 1 (mass 1.9%, hit rate 14.1%) +safest bin by the ACTUAL hit rate: bin 23 (mass 3.1%, hit rate 6.8%) + +| bin | P(hit) | P(arrival bin) | +|---:|---:|---:| +| 0 | 9.8% | 2.4% | +| 1 | 14.1% | 1.9% | +| 2 | 13.3% | 2.6% | +| 3 | 10.8% | 2.3% | +| 4 | 8.8% | 2.5% | +| 5 | 7.8% | 2.9% | +| 6 | 7.9% | 3.3% | +| 7 | 8.8% | 3.5% | +| 8 | 9.2% | 3.6% | +| 9 | 9.1% | 3.8% | +| 10 | 9.8% | 3.9% | +| 11 | 11.0% | 4.0% | +| 12 | 10.6% | 3.9% | +| 13 | 11.1% | 4.1% | +| 14 | 9.0% | 4.0% | +| 15 | 9.8% | 4.6% | +| 16 | 9.8% | 3.8% | +| 17 | 9.2% | 3.9% | +| 18 | 10.7% | 3.6% | +| 19 | 8.8% | 3.5% | +| 20 | 8.5% | 3.4% | +| 21 | 7.9% | 3.5% | +| 22 | 7.5% | 3.3% | +| 23 | 6.8% | 3.1% | +| 24 | 8.3% | 2.8% | +| 25 | 7.2% | 2.6% | +| 26 | 11.5% | 2.3% | +| 27 | 13.1% | 2.2% | +| 28 | 17.5% | 2.6% | +| 29 | 16.2% | 2.8% | +| 30 | 10.9% | 3.3% | + ## MEASURED vs INFERRED * MEASURED: every number in this file, produced by the command in the diff --git a/common_libs/tests/learned_surfer_gate.py b/common_libs/tests/learned_surfer_gate.py index e1a106d..b445571 100644 --- a/common_libs/tests/learned_surfer_gate.py +++ b/common_libs/tests/learned_surfer_gate.py @@ -256,6 +256,52 @@ class CountedModel: for k in range(NBINS)] +def danger_alignment(runs): + """Is the map the mover MINIMISES (P of the arrival bin) aligned with the + map it SHOULD minimise (P(hit | arrival bin))? + + The module picks the lowest-predicted-mass bin. If the mass and the hit + probability are anti-correlated, minimising the mass steers INTO the + bullets, and no amount of state conditioning can fix that. + """ + got, hit = [0] * NBINS, [0] * NBINS + n = 0 + for run in runs: + for sh in run.shots(): + if sh["perp_arr"] is None: + continue + t0, rnd = sh["tick"], sh["rnd"] + rs = run.start[rnd] + r0 = run.by_tick.get(t0) + if r0 is None or t0 <= rs: + continue + p0x, p0y, speed = sh["_x"], sh["_y"], sh["speed"] + tb0 = math.atan2(r0["ey"] - p0y, r0["ex"] - p0x) + maxea = math.asin(min(8.0 / speed, 1.0)) + kf = max(1, int(math.ceil( + math.hypot(r0["ex"] - p0x, r0["ey"] - p0y) / speed))) + r = run.by_tick.get(t0 + kf) + if r is None: + continue + off = wrap180(math.atan2(r["ey"] - p0y, r["ex"] - p0x) - tb0) + b = gf_to_bin(max(-1.0, min(1.0, off / maxea))) + got[b] += 1 + hit[b] += sh["hit"] + n += 1 + mass = [g / n for g in got] + rate = [(hit[b] / got[b]) if got[b] else float("nan") for b in range(NBINS)] + used = [b for b in range(NBINS) if got[b] > 0] + corr = statistics.correlation([mass[b] for b in used], + [rate[b] for b in used]) + base = sum(hit) / n + safest_mass = min(used, key=lambda b: mass[b]) + safest_hit = min(used, key=lambda b: rate[b]) + return dict(n=n, base_hit=base, corr=corr, + safest_mass=(safest_mass, mass[safest_mass], rate[safest_mass]), + safest_hit=(safest_hit, mass[safest_hit], rate[safest_hit]), + rate=[(b, rate[b], mass[b]) for b in used]) + + def log2(x): return math.log2(max(x, 1e-12)) @@ -569,6 +615,26 @@ def main(): f"top-1 {fixed_agg['state_top1']:.4f}") out() + out("## F. is the DANGER MAP the module minimises the RIGHT one? " + "(MEASURED)") + out() + al = danger_alignment(runs) + out(f"shots {al['n']}, base hit rate {al['base_hit']*100:.2f}%") + out(f"corr( P(arrival bin) , P(hit | arrival bin) ) over the 31 bins = " + f"{al['corr']:+.3f}") + b, m, r = al["safest_mass"] + out(f"safest bin by the histogram MASS the mover minimises: bin {b} " + f"(mass {m*100:.1f}%, hit rate {r*100:.1f}%)") + b, m, r = al["safest_hit"] + out(f"safest bin by the ACTUAL hit rate: bin {b} " + f"(mass {m*100:.1f}%, hit rate {r*100:.1f}%)") + out("") + out("| bin | P(hit) | P(arrival bin) |") + out("|---:|---:|---:|") + for b, r, m in al["rate"]: + out(f"| {b} | {r*100:.1f}% | {m*100:.1f}% |") + out() + out("## MEASURED vs INFERRED") out() out("* MEASURED: every number in this file, produced by the command in the") diff --git a/docs/movement_campaign.md b/docs/movement_campaign.md index b81c050..86c5cf6 100644 --- a/docs/movement_campaign.md +++ b/docs/movement_campaign.md @@ -1835,3 +1835,32 @@ better than the global histogram, the unconditional average and chance, with a consistent cross-battle sign. But it clears the bar by ~1% of a bit, so the live panel is the decider, and the pre-registered prediction above is that the module will NOT beat `strafe`. + +### Gate A, second half — is the danger map the module minimises the RIGHT one? + +This is the diagnosis of *why* the offline skill is tiny, and it is +independent of the learner. The mover minimises **P(arrival bin)**. The +quantity it *should* minimise is **P(hit | arrival bin)**. Measured on the same +54 936 shots (gate report section F): + +| quantity | value | +|---|---| +| base hit rate | 9.98% | +| **corr( P(arrival bin), P(hit | arrival bin) )** over the 31 bins | **−0.342** | +| safest bin by the MASS the mover minimises | bin 1 — mass 1.9%, **hit rate 14.1%** | +| safest bin by the ACTUAL hit rate | bin 23 — mass 3.1%, hit rate 6.8% | + +**The histogram the surfer minimises is NEGATIVELY correlated with the hit +probability.** The bins with the least mass (the clamped extremes, where a +strong dodger spends its time) are exactly the bins where this corpus's gun +lands the most hits (bins 1–2 and 28–29: 14–17.5%; bins 23–25: 6.7–7.2%). A +mover that steers to the lowest-mass bin steers *into* the bullets. This is the +mechanistic explanation of the j115 failure and of the result below, and no +amount of state conditioning can repair it: the label is the wrong quantity. + +*(MEASURED: the correlation and the per-bin table. INFERRED: that this is why +the crude surfer lost — it is consistent with j115's 13.51% incoming hit rate +against `strafe`'s 9.40%. What the mover **should** learn is the outcome: a +counted/decayed SBC over states and bins labelled by HIT/MISS would estimate +P(hit | state, bin) directly. That is the natural next experiment and it is NOT +what was measured here.)*