SirStone/SirRoboGarage
1
0
Fork 0
Code Issues 28 Pull Requests Actions Projects Releases Wiki Activity
Files
master
SirRoboGarage/common_libs/tests/fixtures/tfil_heat_pillar_ab_results.json
T
SirStone 48f38b80e7 TFIL heat-time + virtual pillar: live A/B (6 arms x 70 rounds) - neither change beats the pre-change mover 
Runs the pre-registered A/B for the two movement changes in HEAD: the
time-indexed bullet heat (TR_TFIL_HEAT_TIME, fca8993) and the removal of the
invented virtual centre pillar (d0750ab). One frozen binary from HEAD vs real
DrussGT: 6 arms x 10 runs x 7 rounds = 60 battles, 420 rounds, 0 failed.

Judged on damage/run and ROUND WINS only (hit rate and hits-taken are context):
hit rate would have inverted the verdict again - tau3 has the best pooled hit
rate of all arms (11.56%) and the fewest round wins (20/70).

RESULT (vs the reconstructed pre-change mover "old"):
  heat-time HURTS. tau3/tau5/tau9 lose 1.3-1.7 wins/run (p=0.0010-0.0125) and
  deal 22-38 less damage/run (p=0.004-0.047); tau15 is a wash on wins (p=0.64)
  and 22 damage/run lower (p=0.046). Nothing improves either metric.
  pillar removal does nothing measurable. old vs pillaoff: +5.7 damage/run
  (p=0.71), +0.5 wins/run (35 vs 30, p=0.43), 30.8 MORE damage taken/run
  without the pillar (p=0.040). The mechanism check proves the knob works
  (centre-box occupancy 0.09% -> 2.37%, p<0.0001; range 469 -> 443 px,
  p=0.0002), so this is a real behaviour change that buys nothing. At n=10 the
  pillar contrast is inside the MDE (33 damage/run, 1.2 wins/run), so this is
  not a proven regression.

Flags that the shipped default (pillar removed) should be reverted to the
TR_TFIL_PILLAR_ON behaviour; heat-time stays off.

Adds tools/ab/arms_heat_pillar.txt and tools/ab/ab_mechanism.py (per-tick
mechanism check: central-box occupancy, range distribution, live enemy-bullet
proximity) plus the captured summary/report fixtures.
2026-09-25 22:23:11 +02:00

2170 lines
46 KiB
JSON
Raw Permalink Blame History

{
"toolchain": {
"runner": "tools/ab/ab_run.sh",
"analyzer": "tools/ab/ab_analyze.py",
"mechanism": "tools/ab/ab_mechanism.py",
"arm_file": "tools/ab/arms_heat_pillar.txt",
"note": "raw per-tick captures (~306 MB) are NOT committed; this file holds every per-run value and every test the doc cites"
},
"session": {
"commit": "f58d65d2e8206cebd5b7d0a9465950dd9e6d2c28",
"binary_sha256": "74fd010ed1ffc4bb8a8de5569f58843479e1f50164a8a3f61e9c81cef108eaef",
"binary": "frozen/ModularBot/ModularBot_bin",
"rounds": 7,
"runs": 10,
"conc": 7,
"timestamp": "2026-09-25T22:09:42+02:00",
"outdir": "/tmp/ab/heatshift",
"arms": [
{
"name": "old",
"env": "TR_TFIL_PILLAR_ON=1 TR_TFIL_HEAT_TIME=0",
"label": "pre-change mover (virtual centre pillar ON)"
},
{
"name": "pillaoff",
"env": "",
"label": "shipped default at HEAD (pillar removed, flat bullet heat)"
},
{
"name": "tau3",
"env": "TR_TFIL_HEAT_TIME=1 TR_TFIL_HEAT_TAU=3",
"label": "time-indexed heat, tau=3 (short corridor)"
},
{
"name": "tau5",
"env": "TR_TFIL_HEAT_TIME=1 TR_TFIL_HEAT_TAU=5",
"label": "time-indexed heat, tau=5"
},
{
"name": "tau9",
"env": "TR_TFIL_HEAT_TIME=1 TR_TFIL_HEAT_TAU=9",
"label": "time-indexed heat, tau=9 (shipped default tau)"
},
{
"name": "tau15",
"env": "TR_TFIL_HEAT_TIME=1 TR_TFIL_HEAT_TAU=15",
"label": "time-indexed heat, tau=15 (long corridor)"
}
]
},
"arms": [
{
"name": "old",
"env": {
"TR_TFIL_PILLAR_ON": "1",
"TR_TFIL_HEAT_TIME": "0"
},
"label": "pre-change mover (virtual centre pillar ON)",
"summary": {
"runs": 10,
"damage_per_run": 287.4715,
"damage_taken_per_run": 201.52,
"round_wins": 35,
"rounds": 70,
"round_win_pct": 50.0,
"shots_per_run": 783.4,
"hits_dealt_per_run": 86.1,
"hits_taken_per_run": 93.5,
"pooled_hit_rate_pct": 10.991
},
"per_run": [
{
"run": 1,
"damage": 271.494,
"damage_taken": 237.4,
"wins": 3,
"rounds": 7,
"shots": 797,
"hits_taken": 102,
"hits_dealt": 89
},
{
"run": 2,
"damage": 272.9121,
"damage_taken": 232.6,
"wins": 4,
"rounds": 7,
"shots": 675,
"hits_taken": 88,
"hits_dealt": 80
},
{
"run": 3,
"damage": 254.5111,
"damage_taken": 171.2,
"wins": 4,
"rounds": 7,
"shots": 814,
"hits_taken": 74,
"hits_dealt": 79
},
{
"run": 4,
"damage": 269.6784,
"damage_taken": 220.1,
"wins": 2,
"rounds": 7,
"shots": 825,
"hits_taken": 102,
"hits_dealt": 83
},
{
"run": 5,
"damage": 282.045,
"damage_taken": 246.9,
"wins": 2,
"rounds": 7,
"shots": 774,
"hits_taken": 105,
"hits_dealt": 87
},
{
"run": 6,
"damage": 308.7667,
"damage_taken": 211.2,
"wins": 4,
"rounds": 7,
"shots": 867,
"hits_taken": 111,
"hits_dealt": 95
},
{
"run": 7,
"damage": 338.3033,
"damage_taken": 158.6,
"wins": 4,
"rounds": 7,
"shots": 760,
"hits_taken": 91,
"hits_dealt": 87
},
{
"run": 8,
"damage": 305.4958,
"damage_taken": 203.6,
"wins": 4,
"rounds": 7,
"shots": 801,
"hits_taken": 96,
"hits_dealt": 89
},
{
"run": 9,
"damage": 265.6571,
"damage_taken": 181.4,
"wins": 3,
"rounds": 7,
"shots": 757,
"hits_taken": 87,
"hits_dealt": 82
},
{
"run": 10,
"damage": 305.8518,
"damage_taken": 152.2,
"wins": 5,
"rounds": 7,
"shots": 764,
"hits_taken": 79,
"hits_dealt": 90
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.0011399719009073822,
"mean_dist": 452.7461675713594,
"enemy_bullet_100px_frac": 0.28061353947215323,
"mean_minbullet_enemy": 134.82743100531704
},
{
"run": 2,
"box_frac": 0.0,
"mean_dist": 466.2364065937541,
"enemy_bullet_100px_frac": 0.28279524079775126,
"mean_minbullet_enemy": 136.2539357214906
},
{
"run": 3,
"box_frac": 0.0,
"mean_dist": 488.74034234329196,
"enemy_bullet_100px_frac": 0.2634162765266662,
"mean_minbullet_enemy": 139.78159465900254
},
{
"run": 4,
"box_frac": 0.0015905147484094851,
"mean_dist": 481.6269927870462,
"enemy_bullet_100px_frac": 0.2881491488859993,
"mean_minbullet_enemy": 133.79944260402016
},
{
"run": 5,
"box_frac": 8.862105636299185e-05,
"mean_dist": 444.56180576617874,
"enemy_bullet_100px_frac": 0.30118654733478845,
"mean_minbullet_enemy": 132.3970098290515
},
{
"run": 6,
"box_frac": 0.0,
"mean_dist": 483.1054186523611,
"enemy_bullet_100px_frac": 0.27871610915259665,
"mean_minbullet_enemy": 138.0749960473332
},
{
"run": 7,
"box_frac": 0.0013379991766158913,
"mean_dist": 489.1962612297056,
"enemy_bullet_100px_frac": 0.27386463236735015,
"mean_minbullet_enemy": 135.9630194514801
},
{
"run": 8,
"box_frac": 0.0003490705995287547,
"mean_dist": 459.97591591328353,
"enemy_bullet_100px_frac": 0.30017812967765073,
"mean_minbullet_enemy": 134.11072178797272
},
{
"run": 9,
"box_frac": 0.0014388489208633092,
"mean_dist": 475.7590907706026,
"enemy_bullet_100px_frac": 0.26038783948789745,
"mean_minbullet_enemy": 139.32907693550354
},
{
"run": 10,
"box_frac": 0.002617258372568313,
"mean_dist": 450.7985573237218,
"enemy_bullet_100px_frac": 0.2700702301286749,
"mean_minbullet_enemy": 136.2049152714345
}
]
},
{
"name": "pillaoff",
"env": {},
"label": "shipped default at HEAD (pillar removed, flat bullet heat)",
"summary": {
"runs": 10,
"damage_per_run": 281.8139,
"damage_taken_per_run": 232.34,
"round_wins": 30,
"rounds": 70,
"round_win_pct": 42.86,
"shots_per_run": 729.1,
"hits_dealt_per_run": 81.2,
"hits_taken_per_run": 95.0,
"pooled_hit_rate_pct": 11.137
},
"per_run": [
{
"run": 1,
"damage": 206.5108,
"damage_taken": 228.0,
"wins": 1,
"rounds": 7,
"shots": 780,
"hits_taken": 100,
"hits_dealt": 67
},
{
"run": 2,
"damage": 332.2635,
"damage_taken": 194.5,
"wins": 5,
"rounds": 7,
"shots": 814,
"hits_taken": 97,
"hits_dealt": 99
},
{
"run": 3,
"damage": 262.9916,
"damage_taken": 230.5,
"wins": 2,
"rounds": 7,
"shots": 799,
"hits_taken": 91,
"hits_dealt": 77
},
{
"run": 4,
"damage": 331.8501,
"damage_taken": 215.8,
"wins": 4,
"rounds": 7,
"shots": 735,
"hits_taken": 92,
"hits_dealt": 89
},
{
"run": 5,
"damage": 286.3562,
"damage_taken": 215.5,
"wins": 4,
"rounds": 7,
"shots": 674,
"hits_taken": 92,
"hits_dealt": 76
},
{
"run": 6,
"damage": 280.4817,
"damage_taken": 302.5,
"wins": 3,
"rounds": 7,
"shots": 621,
"hits_taken": 92,
"hits_dealt": 80
},
{
"run": 7,
"damage": 243.5204,
"damage_taken": 244.9,
"wins": 2,
"rounds": 7,
"shots": 748,
"hits_taken": 93,
"hits_dealt": 77
},
{
"run": 8,
"damage": 268.7689,
"damage_taken": 240.3,
"wins": 2,
"rounds": 7,
"shots": 710,
"hits_taken": 100,
"hits_dealt": 79
},
{
"run": 9,
"damage": 311.1327,
"damage_taken": 216.3,
"wins": 4,
"rounds": 7,
"shots": 690,
"hits_taken": 89,
"hits_dealt": 83
},
{
"run": 10,
"damage": 294.2635,
"damage_taken": 235.1,
"wins": 3,
"rounds": 7,
"shots": 720,
"hits_taken": 104,
"hits_dealt": 85
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.019086268454493165,
"mean_dist": 447.21456575468875,
"enemy_bullet_100px_frac": 0.29687623696735127,
"mean_minbullet_enemy": 131.33239502164332
},
{
"run": 2,
"box_frac": 0.02096107061098756,
"mean_dist": 451.54408531021846,
"enemy_bullet_100px_frac": 0.27236591116235775,
"mean_minbullet_enemy": 135.76280734589338
},
{
"run": 3,
"box_frac": 0.033249832156470184,
"mean_dist": 449.0620989804667,
"enemy_bullet_100px_frac": 0.319069086056385,
"mean_minbullet_enemy": 126.55720953050556
},
{
"run": 4,
"box_frac": 0.017388173637273775,
"mean_dist": 442.13952501527,
"enemy_bullet_100px_frac": 0.28688420213649973,
"mean_minbullet_enemy": 133.73314448443605
},
{
"run": 5,
"box_frac": 0.02527665036219943,
"mean_dist": 444.1371724160463,
"enemy_bullet_100px_frac": 0.3027996688467169,
"mean_minbullet_enemy": 132.3457856465789
},
{
"run": 6,
"box_frac": 0.03279219629748387,
"mean_dist": 430.7492875685674,
"enemy_bullet_100px_frac": 0.25811836539953414,
"mean_minbullet_enemy": 138.1215025322138
},
{
"run": 7,
"box_frac": 0.01866588844813426,
"mean_dist": 446.24011741323886,
"enemy_bullet_100px_frac": 0.29471073344769794,
"mean_minbullet_enemy": 131.8058582238914
},
{
"run": 8,
"box_frac": 0.020293234602606697,
"mean_dist": 436.7208169386755,
"enemy_bullet_100px_frac": 0.29773475283849254,
"mean_minbullet_enemy": 132.07868485485136
},
{
"run": 9,
"box_frac": 0.019629910953624746,
"mean_dist": 447.44331039382877,
"enemy_bullet_100px_frac": 0.2763264294289161,
"mean_minbullet_enemy": 134.89015204232922
},
{
"run": 10,
"box_frac": 0.02981284199467681,
"mean_dist": 440.0500501474407,
"enemy_bullet_100px_frac": 0.30544584480723586,
"mean_minbullet_enemy": 131.18604658771358
}
]
},
{
"name": "tau3",
"env": {
"TR_TFIL_HEAT_TIME": "1",
"TR_TFIL_HEAT_TAU": "3"
},
"label": "time-indexed heat, tau=3 (short corridor)",
"summary": {
"runs": 10,
"damage_per_run": 261.1283,
"damage_taken_per_run": 298.79,
"round_wins": 20,
"rounds": 70,
"round_win_pct": 28.57,
"shots_per_run": 666.2,
"hits_dealt_per_run": 77.0,
"hits_taken_per_run": 106.2,
"pooled_hit_rate_pct": 11.558
},
"per_run": [
{
"run": 1,
"damage": 213.4121,
"damage_taken": 285.8,
"wins": 1,
"rounds": 7,
"shots": 664,
"hits_taken": 109,
"hits_dealt": 67
},
{
"run": 2,
"damage": 296.2088,
"damage_taken": 251.1,
"wins": 3,
"rounds": 7,
"shots": 774,
"hits_taken": 118,
"hits_dealt": 94
},
{
"run": 3,
"damage": 182.1439,
"damage_taken": 320.9,
"wins": 1,
"rounds": 7,
"shots": 650,
"hits_taken": 99,
"hits_dealt": 57
},
{
"run": 4,
"damage": 284.6862,
"damage_taken": 300.2,
"wins": 3,
"rounds": 7,
"shots": 648,
"hits_taken": 101,
"hits_dealt": 78
},
{
"run": 5,
"damage": 283.4574,
"damage_taken": 274.5,
"wins": 3,
"rounds": 7,
"shots": 579,
"hits_taken": 96,
"hits_dealt": 73
},
{
"run": 6,
"damage": 220.0362,
"damage_taken": 386.3,
"wins": 1,
"rounds": 7,
"shots": 552,
"hits_taken": 96,
"hits_dealt": 62
},
{
"run": 7,
"damage": 289.7124,
"damage_taken": 286.1,
"wins": 4,
"rounds": 7,
"shots": 620,
"hits_taken": 105,
"hits_dealt": 84
},
{
"run": 8,
"damage": 303.5934,
"damage_taken": 341.6,
"wins": 1,
"rounds": 7,
"shots": 706,
"hits_taken": 127,
"hits_dealt": 90
},
{
"run": 9,
"damage": 244.7051,
"damage_taken": 291.2,
"wins": 2,
"rounds": 7,
"shots": 699,
"hits_taken": 98,
"hits_dealt": 76
},
{
"run": 10,
"damage": 293.327,
"damage_taken": 250.2,
"wins": 1,
"rounds": 7,
"shots": 770,
"hits_taken": 113,
"hits_dealt": 89
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.2910357040373263,
"mean_dist": 453.56023541204746,
"enemy_bullet_100px_frac": 0.2986572496787014,
"mean_minbullet_enemy": 137.23406249568063
},
{
"run": 2,
"box_frac": 0.29062784258588065,
"mean_dist": 442.70358815804764,
"enemy_bullet_100px_frac": 0.2994973638661029,
"mean_minbullet_enemy": 134.4727227832659
},
{
"run": 3,
"box_frac": 0.2082709647700656,
"mean_dist": 454.82127743462576,
"enemy_bullet_100px_frac": 0.2723502223961439,
"mean_minbullet_enemy": 140.44048384247955
},
{
"run": 4,
"box_frac": 0.24509600594245035,
"mean_dist": 423.8080829449972,
"enemy_bullet_100px_frac": 0.31089979529826206,
"mean_minbullet_enemy": 133.80018148637546
},
{
"run": 5,
"box_frac": 0.21452078928221932,
"mean_dist": 428.333172617735,
"enemy_bullet_100px_frac": 0.3185728140905532,
"mean_minbullet_enemy": 131.41805949736724
},
{
"run": 6,
"box_frac": 0.20844749809303473,
"mean_dist": 442.9183582203959,
"enemy_bullet_100px_frac": 0.29595798453173533,
"mean_minbullet_enemy": 137.48524284705013
},
{
"run": 7,
"box_frac": 0.30841883493854955,
"mean_dist": 422.9662261053265,
"enemy_bullet_100px_frac": 0.3337219237142014,
"mean_minbullet_enemy": 131.76961169993265
},
{
"run": 8,
"box_frac": 0.30226160589178847,
"mean_dist": 437.2993333984904,
"enemy_bullet_100px_frac": 0.33118692495057794,
"mean_minbullet_enemy": 132.50028020278512
},
{
"run": 9,
"box_frac": 0.21338048374953963,
"mean_dist": 454.24611427086256,
"enemy_bullet_100px_frac": 0.3044358267909688,
"mean_minbullet_enemy": 136.01099514349494
},
{
"run": 10,
"box_frac": 0.3301159219811315,
"mean_dist": 433.12260853334544,
"enemy_bullet_100px_frac": 0.3207043179365959,
"mean_minbullet_enemy": 133.2462604207705
}
]
},
{
"name": "tau5",
"env": {
"TR_TFIL_HEAT_TIME": "1",
"TR_TFIL_HEAT_TAU": "5"
},
"label": "time-indexed heat, tau=5",
"summary": {
"runs": 10,
"damage_per_run": 249.5781,
"damage_taken_per_run": 293.55,
"round_wins": 18,
"rounds": 70,
"round_win_pct": 25.71,
"shots_per_run": 688.9,
"hits_dealt_per_run": 74.6,
"hits_taken_per_run": 105.3,
"pooled_hit_rate_pct": 10.829
},
"per_run": [
{
"run": 1,
"damage": 272.2014,
"damage_taken": 327.5,
"wins": 2,
"rounds": 7,
"shots": 624,
"hits_taken": 102,
"hits_dealt": 77
},
{
"run": 2,
"damage": 235.2229,
"damage_taken": 276.9,
"wins": 2,
"rounds": 7,
"shots": 781,
"hits_taken": 104,
"hits_dealt": 78
},
{
"run": 3,
"damage": 241.3944,
"damage_taken": 339.7,
"wins": 2,
"rounds": 7,
"shots": 681,
"hits_taken": 100,
"hits_dealt": 76
},
{
"run": 4,
"damage": 235.7453,
"damage_taken": 305.4,
"wins": 1,
"rounds": 7,
"shots": 658,
"hits_taken": 105,
"hits_dealt": 69
},
{
"run": 5,
"damage": 281.1003,
"damage_taken": 192.5,
"wins": 3,
"rounds": 7,
"shots": 719,
"hits_taken": 101,
"hits_dealt": 79
},
{
"run": 6,
"damage": 295.3842,
"damage_taken": 228.8,
"wins": 1,
"rounds": 7,
"shots": 792,
"hits_taken": 128,
"hits_dealt": 87
},
{
"run": 7,
"damage": 248.2844,
"damage_taken": 272.5,
"wins": 2,
"rounds": 7,
"shots": 697,
"hits_taken": 113,
"hits_dealt": 76
},
{
"run": 8,
"damage": 242.8076,
"damage_taken": 372.3,
"wins": 2,
"rounds": 7,
"shots": 612,
"hits_taken": 96,
"hits_dealt": 71
},
{
"run": 9,
"damage": 218.2018,
"damage_taken": 328.1,
"wins": 2,
"rounds": 7,
"shots": 616,
"hits_taken": 90,
"hits_dealt": 62
},
{
"run": 10,
"damage": 225.4384,
"damage_taken": 291.8,
"wins": 1,
"rounds": 7,
"shots": 709,
"hits_taken": 114,
"hits_dealt": 71
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.19456402989942997,
"mean_dist": 446.1956673381141,
"enemy_bullet_100px_frac": 0.2989958629058934,
"mean_minbullet_enemy": 136.74856198773608
},
{
"run": 2,
"box_frac": 0.17583894425652571,
"mean_dist": 453.50035740099645,
"enemy_bullet_100px_frac": 0.3181725111658755,
"mean_minbullet_enemy": 128.8950808559423
},
{
"run": 3,
"box_frac": 0.21136256699314906,
"mean_dist": 452.8615689786322,
"enemy_bullet_100px_frac": 0.30016611156002104,
"mean_minbullet_enemy": 135.04043704742466
},
{
"run": 4,
"box_frac": 0.1910756110839799,
"mean_dist": 446.53308774060235,
"enemy_bullet_100px_frac": 0.29698857364067066,
"mean_minbullet_enemy": 135.27990277039206
},
{
"run": 5,
"box_frac": 0.23719796137222401,
"mean_dist": 435.87465737850425,
"enemy_bullet_100px_frac": 0.3242421620568048,
"mean_minbullet_enemy": 131.89419084705114
},
{
"run": 6,
"box_frac": 0.21930118047206043,
"mean_dist": 448.0047357401412,
"enemy_bullet_100px_frac": 0.32355178026668485,
"mean_minbullet_enemy": 129.9461073472292
},
{
"run": 7,
"box_frac": 0.23932950342466403,
"mean_dist": 455.2896035470655,
"enemy_bullet_100px_frac": 0.3103289877461127,
"mean_minbullet_enemy": 133.4789980657075
},
{
"run": 8,
"box_frac": 0.22777082249148986,
"mean_dist": 442.6738546716405,
"enemy_bullet_100px_frac": 0.3036805345467015,
"mean_minbullet_enemy": 135.33962837975042
},
{
"run": 9,
"box_frac": 0.23262309219001195,
"mean_dist": 445.07607614311644,
"enemy_bullet_100px_frac": 0.2870917639056394,
"mean_minbullet_enemy": 136.90134683206486
},
{
"run": 10,
"box_frac": 0.20087832001585326,
"mean_dist": 452.32238605691083,
"enemy_bullet_100px_frac": 0.2968090320648681,
"mean_minbullet_enemy": 137.1465777112292
}
]
},
{
"name": "tau9",
"env": {
"TR_TFIL_HEAT_TIME": "1",
"TR_TFIL_HEAT_TAU": "9"
},
"label": "time-indexed heat, tau=9 (shipped default tau)",
"summary": {
"runs": 10,
"damage_per_run": 265.4155,
"damage_taken_per_run": 231.11,
"round_wins": 22,
"rounds": 70,
"round_win_pct": 31.43,
"shots_per_run": 750.5,
"hits_dealt_per_run": 78.9,
"hits_taken_per_run": 99.5,
"pooled_hit_rate_pct": 10.513
},
"per_run": [
{
"run": 1,
"damage": 294.6292,
"damage_taken": 191.5,
"wins": 3,
"rounds": 7,
"shots": 810,
"hits_taken": 96,
"hits_dealt": 86
},
{
"run": 2,
"damage": 282.7173,
"damage_taken": 190.2,
"wins": 3,
"rounds": 7,
"shots": 775,
"hits_taken": 89,
"hits_dealt": 87
},
{
"run": 3,
"damage": 274.5672,
"damage_taken": 237.1,
"wins": 2,
"rounds": 7,
"shots": 803,
"hits_taken": 113,
"hits_dealt": 84
},
{
"run": 4,
"damage": 238.9766,
"damage_taken": 192.6,
"wins": 2,
"rounds": 7,
"shots": 737,
"hits_taken": 87,
"hits_dealt": 69
},
{
"run": 5,
"damage": 268.2957,
"damage_taken": 259.3,
"wins": 2,
"rounds": 7,
"shots": 729,
"hits_taken": 104,
"hits_dealt": 78
},
{
"run": 6,
"damage": 258.177,
"damage_taken": 220.0,
"wins": 3,
"rounds": 7,
"shots": 784,
"hits_taken": 97,
"hits_dealt": 86
},
{
"run": 7,
"damage": 280.2975,
"damage_taken": 202.7,
"wins": 2,
"rounds": 7,
"shots": 811,
"hits_taken": 104,
"hits_dealt": 85
},
{
"run": 8,
"damage": 262.465,
"damage_taken": 217.7,
"wins": 3,
"rounds": 7,
"shots": 708,
"hits_taken": 101,
"hits_dealt": 71
},
{
"run": 9,
"damage": 227.7898,
"damage_taken": 305.9,
"wins": 1,
"rounds": 7,
"shots": 662,
"hits_taken": 96,
"hits_dealt": 67
},
{
"run": 10,
"damage": 266.2393,
"damage_taken": 294.1,
"wins": 1,
"rounds": 7,
"shots": 686,
"hits_taken": 108,
"hits_dealt": 76
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.07592581756718418,
"mean_dist": 489.35578409528534,
"enemy_bullet_100px_frac": 0.29269574576551155,
"mean_minbullet_enemy": 137.35382343743498
},
{
"run": 2,
"box_frac": 0.1245160052396274,
"mean_dist": 475.3589486217019,
"enemy_bullet_100px_frac": 0.28360212680100017,
"mean_minbullet_enemy": 136.5206865096948
},
{
"run": 3,
"box_frac": 0.07480024917671986,
"mean_dist": 488.7776759832404,
"enemy_bullet_100px_frac": 0.2940416709697642,
"mean_minbullet_enemy": 135.08558205953614
},
{
"run": 4,
"box_frac": 0.07526223315676726,
"mean_dist": 482.0148480879792,
"enemy_bullet_100px_frac": 0.2801304567908688,
"mean_minbullet_enemy": 136.49304004604693
},
{
"run": 5,
"box_frac": 0.07127388375370293,
"mean_dist": 481.00369388411815,
"enemy_bullet_100px_frac": 0.30133642449521564,
"mean_minbullet_enemy": 135.3938644340982
},
{
"run": 6,
"box_frac": 0.08065398423851695,
"mean_dist": 495.8317674052187,
"enemy_bullet_100px_frac": 0.26657171331684026,
"mean_minbullet_enemy": 139.3463122563716
},
{
"run": 7,
"box_frac": 0.09396087485021064,
"mean_dist": 494.12023934747214,
"enemy_bullet_100px_frac": 0.2657710446413541,
"mean_minbullet_enemy": 140.6501748374419
},
{
"run": 8,
"box_frac": 0.12322751003176827,
"mean_dist": 477.3913959978383,
"enemy_bullet_100px_frac": 0.27275692831598813,
"mean_minbullet_enemy": 139.67747373819427
},
{
"run": 9,
"box_frac": 0.1648737565251423,
"mean_dist": 466.95352419462046,
"enemy_bullet_100px_frac": 0.272200124854493,
"mean_minbullet_enemy": 140.51609876714193
},
{
"run": 10,
"box_frac": 0.1553308383358681,
"mean_dist": 460.1262837454722,
"enemy_bullet_100px_frac": 0.3021532356483877,
"mean_minbullet_enemy": 136.5339207365652
}
]
},
{
"name": "tau15",
"env": {
"TR_TFIL_HEAT_TIME": "1",
"TR_TFIL_HEAT_TAU": "15"
},
"label": "time-indexed heat, tau=15 (long corridor)",
"summary": {
"runs": 10,
"damage_per_run": 265.4323,
"damage_taken_per_run": 185.39,
"round_wins": 32,
"rounds": 70,
"round_win_pct": 45.71,
"shots_per_run": 811.8,
"hits_dealt_per_run": 80.6,
"hits_taken_per_run": 89.5,
"pooled_hit_rate_pct": 9.929
},
"per_run": [
{
"run": 1,
"damage": 265.7676,
"damage_taken": 166.0,
"wins": 3,
"rounds": 7,
"shots": 874,
"hits_taken": 94,
"hits_dealt": 86
},
{
"run": 2,
"damage": 263.5857,
"damage_taken": 185.4,
"wins": 4,
"rounds": 7,
"shots": 759,
"hits_taken": 82,
"hits_dealt": 81
},
{
"run": 3,
"damage": 257.8739,
"damage_taken": 175.5,
"wins": 3,
"rounds": 7,
"shots": 809,
"hits_taken": 90,
"hits_dealt": 77
},
{
"run": 4,
"damage": 239.143,
"damage_taken": 193.0,
"wins": 2,
"rounds": 7,
"shots": 830,
"hits_taken": 87,
"hits_dealt": 74
},
{
"run": 5,
"damage": 278.0669,
"damage_taken": 165.8,
"wins": 4,
"rounds": 7,
"shots": 844,
"hits_taken": 93,
"hits_dealt": 84
},
{
"run": 6,
"damage": 290.7293,
"damage_taken": 192.2,
"wins": 4,
"rounds": 7,
"shots": 808,
"hits_taken": 90,
"hits_dealt": 88
},
{
"run": 7,
"damage": 229.9269,
"damage_taken": 183.2,
"wins": 2,
"rounds": 7,
"shots": 818,
"hits_taken": 86,
"hits_dealt": 69
},
{
"run": 8,
"damage": 285.6853,
"damage_taken": 199.2,
"wins": 4,
"rounds": 7,
"shots": 807,
"hits_taken": 94,
"hits_dealt": 87
},
{
"run": 9,
"damage": 283.0388,
"damage_taken": 158.1,
"wins": 4,
"rounds": 7,
"shots": 801,
"hits_taken": 79,
"hits_dealt": 81
},
{
"run": 10,
"damage": 260.5059,
"damage_taken": 235.5,
"wins": 2,
"rounds": 7,
"shots": 768,
"hits_taken": 100,
"hits_dealt": 79
}
],
"mechanism_per_run": [
{
"run": 1,
"box_frac": 0.044687893223915545,
"mean_dist": 506.12446531845086,
"enemy_bullet_100px_frac": 0.25112353406528415,
"mean_minbullet_enemy": 143.30050216458278
},
{
"run": 2,
"box_frac": 0.0296163424544997,
"mean_dist": 497.0395516752881,
"enemy_bullet_100px_frac": 0.2599251276576841,
"mean_minbullet_enemy": 140.3638343569318
},
{
"run": 3,
"box_frac": 0.030041145032901933,
"mean_dist": 497.9521154205033,
"enemy_bullet_100px_frac": 0.26915408389995177,
"mean_minbullet_enemy": 139.47962561501913
},
{
"run": 4,
"box_frac": 0.0523732228697578,
"mean_dist": 514.1270953634627,
"enemy_bullet_100px_frac": 0.25211711276248855,
"mean_minbullet_enemy": 141.56274795846343
},
{
"run": 5,
"box_frac": 0.04997248846090457,
"mean_dist": 500.641041513241,
"enemy_bullet_100px_frac": 0.25136230383444813,
"mean_minbullet_enemy": 141.800483076964
},
{
"run": 6,
"box_frac": 0.05493488628420456,
"mean_dist": 488.5266229241019,
"enemy_bullet_100px_frac": 0.24526433773357878,
"mean_minbullet_enemy": 142.86119086117716
},
{
"run": 7,
"box_frac": 0.035455319958077394,
"mean_dist": 508.25928488962353,
"enemy_bullet_100px_frac": 0.262903024264981,
"mean_minbullet_enemy": 140.52582288753027
},
{
"run": 8,
"box_frac": 0.05533757079116542,
"mean_dist": 503.1061101014985,
"enemy_bullet_100px_frac": 0.25709321388672807,
"mean_minbullet_enemy": 141.5183439735951
},
{
"run": 9,
"box_frac": 0.04605386661058597,
"mean_dist": 503.0301344463501,
"enemy_bullet_100px_frac": 0.2516549602652214,
"mean_minbullet_enemy": 141.26588640590836
},
{
"run": 10,
"box_frac": 0.06976031831484165,
"mean_dist": 482.15156152486344,
"enemy_bullet_100px_frac": 0.2561333877300892,
"mean_minbullet_enemy": 140.85777412432188
}
]
}
],
"tests_pairwise": [
{
"metric": "dmg_per_run",
"a": "old",
"b": "pillaoff",
"mean_a": 287.4715,
"mean_b": 281.8139,
"diff_a_minus_b": 5.6576,
"p_perm": 0.7096386585550672,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.8501067391385257,
"U": 47.0
},
{
"metric": "round_wins",
"a": "old",
"b": "pillaoff",
"mean_a": 3.5,
"mean_b": 3.0,
"diff_a_minus_b": 0.5,
"p_perm": 0.4317045183918249,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.3630710161694755,
"U": 38.0
},
{
"metric": "dmg_per_run",
"a": "old",
"b": "tau3",
"mean_a": 287.4715,
"mean_b": 261.1283,
"diff_a_minus_b": 26.3433,
"p_perm": 0.11366342635692481,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.3074894566186813,
"U": 36.0
},
{
"metric": "round_wins",
"a": "old",
"b": "tau3",
"mean_a": 3.5,
"mean_b": 2.0,
"diff_a_minus_b": 1.5,
"p_perm": 0.012470501634588321,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.010107637189318341,
"U": 16.5
},
{
"metric": "dmg_per_run",
"a": "old",
"b": "tau5",
"mean_a": 287.4715,
"mean_b": 249.5781,
"diff_a_minus_b": 37.8935,
"p_perm": 0.004221784407542921,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.00910849639803097,
"U": 15.0
},
{
"metric": "round_wins",
"a": "old",
"b": "tau5",
"mean_a": 3.5,
"mean_b": 1.8,
"diff_a_minus_b": 1.7,
"p_perm": 0.0010283833813245578,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.0013782085540494599,
"U": 9.0
},
{
"metric": "dmg_per_run",
"a": "old",
"b": "tau9",
"mean_a": 287.4715,
"mean_b": 265.4155,
"diff_a_minus_b": 22.0561,
"p_perm": 0.046818506570828554,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.10410988966022679,
"U": 28.0
},
{
"metric": "round_wins",
"a": "old",
"b": "tau9",
"mean_a": 3.5,
"mean_b": 2.2,
"diff_a_minus_b": 1.3,
"p_perm": 0.00969927904912425,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.008697762328283905,
"U": 16.0
},
{
"metric": "dmg_per_run",
"a": "old",
"b": "tau15",
"mean_a": 287.4715,
"mean_b": 265.4323,
"diff_a_minus_b": 22.0392,
"p_perm": 0.0459633246010955,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.10410988966022679,
"U": 28.0
},
{
"metric": "round_wins",
"a": "old",
"b": "tau15",
"mean_a": 3.5,
"mean_b": 3.2,
"diff_a_minus_b": 0.3,
"p_perm": 0.6369265409513087,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.5127031419699851,
"U": 41.5
},
{
"metric": "dmg_per_run",
"a": "pillaoff",
"b": "tau3",
"mean_a": 281.8139,
"mean_b": 261.1283,
"diff_a_minus_b": 20.6857,
"p_perm": 0.2721210677867025,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.4726755935115873,
"U": 40.0
},
{
"metric": "round_wins",
"a": "pillaoff",
"b": "tau3",
"mean_a": 3.0,
"mean_b": 2.0,
"diff_a_minus_b": 1.0,
"p_perm": 0.11488666132628982,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.08689386890718304,
"U": 27.5
},
{
"metric": "dmg_per_run",
"a": "pillaoff",
"b": "tau5",
"mean_a": 281.8139,
"mean_b": 249.5781,
"diff_a_minus_b": 32.2359,
"p_perm": 0.042867349368897356,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.05390255716938722,
"U": 24.0
},
{
"metric": "round_wins",
"a": "pillaoff",
"b": "tau5",
"mean_a": 3.0,
"mean_b": 1.8,
"diff_a_minus_b": 1.2,
"p_perm": 0.026240013856112927,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.025366984290276135,
"U": 21.5
},
{
"metric": "dmg_per_run",
"a": "pillaoff",
"b": "tau9",
"mean_a": 281.8139,
"mean_b": 265.4155,
"diff_a_minus_b": 16.3985,
"p_perm": 0.25432462274567535,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.18587673236587593,
"U": 32.0
},
{
"metric": "round_wins",
"a": "pillaoff",
"b": "tau9",
"mean_a": 3.0,
"mean_b": 2.2,
"diff_a_minus_b": 0.8,
"p_perm": 0.15477711143345818,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.14605298094507024,
"U": 31.0
},
{
"metric": "dmg_per_run",
"a": "pillaoff",
"b": "tau15",
"mean_a": 281.8139,
"mean_b": 265.4323,
"diff_a_minus_b": 16.3816,
"p_perm": 0.25314468812920826,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.18587673236587593,
"U": 32.0
},
{
"metric": "round_wins",
"a": "pillaoff",
"b": "tau15",
"mean_a": 3.0,
"mean_b": 3.2,
"diff_a_minus_b": -0.2,
"p_perm": 0.837840178397454,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.7203650399405125,
"U": 45.0
},
{
"metric": "dmg_per_run",
"a": "tau3",
"b": "tau5",
"mean_a": 261.1283,
"mean_b": 249.5781,
"diff_a_minus_b": 11.5502,
"p_perm": 0.46741648444434825,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.3447042220069577,
"U": 37.0
},
{
"metric": "round_wins",
"a": "tau3",
"b": "tau5",
"mean_a": 2.0,
"mean_b": 1.8,
"diff_a_minus_b": 0.2,
"p_perm": 0.8120764684232177,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.9041815382841777,
"U": 48.0
},
{
"metric": "dmg_per_run",
"a": "tau3",
"b": "tau9",
"mean_a": 261.1283,
"mean_b": 265.4155,
"diff_a_minus_b": -4.2872,
"p_perm": 0.7790924246032606,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.570750388058174,
"U": 42.0
},
{
"metric": "round_wins",
"a": "tau3",
"b": "tau9",
"mean_a": 2.0,
"mean_b": 2.2,
"diff_a_minus_b": -0.2,
"p_perm": 0.819654030180346,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.6046563028662904,
"U": 43.0
},
{
"metric": "dmg_per_run",
"a": "tau3",
"b": "tau15",
"mean_a": 261.1283,
"mean_b": 265.4323,
"diff_a_minus_b": -4.3041,
"p_perm": 0.7775985624282838,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.6231762238821174,
"U": 43.0
},
{
"metric": "round_wins",
"a": "tau3",
"b": "tau15",
"mean_a": 2.0,
"mean_b": 3.2,
"diff_a_minus_b": -1.2,
"p_perm": 0.0355820649938297,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.025965522982741577,
"U": 21.0
},
{
"metric": "dmg_per_run",
"a": "tau5",
"b": "tau9",
"mean_a": 249.5781,
"mean_b": 265.4155,
"diff_a_minus_b": -15.8374,
"p_perm": 0.1358656823053108,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.18587673236587593,
"U": 32.0
},
{
"metric": "round_wins",
"a": "tau5",
"b": "tau9",
"mean_a": 1.8,
"mean_b": 2.2,
"diff_a_minus_b": -0.4,
"p_perm": 0.35670830717270346,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.2333448892485861,
"U": 35.0
},
{
"metric": "dmg_per_run",
"a": "tau5",
"b": "tau15",
"mean_a": 249.5781,
"mean_b": 265.4323,
"diff_a_minus_b": -15.8543,
"p_perm": 0.1337331399250904,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.16197241048012603,
"U": 31.0
},
{
"metric": "round_wins",
"a": "tau5",
"b": "tau15",
"mean_a": 1.8,
"mean_b": 3.2,
"diff_a_minus_b": -1.4,
"p_perm": 0.0034099027907077443,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.0034248479573704867,
"U": 13.0
},
{
"metric": "dmg_per_run",
"a": "tau9",
"b": "tau15",
"mean_a": 265.4155,
"mean_b": 265.4323,
"diff_a_minus_b": -0.0169,
"p_perm": 0.9984411872956764,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 1.0,
"U": 50.0
},
{
"metric": "round_wins",
"a": "tau9",
"b": "tau15",
"mean_a": 2.2,
"mean_b": 3.2,
"diff_a_minus_b": -1.0,
"p_perm": 0.03557123990560523,
"method": "exact",
"draws": 184756,
"mc_se": 0.0,
"p_mannwhitney": 0.02983453375806475,
"U": 22.0
}
],
"tests_vs_reference_extra": [
{
"metric": "damage_taken",
"a": "old",
"b": "pillaoff",
"mean_a": 201.52,
"mean_b": 232.34,
"diff_a_minus_b": -30.82,
"p_perm": 0.03981467448959709,
"method": "exact",
"p_mannwhitney": 0.10410988966022679,
"U": 28.0
},
{
"metric": "damage_taken",
"a": "old",
"b": "tau3",
"mean_a": 201.52,
"mean_b": 298.79,
"diff_a_minus_b": -97.27,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"p_mannwhitney": 0.0001826717911095504,
"U": 0.0
},
{
"metric": "damage_taken",
"a": "old",
"b": "tau5",
"mean_a": 201.52,
"mean_b": 293.55,
"diff_a_minus_b": -92.03,
"p_perm": 0.0004438286172032302,
"method": "exact",
"p_mannwhitney": 0.0022022199424970835,
"U": 9.0
},
{
"metric": "damage_taken",
"a": "old",
"b": "tau9",
"mean_a": 201.52,
"mean_b": 231.11,
"diff_a_minus_b": -29.59,
"p_perm": 0.10357444413171968,
"method": "exact",
"p_mannwhitney": 0.2413215930171802,
"U": 34.0
},
{
"metric": "damage_taken",
"a": "old",
"b": "tau15",
"mean_a": 201.52,
"mean_b": 185.39,
"diff_a_minus_b": 16.13,
"p_perm": 0.2213297538374938,
"method": "exact",
"p_mannwhitney": 0.3074894566186813,
"U": 36.0
},
{
"metric": "hits_taken",
"a": "old",
"b": "pillaoff",
"mean_a": 93.5,
"mean_b": 95.0,
"diff_a_minus_b": -1.5,
"p_perm": 0.7349260646474268,
"method": "exact",
"p_mannwhitney": 0.7617599689873358,
"U": 45.5
},
{
"metric": "hits_taken",
"a": "old",
"b": "tau3",
"mean_a": 93.5,
"mean_b": 106.2,
"diff_a_minus_b": -12.7,
"p_perm": 0.02087077009677629,
"method": "exact",
"p_mannwhitney": 0.048854007021222536,
"U": 23.5
},
{
"metric": "hits_taken",
"a": "old",
"b": "tau5",
"mean_a": 93.5,
"mean_b": 105.3,
"diff_a_minus_b": -11.8,
"p_perm": 0.031749983762367665,
"method": "exact",
"p_mannwhitney": 0.06342254736585444,
"U": 25.0
},
{
"metric": "hits_taken",
"a": "old",
"b": "tau9",
"mean_a": 93.5,
"mean_b": 99.5,
"diff_a_minus_b": -6.0,
"p_perm": 0.2099634112018013,
"method": "exact",
"p_mannwhitney": 0.2886460175593255,
"U": 35.5
},
{
"metric": "hits_taken",
"a": "old",
"b": "tau15",
"mean_a": 93.5,
"mean_b": 89.5,
"diff_a_minus_b": 4.0,
"p_perm": 0.3676524713676416,
"method": "exact",
"p_mannwhitney": 0.34379556718772664,
"U": 37.0
},
{
"metric": "shots",
"a": "old",
"b": "pillaoff",
"mean_a": 783.4,
"mean_b": 729.1,
"diff_a_minus_b": 54.3,
"p_perm": 0.042759098486652665,
"method": "exact",
"p_mannwhitney": 0.0492807602200552,
"U": 23.5
},
{
"metric": "shots",
"a": "old",
"b": "tau3",
"mean_a": 783.4,
"mean_b": 666.2,
"diff_a_minus_b": 117.2,
"p_perm": 0.0009201324990798675,
"method": "exact",
"p_mannwhitney": 0.0031856490798944604,
"U": 10.5
},
{
"metric": "shots",
"a": "old",
"b": "tau5",
"mean_a": 783.4,
"mean_b": 688.9,
"diff_a_minus_b": 94.5,
"p_perm": 0.0025980211738725672,
"method": "exact",
"p_mannwhitney": 0.0072845570094796685,
"U": 14.0
},
{
"metric": "shots",
"a": "old",
"b": "tau9",
"mean_a": 783.4,
"mean_b": 750.5,
"diff_a_minus_b": 32.9,
"p_perm": 0.17942583732057416,
"method": "exact",
"p_mannwhitney": 0.27303633975118824,
"U": 35.0
},
{
"metric": "shots",
"a": "old",
"b": "tau15",
"mean_a": 783.4,
"mean_b": 811.8,
"diff_a_minus_b": -28.4,
"p_perm": 0.16361038342462492,
"method": "exact",
"p_mannwhitney": 0.1507733378534401,
"U": 30.5
},
{
"metric": "hits_dealt",
"a": "old",
"b": "pillaoff",
"mean_a": 86.1,
"mean_b": 81.2,
"diff_a_minus_b": 4.9,
"p_perm": 0.14680984650024897,
"method": "exact",
"p_mannwhitney": 0.08106336915577778,
"U": 26.5
},
{
"metric": "hits_dealt",
"a": "old",
"b": "tau3",
"mean_a": 86.1,
"mean_b": 77.0,
"diff_a_minus_b": 9.1,
"p_perm": 0.04898352421572236,
"method": "exact",
"p_mannwhitney": 0.11159974960132965,
"U": 28.5
},
{
"metric": "hits_dealt",
"a": "old",
"b": "tau5",
"mean_a": 86.1,
"mean_b": 74.6,
"diff_a_minus_b": 11.5,
"p_perm": 0.0005412544112234515,
"method": "exact",
"p_mannwhitney": 0.0008495159836552036,
"U": 5.5
},
{
"metric": "hits_dealt",
"a": "old",
"b": "tau9",
"mean_a": 86.1,
"mean_b": 78.9,
"diff_a_minus_b": 7.2,
"p_perm": 0.02700859512005023,
"method": "exact",
"p_mannwhitney": 0.03082895140322005,
"U": 21.0
},
{
"metric": "hits_dealt",
"a": "old",
"b": "tau15",
"mean_a": 86.1,
"mean_b": 80.6,
"diff_a_minus_b": 5.5,
"p_perm": 0.04386325748554851,
"method": "exact",
"p_mannwhitney": 0.04876874101461294,
"U": 23.5
}
],
"mde": [
{
"metric": "dmg_per_run",
"n_per_arm": 10,
"sd_control": 25.9821,
"mde_abs": 32.5531,
"control_mean": 287.4715,
"mde_pct_of_control_mean": 11.32
},
{
"metric": "round_wins",
"n_per_arm": 10,
"sd_control": 0.9718,
"mde_abs": 1.2176,
"control_mean": 3.5,
"mde_pct_of_control_mean": 34.79
}
],
"round_level_fisher": [
{
"metric": "round_wins_pooled",
"a": "old",
"b": "pillaoff",
"ref_wins": 35,
"ref_rounds": 70,
"arm_wins": 30,
"arm_rounds": 70,
"p_fisher": 0.49801877186288623
},
{
"metric": "round_wins_pooled",
"a": "old",
"b": "tau3",
"ref_wins": 35,
"ref_rounds": 70,
"arm_wins": 20,
"arm_rounds": 70,
"p_fisher": 0.0150499158661199
},
{
"metric": "round_wins_pooled",
"a": "old",
"b": "tau5",
"ref_wins": 35,
"ref_rounds": 70,
"arm_wins": 18,
"arm_rounds": 70,
"p_fisher": 0.005073229951345803
},
{
"metric": "round_wins_pooled",
"a": "old",
"b": "tau9",
"ref_wins": 35,
"ref_rounds": 70,
"arm_wins": 22,
"arm_rounds": 70,
"p_fisher": 0.03855970003210523
},
{
"metric": "round_wins_pooled",
"a": "old",
"b": "tau15",
"ref_wins": 35,
"ref_rounds": 70,
"arm_wins": 32,
"arm_rounds": 70,
"p_fisher": 0.7352259013723954
}
],
"mechanism_tests": [
{
"metric": "central_box_occupancy_pct",
"a": "old",
"b": "pillaoff",
"mean_a": 0.086,
"mean_b": 2.372,
"diff_b_minus_a": 2.286,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"mde_abs": 0.113
},
{
"metric": "central_box_occupancy_pct",
"a": "old",
"b": "tau3",
"mean_a": 0.086,
"mean_b": 26.122,
"diff_b_minus_a": 26.036,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"mde_abs": 0.113
},
{
"metric": "central_box_occupancy_pct",
"a": "old",
"b": "tau5",
"mean_a": 0.086,
"mean_b": 21.299,
"diff_b_minus_a": 21.214,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"mde_abs": 0.113
},
{
"metric": "central_box_occupancy_pct",
"a": "old",
"b": "tau9",
"mean_a": 0.086,
"mean_b": 10.398,
"diff_b_minus_a": 10.313,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"mde_abs": 0.113
},
{
"metric": "central_box_occupancy_pct",
"a": "old",
"b": "tau15",
"mean_a": 0.086,
"mean_b": 4.682,
"diff_b_minus_a": 4.597,
"p_perm": 1.082508822446903e-05,
"method": "exact",
"mde_abs": 0.113
},
{
"metric": "mean_dist_to_enemy_px",
"a": "old",
"b": "pillaoff",
"mean_a": 469.275,
"mean_b": 443.53,
"diff_b_minus_a": -25.745,
"p_perm": 0.00019485158804044252,
"method": "exact",
"mde_abs": 20.809
},
{
"metric": "mean_dist_to_enemy_px",
"a": "old",
"b": "tau3",
"mean_a": 469.275,
"mean_b": 439.378,
"diff_b_minus_a": -29.897,
"p_perm": 0.0004330035289787612,
"method": "exact",
"mde_abs": 20.809
},
{
"metric": "mean_dist_to_enemy_px",
"a": "old",
"b": "tau5",
"mean_a": 469.275,
"mean_b": 447.833,
"diff_b_minus_a": -21.441,
"p_perm": 0.0013964363809565049,
"method": "exact",
"mde_abs": 20.809
},
{
"metric": "mean_dist_to_enemy_px",
"a": "old",
"b": "tau9",
"mean_a": 469.275,
"mean_b": 481.093,
"diff_b_minus_a": 11.819,
"p_perm": 0.08132888783043582,
"method": "exact",
"mde_abs": 20.809
},
{
"metric": "mean_dist_to_enemy_px",
"a": "old",
"b": "tau15",
"mean_a": 469.275,
"mean_b": 500.096,
"diff_b_minus_a": 30.821,
"p_perm": 0.0001082508822446903,
"method": "exact",
"mde_abs": 20.809
},
{
"metric": "enemy_bullet_within_100px_pct",
"a": "old",
"b": "pillaoff",
"mean_a": 27.994,
"mean_b": 29.103,
"diff_b_minus_a": 1.11,
"p_perm": 0.1381064755677759,
"method": "exact",
"mde_abs": 1.736
},
{
"metric": "enemy_bullet_within_100px_pct",
"a": "old",
"b": "tau3",
"mean_a": 27.994,
"mean_b": 30.86,
"diff_b_minus_a": 2.866,
"p_perm": 0.0014289116456299119,
"method": "exact",
"mde_abs": 1.736
},
{
"metric": "enemy_bullet_within_100px_pct",
"a": "old",
"b": "tau5",
"mean_a": 27.994,
"mean_b": 30.6,
"diff_b_minus_a": 2.606,
"p_perm": 0.0005196042347745134,
"method": "exact",
"mde_abs": 1.736
},
{
"metric": "enemy_bullet_within_100px_pct",
"a": "old",
"b": "tau9",
"mean_a": 27.994,
"mean_b": 28.313,
"diff_b_minus_a": 0.319,
"p_perm": 0.606605468834571,
"method": "exact",
"mde_abs": 1.736
},
{
"metric": "enemy_bullet_within_100px_pct",
"a": "old",
"b": "tau15",
"mean_a": 27.994,
"mean_b": 25.567,
"diff_b_minus_a": -2.426,
"p_perm": 9.742579402022126e-05,
"method": "exact",
"mde_abs": 1.736
},
{
"metric": "mean_dist_nearest_enemy_bullet_px",
"a": "old",
"b": "pillaoff",
"mean_a": 136.074,
"mean_b": 132.781,
"diff_b_minus_a": -3.293,
"p_perm": 0.015544826690337526,
"method": "exact",
"mde_abs": 3.026
},
{
"metric": "mean_dist_nearest_enemy_bullet_px",
"a": "old",
"b": "tau3",
"mean_a": 136.074,
"mean_b": 134.838,
"diff_b_minus_a": -1.236,
"p_perm": 0.31056095607179196,
"method": "exact",
"mde_abs": 3.026
},
{
"metric": "mean_dist_nearest_enemy_bullet_px",
"a": "old",
"b": "tau5",
"mean_a": 136.074,
"mean_b": 134.067,
"diff_b_minus_a": -2.007,
"p_perm": 0.11298144579878326,
"method": "exact",
"mde_abs": 3.026
},
{
"metric": "mean_dist_nearest_enemy_bullet_px",
"a": "old",
"b": "tau9",
"mean_a": 136.074,
"mean_b": 137.757,
"diff_b_minus_a": 1.683,
"p_perm": 0.11325207300439498,
"method": "exact",
"mde_abs": 3.026
},
{
"metric": "mean_dist_nearest_enemy_bullet_px",
"a": "old",
"b": "tau15",
"mean_a": 136.074,
"mean_b": 141.354,
"diff_b_minus_a": 5.279,
"p_perm": 2.165017644893806e-05,
"method": "exact",
"mde_abs": 3.026
}
]
}
Reference in New Issue View Git Blame Copy Permalink