SirRoboGarage/common_libs/tests/run_prediction_quality.nim

## Offline PREDICTION-QUALITY runner — the campaign's measurement sweep.
##
## Reads the recorded live-vs-real-DrussGT corpus, drives each arm over the
## recorded enemy trajectory, and scores every tick×power-bin prediction against
## the aim-independent interception point (see prediction_quality.nim). Owns the
## RANGE-BAND table that is "the bar" for the BitBrain campaign.
##
## NO CLOSED-LOOP CLAIM IS MADE HERE. Every arm below is an open-loop prediction
## scored on a FIXED trajectory. Wins, damage and survival are decided live.
##
## Usage:
##   nim c -r --nimcache:/tmp/nc_j98 common_libs/tests/run_prediction_quality.nim \
##       [--corpus /tmp/tfil_ab2/out] [--limit N] [--timing]
##
## `--limit N` keeps only the first N runs (sorted), for fast iteration.

import std/[os, strformat, strutils, times, math]
import gun_harness/[gun_interface, virtual_bullets, prediction_quality]
import guns/[head_on, pattern_matcher, tm_horizon, bitbrain_gun]

# arm indices (fixed order = fixed output)
const
  A_ORACLE* = 0
  A_ORACLEQ* = 1
  A_HEADON* = 2
  A_PATTERN* = 3
  A_G15* = 4
  A_G20* = 5
  A_G30* = 6
  A_NAIVE* = 7
  A_TMH* = 8
  A_BB* = 9
  ArmNames* = ["Oracle", "OracleQuant", "HeadOn", "Pattern", "PatternGain1.5",
               "PatternGain2.0", "PatternGain3.0", "NaiveLinear", "TMHorizon", "BitBrain"]

# ── the naive-linear control (job-95's LIN_M = 4 extrapolation) ──────────────
#
# Velocity = (pos(t) - pos(t-4)) / 4, then iterate the interception equation.
# This is the trivial predictive gun the lead-capture analysis used as its
# ceiling control; it captures ~2x the lead response Pattern does at 450+ and is
# included to resolve that tension against the angular-error ruler.

type
  NaiveLinearGun = object
    hist: array[5, tuple[x, y: float]]
    count: int
    lastTick: int

proc predict*(g: var NaiveLinearGun, state: WorldState,
              bulletSpeed: float): GunPrediction =
  if state.tick != g.lastTick:
    for i in countdown(4, 1): g.hist[i] = g.hist[i - 1]
    g.hist[0] = (state.enemyX, state.enemyY)
    if g.count < 5: inc g.count
    g.lastTick = state.tick
  if g.count < 5 or bulletSpeed <= 0.0:
    return GunPrediction(x: state.enemyX, y: state.enemyY)
  let vx = (g.hist[0].x - g.hist[4].x) / 4.0
  let vy = (g.hist[0].y - g.hist[4].y) / 4.0
  let ox = state.selfX
  let oy = state.selfY
  var t = hypot(state.enemyX - ox, state.enemyY - oy) / bulletSpeed
  for _ in 0..<40:
    t = hypot(state.enemyX + vx * t - ox, state.enemyY + vy * t - oy) / bulletSpeed
  GunPrediction(x: state.enemyX + vx * t, y: state.enemyY + vy * t)

proc onResult*(g: var NaiveLinearGun, e: FeedbackEvent) = discard

# ── runner ───────────────────────────────────────────────────────────────────

type Ctx = object
  c: Corpus
  cont: bool
  pattern: PatternMatcherGun
  naive: NaiveLinearGun
  tmh: TmHorizonGun
  bb: BitBrainGun
  headon: HeadOnGun
  st: WorldState
  enemy: seq[EnemyInfo]

proc runRound(ctx: var Ctx, arms: var seq[ArmAcc], r: int) =
  let c = ctx.c
  let base = int(c.rStart[r]) - c.base
  let cnt = int(c.rCount[r])
  let iEnd = base + cnt
  ctx.enemy[0] = EnemyInfo(id: 1)
  for i in base ..< iEnd:
    let ox = c.sx(i)
    let oy = c.sy(i)
    let localTick = int(c.tick[i]) - int(c.rStart[r])
    ctx.enemy[0].x = c.ex(i)
    ctx.enemy[0].y = c.ey(i)
    ctx.enemy[0].heading = c.eh(i)
    ctx.enemy[0].speed = c.es(i)
    ctx.enemy[0].energy = c.ee(i)
    ctx.enemy[0].lastSeenTick = localTick
    ctx.st.enemyX = c.ex(i)
    ctx.st.enemyY = c.ey(i)
    ctx.st.enemyHeading = c.eh(i)
    ctx.st.enemySpeed = c.es(i)
    ctx.st.enemyEnergy = c.ee(i)
    ctx.st.selfX = ox
    ctx.st.selfY = oy
    ctx.st.selfHeading = c.sh(i)
    ctx.st.selfSpeed = c.ss(i)
    ctx.st.selfEnergy = c.se(i)
    ctx.st.selfRadarHeading = c.sh(i)
    ctx.st.tick = localTick
    let los = bearingDeg(ox, oy, c.ex(i), c.ey(i))
    for bin in 0 ..< len(PowerBins):
      let speed = bulletSpeed(PowerBins[bin])
      let ib = interceptBearing(c, i, iEnd, ox, oy, speed, ctx.cont)
      if not ib.ok:
        for ai in 0 ..< arms.len: skip(arms[ai])
        continue
      let ibq = interceptBearingQuant(c, i, iEnd, ox, oy, speed)
      let rng = ib.range
      let targetLead = wrap180(ib.bearing - los)
      # oracle: aims at the active ruler's true interception point -> 0 error
      arms[A_ORACLE].record(rng, 0.0, targetLead, targetLead)
      # the analyze_lead_capture_by_range.py integer-tick intercept, scored on the
      # SAME ruler: this is what the coarse solve's own oracle would reach.
      let qLead = if ibq.ok: wrap180(ibq.bearing - los) else: targetLead
      arms[A_ORACLEQ].record(rng, wrap180(qLead - targetLead), qLead, targetLead)
      # head-on: aim at the enemy's CURRENT position (worst realistic gun)
      arms[A_HEADON].record(rng, wrap180(los - ib.bearing), 0.0, targetLead)
      # pattern + lead-gain sweep (gain scales Pattern's lead over LOS)
      let pp = ctx.pattern.predict(ctx.st, speed)
      let pb = bearingDeg(ox, oy, pp.x, pp.y)
      let plead = wrap180(pb - los)
      arms[A_PATTERN].record(rng, wrap180(plead - targetLead), plead, targetLead)
      arms[A_G15].record(rng, wrap180(1.5 * plead - targetLead), 1.5 * plead, targetLead)
      arms[A_G20].record(rng, wrap180(2.0 * plead - targetLead), 2.0 * plead, targetLead)
      arms[A_G30].record(rng, wrap180(3.0 * plead - targetLead), 3.0 * plead, targetLead)
      # naive linear
      let np = predict(ctx.naive, ctx.st, speed)
      let nl = wrap180(bearingDeg(ox, oy, np.x, np.y) - los)
      arms[A_NAIVE].record(rng, wrap180(nl - targetLead), nl, targetLead)
      # TMHorizon
      let tp = predict(ctx.tmh, ctx.st, speed)
      let tl = wrap180(bearingDeg(ox, oy, tp.x, tp.y) - los)
      arms[A_TMH].record(rng, wrap180(tl - targetLead), tl, targetLead)
      # BitBrain (base Pattern + ADE/SBC corrector)
      let bp = predict(ctx.bb, ctx.st, speed)
      let bl = wrap180(bearingDeg(ox, oy, bp.x, bp.y) - los)
      arms[A_BB].record(rng, wrap180(bl - targetLead), bl, targetLead)

proc runOne(runPath: string, arms: var seq[ArmAcc], shotsCont, shotsQuant: var ShotStat,
            doShots: bool, timing: bool, cont: bool): int =
  let t0 = epochTime()
  let c = loadCorpus(runPath)
  if c.n == 0: return 0
  var ctx = Ctx(
    c: c,
    cont: cont,
    pattern: PatternMatcherGun(),
    naive: NaiveLinearGun(lastTick: -1),
    tmh: initTmHorizonGun(),
    bb: initBitBrainGun(),
    headon: HeadOnGun(),
    st: WorldState(arenaWidth: c.arenaW, arenaHeight: c.arenaH),
    enemy: newSeq[EnemyInfo](1))
  for r in 0 ..< c.rStart.len:
    runRound(ctx, arms, r)
  if doShots:
    let ev = parseEvents(eventsPathFor(runPath))
    for mode in [true, false]:
      let s = validateShots(c, ev, mode)
      var dst = if mode: addr shotsCont else: addr shotsQuant
      dst.hits += s.hits
      dst.misses += s.misses
      dst.hitSumDeg += s.hitSumDeg
      dst.missSumDeg += s.missSumDeg
      dst.hitSumPx += s.hitSumPx
      dst.missSumPx += s.missSumPx
  if timing:
    stderr.writeLine(fmt"  {extractFilename(runPath):<16} ticks={c.n:<7} {epochTime()-t0:>6.2f}s")
  c.n

proc fmt4(x: float): string =
  if x.classify in {fcNan, fcInf, fcNegInf}: "-" else: fmt"{x:.4f}"

proc fmt3(x: float): string =
  if x.classify in {fcNan, fcInf, fcNegInf}: "-" else: fmt"{x:.3f}"

proc main() =
  var corpusRoot = "/tmp/tfil_ab2/out"
  var limit = 0
  var doShots = true
  var timing = false
  var cont = true
  var i = 1
  while i <= paramCount():
    case paramStr(i)
    of "--corpus": inc i; corpusRoot = paramStr(i)
    of "--limit": inc i; limit = parseInt(paramStr(i))
    of "--no-shots": doShots = false
    of "--timing": timing = true
    of "--ruler":
      inc i
      cont = paramStr(i) != "quant"
    else:
      stderr.writeLine("unknown arg: " & paramStr(i))
      quit(2)
    inc i

  var runs = discoverRuns(corpusRoot)
  if limit > 0 and runs.len > limit: runs.setLen(limit)
  if runs.len == 0:
    stderr.writeLine("no runs found under " & corpusRoot)
    quit(1)

  var arms: seq[ArmAcc]
  for nm in ArmNames: arms.add ArmAcc(name: nm)
  var shotsCont, shotsQuant: ShotStat

  var t0 = epochTime()
  var ticks = 0
  for rp in runs:
    ticks += runOne(rp, arms, shotsCont, shotsQuant, doShots, timing, cont)
  let elapsed = epochTime() - t0

  echo "=".repeat(120)
  echo "OFFLINE PREDICTION QUALITY  --  per-gun single-tick aim error vs the true interception point"
  echo "=".repeat(120)
  echo fmt"corpus        : {corpusRoot}"
  let rulerName = if cont: "continuous (physically exact)" else: "integer-tick (analyze_lead_capture_by_range.py)"
  echo fmt"ruler         : {rulerName}"
  echo fmt"runs          : {runs.len}"
  echo fmt"recorded ticks: {ticks}"
  let tickBins = ticks * len(PowerBins)
  echo fmt"tick x bin    : {tickBins}"
  echo fmt"wall time     : {elapsed:.2f}s   ({elapsed / max(1.0, float(tickBins)) * 1000.0:.4f} ms per tick-bin)"
  echo fmt"per-arm speed : {elapsed / max(1.0, float(tickBins)) * 1000.0:.4f} s per 1000 tick-bins per arm"
  echo ""
  echo "NOTE: offline OPEN-LOOP prediction quality only. No win/damage/survival claim."
  echo ""

  # validation block
  echo "=".repeat(120)
  echo "VALIDATION  --  the ruler must pass ALL of these before any number below is trusted"
  echo "=".repeat(120)
  if doShots and shotsCont.hits > 0 and shotsQuant.hits > 0:
    echo fmt"1. recorded shots (OUR actual server-fired bearings vs the SAME interception solve):"
    for mode in [("continuous", shotsCont), ("integer", shotsQuant)]:
      let s = mode[1]
      let sepOk = if separationPx(s) > 2.0: "OK" else: "WEAK"
      echo fmt"   ruler={mode[0]:<11} hits n={s.hits:<6} mean|err|={meanHitDeg(s):>7.3f} deg / {meanHitPx(s):>7.1f} px | " &
           fmt"misses n={s.misses:<6} mean|err|={meanMissDeg(s):>7.3f} deg / {meanMissPx(s):>7.1f} px | " &
           fmt"separation {separationDeg(s):>6.2f}x deg / {separationPx(s):>6.2f}x px -> {sepOk}"
  else:
    echo "1. recorded-shot validation: SKIPPED"
  let oMax = max([arms[A_ORACLE].bands[0].maxAbs, arms[A_ORACLE].bands[1].maxAbs,
                  arms[A_ORACLE].bands[2].maxAbs, arms[A_ORACLE].bands[3].maxAbs,
                  arms[A_ORACLE].bands[4].maxAbs])
  let oOk = if oMax < 1e-6: "OK" else: "BROKEN"
  echo fmt"2. perfect-oracle gun max |err| over all tick-bins = {oMax:.6f} deg  -> {oOk}"
  # ordering check: the static LOS gun must be worse than every predictive gun.
  proc overallMean(arms: seq[ArmAcc], ai: int): float =
    var sAbs = 0.0
    var n = 0
    for b in 0 ..< NBands:
      sAbs += arms[ai].bands[b].sumAbs
      n += arms[ai].bands[b].n
    if n > 0: sAbs / float(n) else: 0.0
  let mHead = overallMean(arms, A_HEADON)
  let mPat = overallMean(arms, A_PATTERN)
  let mTmh = overallMean(arms, A_TMH)
  let mBb = overallMean(arms, A_BB)
  let mLin = overallMean(arms, A_NAIVE)
  let ordOk = mHead > mPat and mHead > mTmh and mHead > mBb
  echo fmt"3. HeadOn (static LOS) mean|err| = {mHead:.3f} deg vs Pattern {mPat:.3f} / TMHorizon {mTmh:.3f} / BitBrain {mBb:.3f}"
  let ordMsg = if ordOk: "OK (static gun worst among real guns)" else: "UNEXPECTED: a predictive gun is worse than static LOS"
  echo fmt"     -> {ordMsg}"
  echo fmt"     NaiveLinear mean|err| = {mLin:.3f} deg (over-leads; see the lead-gain sweep for why a larger"
  echo fmt"     lead *response* does not mean a smaller angular error)"
  echo "4. determinism: run twice and diff stdout (see fixture; verified separately)."
  echo ""

  echo "=".repeat(120)
  echo "THE BAR  --  per-band mean ABSOLUTE angular aim error (deg), RMSE, sign, hit-proxy"
  echo "=".repeat(120)
  echo "hitProxy = fraction of tick-bins with |err| <= atan(18/range) (the angular half-width of the target disc)."
  echo ""
  stdout.write formatArmTable(arms)

  echo ""
  echo "=".repeat(120)
  echo "HEADROOM  --  the direct answer: how far each arm is from the oracle ceiling, per band"
  echo "=" .repeat(120)
  let hdr = "band     Pattern n  Pattern|err|  Pattern hpx   Oracle hpx   headroom pp   naive hpx  TMHoriz hpx  BitBrain hpx"
  echo hdr
  echo "-".repeat(hdr.len)
  for b in 0 ..< NBands:
    let pat = arms[A_PATTERN].bands[b]
    let orc = arms[A_ORACLE].bands[b]
    let hp = pat.hitProxy
    let ohp = orc.hitProxy
    echo fmt"{BandLabels[b]:<9} {pat.n:>8} {fmt3(meanAbs(pat)):>12} {fmt4(hp):>12} {fmt4(ohp):>12} {ohp - hp:>13.4f} {fmt4(arms[A_NAIVE].bands[b].hitProxy):>11} {fmt4(arms[A_TMH].bands[b].hitProxy):>12} {fmt4(arms[A_BB].bands[b].hitProxy):>13}"
  echo ""
  echo "hitProxy = fraction of tick-bins aimed within atan(18/range) of the true interception point."
  echo "headroom pp = oracle hitProxy - Pattern hitProxy = the absolute hit-probability points available"
  echo "to a perfect predictor (the campaign is playing for a slice of this)."
  echo ""
  let hdrq = "band     OracleQuant hpx   integer-solve coarseness"
  echo hdrq
  echo "-".repeat(hdrq.len)
  for b in 0 ..< NBands:
    let oq = arms[A_ORACLEQ].bands[b]
    echo fmt"{BandLabels[b]:<9} {fmt4(oq.hitProxy):>15}   {fmt3(meanAbs(oq)):>10} deg mean |err|"
  echo "(OracleQuant aims at the analyze_lead_capture_by_range.py integer-tick intercept and is scored"
  echo " on the active ruler. On the continuous ruler it measures how much of a gun's 'error' the coarse"
  echo " solve itself would produce; on the integer ruler it is identically zero.)"

  echo ""
  echo "=".repeat(120)
  echo "LEAD-GAIN SWEEP ON PATTERN  --  multiply Pattern's lead (deg over LOS) by a constant"
  echo "=".repeat(120)
  let hdr2 = "band         gain=1.0   gain=1.5   gain=2.0   gain=3.0   best-gain"
  echo hdr2
  echo "-".repeat(hdr2.len)
  for b in 0 ..< NBands:
    let g1 = meanAbs(arms[A_PATTERN].bands[b])
    let g15 = meanAbs(arms[A_G15].bands[b])
    let g20 = meanAbs(arms[A_G20].bands[b])
    let g30 = meanAbs(arms[A_G30].bands[b])
    var best = "1.0"
    var bestV = g1
    if g15 < bestV: bestV = g15; best = "1.5"
    if g20 < bestV: bestV = g20; best = "2.0"
    if g30 < bestV: bestV = g30; best = "3.0"
    echo fmt"{BandLabels[b]:<9} {fmt3(g1):>10} {fmt3(g15):>10} {fmt3(g20):>10} {fmt3(g30):>10}   {best} ({fmt3(bestV)})"

  echo ""
  echo "=".repeat(120)
  echo "LEAD INFORMATIVENESS  --  capture slope (regression of applied lead on required lead) and lead correlation"
  echo "=".repeat(120)
  echo "capture slope is job-95's metric (1.0 = perfect proportional response). corr is the Pearson"
  echo "correlation of the arm's lead with the REQUIRED lead: a large slope on an uncorrelated lead is"
  echo "just amplified noise. This is the table that resolves the 'naive-linear captures 2x the lead but"
  echo "hits less' tension."
  echo ""
  let hdr3 = "band      HO |err|  HO |req|   Pat|req|   Pat cap  Pat corr   Lin cap  Lin corr   TMH cap  TMH corr    BB cap   BB corr"
  echo hdr3
  echo "-".repeat(hdr3.len)
  for b in 0 ..< NBands:
    let sp = arms[A_PATTERN].bands[b]
    let sn = arms[A_NAIVE].bands[b]
    let st = arms[A_TMH].bands[b]
    let sb = arms[A_BB].bands[b]
    echo fmt"{BandLabels[b]:<9} {fmt3(meanAbs(arms[A_HEADON].bands[b])):>9} {fmt3(meanAbsReq(arms[A_HEADON].bands[b])):>9} {fmt3(meanAbsReq(sp)):>9} {fmt3(captureSlope(sp)):>10} {fmt3(leadCorr(sp)):>10} " &
         fmt"{fmt3(captureSlope(sn)):>10} {fmt3(leadCorr(sn)):>10} {fmt3(captureSlope(st)):>10} " &
         fmt"{fmt3(leadCorr(st)):>10} {fmt3(captureSlope(sb)):>10} {fmt3(leadCorr(sb)):>10}"

when isMainModule:
  main()