SirRoboGarage/common_libs/tests/measure_melee_bitbrain_ab.nim

## MELEE A/B: is BitBrain actually better than Pattern in melee?
##
## The repo's FIRST melee measurement. Runs a 4-bot Free-For-All
## (ModularBot + WaveSurfer + PatternMover + RandomMover) for each arm, where
## the ONLY thing that differs between arms is ModularBot's gun-rack environment
## (`TR_RACK_*` / `TR_BITBRAIN_*`). Every other bot is held fixed.
##
## Why melee needs its own measurement: Pattern's advantage comes from
## accumulating history with ONE enemy. Melee rotates targets, fragmenting
## per-enemy history, so a gun that resets per target and re-adapts fast
## (BitBrain, `TR_BITBRAIN_RESET_ON_TARGET=on` by default) could gain exactly
## where Pattern loses. That is the hypothesis; this runner produces the number.
##
## Design (see docs/melee_bitbrain_ab.md):
##   * arms differ ONLY in the ModularBot wrapper script's exported env;
##   * each run gets its own ModularBot bot-dir + stdout/stderr log, so liveness
##     (`[env] rack active melee = ...`) and target-switching evidence
##     (`[config] ... target=#N`, `[bb-reset] reason=target_change`) are captured;
##   * the runner is driven in one process per arm from a shell driver; each
##     process gets its own Tank Royale server (ephemeral port).
##
## Usage:
##   nim c -r --nimcache:/tmp/nc_j116 --path:common_libs \
##     common_libs/tests/measure_melee_bitbrain_ab.nim --list-arms
##
##   MELEE_SERVER_JAR=<jar> \
##   common_libs/tests/measure_melee_bitbrain_ab.nim \
##       --outdir /tmp/melee_ab --build-only
##
##   common_libs/tests/measure_melee_bitbrain_ab.nim \
##       --outdir /tmp/melee_ab --arm pattern --run-start 1 --run-count 16
##
## Arm env (verbatim, from the arms table below) is written into the per-run
## wrapper script; the runner process' own environment is NOT mutated for the
## bot, so parallel arm processes cannot leak into each other.

import std/[os, osproc, strutils, strformat, json, sequtils]

import test_framework/server_manager
import test_framework/runner_process
import test_framework/battle_result

# ── the field (fixed across arms) ────────────────────────────────────────────
const
  repoSuffix = "common_libs" / "test_framework" / "adversaries"
  AdversaryNames = ["WaveSurfer", "PatternMover", "RandomMover"]
    ## Mixed field: one strong reactive mover (WaveSurfer, linear gun), one
    ## deterministic pattern mover (PatternMover), one erratic mover
    ## (RandomMover). All three shoot.

# ── the arms (ONLY ModularBot's gun rack differs) ────────────────────────────
type
  Arm = object
    name*: string
    env*:  seq[string]   ## "KEY=VALUE", exported in the per-run wrapper script
    note*: string

const Arms*: array[4, Arm] = [
  Arm(name: "pattern", env: @[],
      note: "shipped default: Pattern-only rack, no env"),
  Arm(name: "bb_round",
      env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee",
             "TR_BITBRAIN_MEM=perRound", "TR_BITBRAIN_LOG=1"],
      note: "BitBrain-only melee rack, perRound memory"),
  Arm(name: "bb_ret",
      env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee",
             "TR_BITBRAIN_MEM=retained", "TR_BITBRAIN_LOG=1"],
      note: "BitBrain-only melee rack, retained memory"),
  Arm(name: "bb_learn",
      env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee",
             "TR_BITBRAIN_MEM=decay", "TR_BITBRAIN_GAINS=1.0,1.25,1.5,2.0",
             "TR_BITBRAIN_LOG=1"],
      note: "BitBrain-only melee rack, decay memory + learned gains (user config)"),
]

const
  ModularBotJson = """{
  "name": "ModularBot",
  "version": "0.1.0",
  "authors": ["Davide Cappellini"],
  "description": "frozen melee A/B build of ModularBot",
  "homepage": "",
  "countryCodes": ["IT"],
  "gameTypes": ["classic", "1v1"],
  "platform": "Nim",
  "programmingLang": "Nim"
}
"""

# ANSI SGR sequences (the bot's config line color-highlights changed fields).
const AnsiRe = "\x1b\\[[0-9;]*m"

proc stripAnsi(s: string): string =
  result = newStringOfCap(s.len)
  var i = 0
  while i < s.len:
    if s[i] == '\x1b' and i + 1 < s.len and s[i+1] == '[':
      i += 2
      while i < s.len and s[i] notin {'m'}: inc i
      if i < s.len: inc i          # skip the terminating 'm'
    else:
      result.add s[i]
      inc i

proc repoRoot(): string =
  currentSourcePath().parentDir.parentDir.parentDir

proc quote(s: string): string = quoteShell(s)

proc params(): seq[string] =
  for i in 1 .. paramCount(): result.add paramStr(i)

proc findParam(args: seq[string], name: string, default = ""): string =
  for i in 0 ..< args.len:
    if args[i] == name and i + 1 < args.len: return args[i+1]
  default

proc hasFlag(args: seq[string], name: string): bool =
  for a in args:
    if a == name: return true
  false

# ── frozen ModularBot build (from `git archive HEAD`, like tools/ab/ab_run.sh) ─

proc binarySha(bin: string): string =
  let (outp, rc) = execCmdEx("sha256sum " & quote(bin))
  if rc == 0: outp.strip().split(' ')[0] else: "unknown"

proc headCommit(repo: string): string =
  let (outp, rc) = execCmdEx("git -C " & quote(repo) & " rev-parse HEAD")
  if rc == 0: outp.strip() else: "unknown"

proc buildFrozen(repo, outdir: string): string =
  ## Compile ONE frozen ModularBot from the committed HEAD snapshot and return
  ## its path. A dirty tree cannot leak into the measurement.
  let bin = outdir / "frozen" / "ModularBot_bin"
  if fileExists(bin): return bin
  createDir(outdir / "frozen")
  let snapshot = outdir / "frozen" / "head"
  createDir(snapshot)
  let (arcOut, arcRc) = execCmdEx(
    "git -C " & quote(repo) & " archive HEAD | tar -x -C " & quote(snapshot))
  if arcRc != 0:
    raise newException(IOError, "git archive HEAD failed:\n" & arcOut)
  let garage = snapshot / "ModularBot_garage"
  if not dirExists(garage):
    raise newException(IOError, "no ModularBot_garage in HEAD snapshot")
  let nimcache = outdir / "nimcache"
  let cmd = "nim c -d:release --nimcache:" & quote(nimcache) &
            " --out:" & quote(bin) & " src/ModularBot.nim"
  let (outp, rc) = execCmdEx(cmd, workingDir = garage)
  if rc != 0 or not fileExists(bin):
    raise newException(OSError, "frozen ModularBot build failed:\n" & outp)
  result = bin

# ── per-run wrapper bot dir ──────────────────────────────────────────────────

proc writeWrapper(botDir: string, env: seq[string], bin, stdoutLog, stderrLog: string) =
  createDir(botDir)
  writeFile(botDir / "ModularBot.json", ModularBotJson)
  var sh = "#!/bin/sh\ncd \"$(dirname \"$0\")\"\n"
  for kv in env:
    let p = kv.split('=', 1)
    sh.add("export " & p[0] & "=\"" & p[1] & "\"\n")
  sh.add("exec ./ModularBot_bin > " & quote(stdoutLog) &
         " 2> " & quote(stderrLog) & "\n")
  writeFile(botDir / "ModularBot.sh", sh)
  discard execCmdEx("chmod +x " & quote(botDir / "ModularBot.sh"))
  if fileExists(botDir / "ModularBot_bin"): removeFile(botDir / "ModularBot_bin")
  createSymlink(bin, botDir / "ModularBot_bin")

# ── liveness / target-switching extraction from the bot's stdout ─────────────

type Live = object
  envRackPattern: string
  envRackBitbrain: string
  envBitbrainMem: string
  envBitbrainGains: string
  envMovement: string
  rackActiveMelee: string
  rackActive1v1: string
  rackMeleeLines: int
  configLines: int
  distinctTargets: int
  targetChanges: int
  bbResetTargetChange: int
  bbLines: int
  resultEndWon: string

proc envLine(text, key: string): string =
  ## Value of an `[env] KEY = VALUE` (or `[env] KEY=VALUE`) boot-report line,
  ## with the `(source: ...)` annotation stripped. Empty when absent.
  let prefix = "[env] " & key
  for line in text.splitLines():
    let s = line.strip()
    if not s.startsWith(prefix): continue
    var rest = s[prefix.len .. ^1].strip()
    if rest.len == 0 or rest[0] != '=': continue
    rest = rest[1 .. ^1].strip()
    let si = rest.find("(source:")
    if si >= 0: rest = rest[0 ..< si].strip()
    return rest

proc parseLive(text: string): Live =
  var lastTarget = -1
  var seen: seq[int]
  for raw in text.splitLines():
    let s = stripAnsi(raw).strip()
    if s.startsWith("[config] "):
      inc result.configLines
      let i = s.find("target=#")
      if i >= 0:
        var j = i + "target=#".len
        var neg = false
        if j < s.len and s[j] == '-': neg = true; inc j
        var num = 0
        var any = false
        while j < s.len and s[j] in {'0'..'9'}:
          num = num * 10 + (ord(s[j]) - ord('0')); inc j; any = true
        if any and not neg:
          if lastTarget >= 0 and num != lastTarget: inc result.targetChanges
          lastTarget = num
          if num notin seen: seen.add num
    elif s.startsWith("[rack] "):
      if " mode=melee " in s or s.endsWith(" mode=melee"):
        inc result.rackMeleeLines
    elif s.startsWith("[bb-reset] ") and "reason=target_change" in s:
      inc result.bbResetTargetChange
    elif s.startsWith("[bb] "):
      inc result.bbLines
    elif s.startsWith("[result] battle END:"):
      let i = s.find("rounds won ")
      if i >= 0: result.resultEndWon = s[i + "rounds won ".len .. ^1].strip()
  result.distinctTargets = seen.len
  result.envRackPattern     = envLine(text, "TR_RACK_PATTERN")
  result.envRackBitbrain    = envLine(text, "TR_RACK_BITBRAIN")
  result.envBitbrainMem     = envLine(text, "TR_BITBRAIN_MEM")
  result.envBitbrainGains   = envLine(text, "TR_BITBRAIN_GAINS")
  result.envMovement        = envLine(text, "TR_MOVEMENT")
  result.rackActiveMelee    = envLine(text, "rack active melee")
  result.rackActive1v1      = envLine(text, "rack active 1v1")

# ── one run ──────────────────────────────────────────────────────────────────

proc runOne(repo, outdir, bin, armName: string, env: seq[string],
            run, rounds, timeout: int, commit, binsha: string): bool =
  let armDir = outdir / armName
  createDir(armDir)
  let work = outdir / "work" / armName / ("run" & $run)
  let botDir = work / "ModularBot"
  let outLog = armDir / ("run" & $run & ".bot.stdout.log")
  let errLog = armDir / ("run" & $run & ".bot.stderr.log")
  let runLog = armDir / ("run" & $run & ".runner.log")
  let jsonPath = armDir / ("run" & $run & ".json")
  if fileExists(outLog): removeFile(outLog)
  if fileExists(errLog): removeFile(errLog)
  writeWrapper(botDir, env, bin, outLog, errLog)

  var advDirs: seq[string]
  for a in AdversaryNames: advDirs.add repo / repoSuffix / a

  var ok = true
  var raw = ""
  try:
    raw = runBattleRunner(getServerUrl(), @[botDir] & advDirs, rounds, timeout, true)
  except CatchableError as e:
    ok = false
    raw = "EXCEPTION: " & e.msg
  writeFile(runLog, raw)

  if not ok:
    writeFile(jsonPath, $(%*{"arm": armName, "run": run, "rounds": rounds,
                              "commit": commit, "binary_sha256": binsha,
                              "ok": false, "error": raw}))
    stderr.writeLine(fmt"[melee] {armName} run {run}: FAILED")
    return false

  let r = parseServerOutput(raw)
  var perRound: seq[tuple[round, score, rank: int, survived: bool]]
  var botRd: seq[JsonNode]
  for rr in r.rounds:
    for br in rr.results:
      if br.name == "ModularBot":
        perRound.add (rr.round, br.score, br.rank, br.survived)

  var modJ = %*{"name": "ModularBot", "found": false}
  for b in r.results:
    if b.name == "ModularBot":
      modJ = %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank,
                "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount,
                "found": true}
  var field = newJArray()
  for b in r.results:
    if b.name == "ModularBot": continue
    field.add %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank,
                 "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount}

  for pr in perRound:
    botRd.add %*{"round": pr.round, "score": pr.score, "rank": pr.rank,
                 "survived": pr.survived}

  var live = Live()
  if fileExists(outLog):
    live = parseLive(readFile(outLog))

  let row = %*{
    "arm": armName, "run": run, "rounds": rounds,
    "commit": commit, "binary_sha256": binsha,
    "ok": true,
    "modularbot": modJ,
    "field": field,
    "perRound": botRd,
    "live": %*{
      "env_TR_RACK_PATTERN": live.envRackPattern,
      "env_TR_RACK_BITBRAIN": live.envRackBitbrain,
      "env_TR_BITBRAIN_MEM": live.envBitbrainMem,
      "env_TR_BITBRAIN_GAINS": live.envBitbrainGains,
      "env_TR_MOVEMENT": live.envMovement,
      "rack_active_melee": live.rackActiveMelee,
      "rack_active_1v1": live.rackActive1v1,
      "rack_melee_lines": live.rackMeleeLines,
      "config_lines": live.configLines,
      "distinct_targets": live.distinctTargets,
      "target_changes": live.targetChanges,
      "bb_reset_target_change": live.bbResetTargetChange,
      "bb_lines": live.bbLines,
      "result_end_won": live.resultEndWon
    }
  }
  writeFile(jsonPath, $(row))
  let mb = r.results.filterIt(it.name == "ModularBot")
  let wins = if mb.len > 0: mb[0].firstPlaces else: 0
  let rk = if mb.len > 0: mb[0].rank else: 0
  let sc = if mb.len > 0: mb[0].totalScore else: 0
  echo fmt"[melee] {armName} run {run}: wins={wins}/{rounds} rank={rk} score={sc} " &
       fmt"targets={live.distinctTargets} changes={live.targetChanges} " &
       fmt"rackMelee={live.rackActiveMelee} bbreset={live.bbResetTargetChange}"
  result = true

# ── main ─────────────────────────────────────────────────────────────────────

proc main() =
  let repo = repoRoot()
  let args = params()
  if hasFlag(args, "--list-arms") or hasFlag(args, "--help") or args.len == 0:
    echo "arms:"
    for a in Arms:
      echo a.name & "\t" & a.env.join(" ") & "\t" & a.note
    return
  let outdir = findParam(args, "--outdir", "/tmp/melee_bitbrain_ab")
  createDir(outdir)
  let commit = headCommit(repo)
  let bin = buildFrozen(repo, outdir)
  let binsha = binarySha(bin)
  echo "[melee] commit=", commit, " binary_sha256=", binsha
  if hasFlag(args, "--build-only"): return

  let armName = findParam(args, "--arm")
  if armName.len == 0:
    stderr.writeLine("--arm is required (or --list-arms / --build-only)")
    quit(1)
  var arm: Arm
  var found = false
  for a in Arms:
    if a.name == armName: arm = a; found = true
  if not found:
    stderr.writeLine("unknown arm: " & armName)
    quit(1)

  let runStart = parseInt(findParam(args, "--run-start", "1"))
  let runCount = parseInt(findParam(args, "--run-count", "15"))
  let rounds = parseInt(findParam(args, "--rounds", "7"))
  let timeout = parseInt(findParam(args, "--timeout", "900000"))
  let retries = parseInt(findParam(args, "--retries", "2"))

  # `--runs 1,3,5` overrides the contiguous range (used to fill failed runs).
  var runList: seq[int]
  let runsSpec = findParam(args, "--runs", "")
  if runsSpec.len > 0:
    for t in runsSpec.split(','):
      let s = t.strip()
      if s.len > 0: runList.add parseInt(s)
  else:
    for run in runStart ..< runStart + runCount: runList.add run

  ensureServer()
  echo "[melee] server=", getServerUrl(), " arm=", armName,
       " runs=", runList.len, " rounds=", rounds, " retries=", retries

  var ok = 0
  for run in runList:
    var done = false
    for attempt in 1 .. retries + 1:
      if runOne(repo, outdir, bin, armName, arm.env, run, rounds, timeout, commit, binsha):
        done = true
        break
      if attempt <= retries:
        stderr.writeLine(fmt"[melee] {armName} run {run}: retry {attempt}/{retries} in 8s")
        sleep(8000)
    if done: inc ok
  echo fmt"[melee] {armName}: {ok}/{runList.len} runs ok"

when isMainModule:
  main()