j124 fair melee: add strong-field melee harness + driver (legacy champion field)

2026-09-26 04:08:44 +02:00
parent 06da7abb0e
commit 6c21dc4c58
2 changed files with 511 additions and 0 deletions
@@ -0,0 +1,414 @@
+## MELEE A/B — STRONG FIELD: is BitBrain actually better than Pattern in melee?
+##
+## This is the FAIR-FIELD rerun of the repo's first melee measurement
+## (`measure_melee_bitbrain_ab.nim` / docs/melee_bitbrain_ab.md, j116). That test
+## used a weak in-repo field (WaveSurfer + PatternMover + RandomMover) against
+## which ModularBot won 97-100% of rounds REGARDLESS of gun — a ceiling effect
+## that made the gun question untestable. Here the field is three
+## battle-validated legacy champions (default Diamond + Dookious + GresSuffurd)
+## that can actually punish a bad gun.
+##
+## Runs a 4-bot Free-For-All (ModularBot + 3 legacy champions) for each arm,
+## where the ONLY thing that differs between arms is ModularBot's gun-rack
+## environment (`TR_RACK_*` / `TR_BITBRAIN_*`). Every other bot is held fixed.
+##
+## Field override: `MELEE_FIELD=/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,...`
+## (absolute dirs; a bare name is resolved under the in-repo adversaries dir).
+##
+## Design (see docs/spinner_and_melee_claims.md):
+##   * arms differ ONLY in the ModularBot wrapper script's exported env;
+##   * each run gets its own ModularBot bot-dir + stdout/stderr log, so liveness
+##     (`[env] rack active melee = ...`) and target-switching evidence
+##     (`[config] ... target=#N`, `[bb-reset] reason=target_change`) are captured;
+##   * the runner is driven in one process per arm from a shell driver; each
+##     process gets its own Tank Royale server (ephemeral port).
+##
+## Usage:
+##   nim c -r --nimcache:/tmp/nc_j124 --path:common_libs \
+##     common_libs/tests/measure_melee_strong_field.nim --list-arms
+##
+##   MELEE_FIELD=/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd \
+##   common_libs/tests/measure_melee_strong_field.nim \
+##       --outdir /tmp/melee_strong --build-only
+##
+## Arm env (verbatim, from the arms table below) is written into the per-run
+## wrapper script; the runner process' own environment is NOT mutated for the
+## bot, so parallel arm processes cannot leak into each other.
+
+import std/[os, osproc, strutils, strformat, json, sequtils]
+
+import test_framework/server_manager
+import test_framework/runner_process
+import test_framework/battle_result
+
+# ── the field (fixed across arms) ────────────────────────────────────────────
+const
+  repoSuffix = "common_libs" / "test_framework" / "adversaries"
+  DefaultField = @["/tmp/tr_bots/Diamond", "/tmp/tr_bots/Dookious",
+                   "/tmp/tr_bots/GresSuffurd"]
+    ## Three battle-validated legacy champions: wave-surfing dodgers with real
+    ## guns (tools/robocode_shim/robots.json). They punish a bad gun, unlike the
+    ## j116 in-repo field where ModularBot won regardless of the rack.
+
+proc fieldDirs(repo: string): seq[string] =
+  ## `MELEE_FIELD` (comma-separated) overrides the default field. A bare name is
+  ## resolved under the in-repo adversaries dir; an absolute path is used as-is.
+  let spec = getEnv("MELEE_FIELD")
+  let raw = if spec.len > 0: spec.split(',') else: DefaultField
+  for item in raw:
+    let t = item.strip()
+    if t.len == 0: continue
+    result.add(if t.startsWith("/"): t else: repo / repoSuffix / t)
+
+# ── the arms (ONLY ModularBot's gun rack differs) ────────────────────────────
+type
+  Arm = object
+    name*: string
+    env*:  seq[string]   ## "KEY=VALUE", exported in the per-run wrapper script
+    note*: string
+
+const Arms*: array[3, Arm] = [
+  Arm(name: "pattern", env: @[],
+      note: "shipped default: Pattern-only rack, no env"),
+  Arm(name: "bb_ret",
+      env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee",
+             "TR_BITBRAIN_MEM=retained", "TR_BITBRAIN_LOG=1"],
+      note: "BitBrain-only melee rack, retained memory"),
+  Arm(name: "bb_decay",
+      env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee",
+             "TR_BITBRAIN_MEM=decay", "TR_BITBRAIN_GAINS=1.0,1.25,1.5,2.0",
+             "TR_BITBRAIN_LOG=1"],
+      note: "BitBrain-only melee rack, decay memory + learned gains"),
+]
+
+const
+  ModularBotJson = """{
+  "name": "ModularBot",
+  "version": "0.1.0",
+  "authors": ["Davide Cappellini"],
+  "description": "frozen melee A/B build of ModularBot",
+  "homepage": "",
+  "countryCodes": ["IT"],
+  "gameTypes": ["classic", "1v1"],
+  "platform": "Nim",
+  "programmingLang": "Nim"
+}
+"""
+
+# ANSI SGR sequences (the bot's config line color-highlights changed fields).
+const AnsiRe = "\x1b\\[[0-9;]*m"
+
+proc stripAnsi(s: string): string =
+  result = newStringOfCap(s.len)
+  var i = 0
+  while i < s.len:
+    if s[i] == '\x1b' and i + 1 < s.len and s[i+1] == '[':
+      i += 2
+      while i < s.len and s[i] notin {'m'}: inc i
+      if i < s.len: inc i          # skip the terminating 'm'
+    else:
+      result.add s[i]
+      inc i
+
+proc repoRoot(): string =
+  currentSourcePath().parentDir.parentDir.parentDir
+
+proc quote(s: string): string = quoteShell(s)
+
+proc params(): seq[string] =
+  for i in 1 .. paramCount(): result.add paramStr(i)
+
+proc findParam(args: seq[string], name: string, default = ""): string =
+  for i in 0 ..< args.len:
+    if args[i] == name and i + 1 < args.len: return args[i+1]
+  default
+
+proc hasFlag(args: seq[string], name: string): bool =
+  for a in args:
+    if a == name: return true
+  false
+
+# ── frozen ModularBot build (from `git archive HEAD`, like tools/ab/ab_run.sh) ─
+
+proc binarySha(bin: string): string =
+  let (outp, rc) = execCmdEx("sha256sum " & quote(bin))
+  if rc == 0: outp.strip().split(' ')[0] else: "unknown"
+
+proc headCommit(repo: string): string =
+  let (outp, rc) = execCmdEx("git -C " & quote(repo) & " rev-parse HEAD")
+  if rc == 0: outp.strip() else: "unknown"
+
+proc buildFrozen(repo, outdir: string): string =
+  ## Compile ONE frozen ModularBot from the committed HEAD snapshot and return
+  ## its path. A dirty tree cannot leak into the measurement.
+  let bin = outdir / "frozen" / "ModularBot_bin"
+  if fileExists(bin): return bin
+  createDir(outdir / "frozen")
+  let snapshot = outdir / "frozen" / "head"
+  createDir(snapshot)
+  let (arcOut, arcRc) = execCmdEx(
+    "git -C " & quote(repo) & " archive HEAD | tar -x -C " & quote(snapshot))
+  if arcRc != 0:
+    raise newException(IOError, "git archive HEAD failed:\n" & arcOut)
+  let garage = snapshot / "ModularBot_garage"
+  if not dirExists(garage):
+    raise newException(IOError, "no ModularBot_garage in HEAD snapshot")
+  let nimcache = "/tmp/nc_j124"
+  let cmd = "nim c -d:release --nimcache:" & quote(nimcache) &
+            " --out:" & quote(bin) & " src/ModularBot.nim"
+  let (outp, rc) = execCmdEx(cmd, workingDir = garage)
+  if rc != 0 or not fileExists(bin):
+    raise newException(OSError, "frozen ModularBot build failed:\n" & outp)
+  result = bin
+
+# ── per-run wrapper bot dir ──────────────────────────────────────────────────
+
+proc writeWrapper(botDir: string, env: seq[string], bin, stdoutLog, stderrLog: string) =
+  createDir(botDir)
+  writeFile(botDir / "ModularBot.json", ModularBotJson)
+  var sh = "#!/bin/sh\ncd \"$(dirname \"$0\")\"\n"
+  for kv in env:
+    let p = kv.split('=', 1)
+    sh.add("export " & p[0] & "=\"" & p[1] & "\"\n")
+  sh.add("exec ./ModularBot_bin > " & quote(stdoutLog) &
+         " 2> " & quote(stderrLog) & "\n")
+  writeFile(botDir / "ModularBot.sh", sh)
+  discard execCmdEx("chmod +x " & quote(botDir / "ModularBot.sh"))
+  if fileExists(botDir / "ModularBot_bin"): removeFile(botDir / "ModularBot_bin")
+  createSymlink(bin, botDir / "ModularBot_bin")
+
+# ── liveness / target-switching extraction from the bot's stdout ─────────────
+
+type Live = object
+  envRackPattern: string
+  envRackBitbrain: string
+  envBitbrainMem: string
+  envBitbrainGains: string
+  envMovement: string
+  rackActiveMelee: string
+  rackActive1v1: string
+  rackMeleeLines: int
+  configLines: int
+  distinctTargets: int
+  targetChanges: int
+  bbResetTargetChange: int
+  bbLines: int
+  resultEndWon: string
+
+proc envLine(text, key: string): string =
+  ## Value of an `[env] KEY = VALUE` (or `[env] KEY=VALUE`) boot-report line,
+  ## with the `(source: ...)` annotation stripped. Empty when absent.
+  let prefix = "[env] " & key
+  for line in text.splitLines():
+    let s = line.strip()
+    if not s.startsWith(prefix): continue
+    var rest = s[prefix.len .. ^1].strip()
+    if rest.len == 0 or rest[0] != '=': continue
+    rest = rest[1 .. ^1].strip()
+    let si = rest.find("(source:")
+    if si >= 0: rest = rest[0 ..< si].strip()
+    return rest
+
+proc parseLive(text: string): Live =
+  var lastTarget = -1
+  var seen: seq[int]
+  for raw in text.splitLines():
+    let s = stripAnsi(raw).strip()
+    if s.startsWith("[config] "):
+      inc result.configLines
+      let i = s.find("target=#")
+      if i >= 0:
+        var j = i + "target=#".len
+        var neg = false
+        if j < s.len and s[j] == '-': neg = true; inc j
+        var num = 0
+        var any = false
+        while j < s.len and s[j] in {'0'..'9'}:
+          num = num * 10 + (ord(s[j]) - ord('0')); inc j; any = true
+        if any and not neg:
+          if lastTarget >= 0 and num != lastTarget: inc result.targetChanges
+          lastTarget = num
+          if num notin seen: seen.add num
+    elif s.startsWith("[rack] "):
+      if " mode=melee " in s or s.endsWith(" mode=melee"):
+        inc result.rackMeleeLines
+    elif s.startsWith("[bb-reset] ") and "reason=target_change" in s:
+      inc result.bbResetTargetChange
+    elif s.startsWith("[bb] "):
+      inc result.bbLines
+    elif s.startsWith("[result] battle END:"):
+      let i = s.find("rounds won ")
+      if i >= 0: result.resultEndWon = s[i + "rounds won ".len .. ^1].strip()
+  result.distinctTargets = seen.len
+  result.envRackPattern     = envLine(text, "TR_RACK_PATTERN")
+  result.envRackBitbrain    = envLine(text, "TR_RACK_BITBRAIN")
+  result.envBitbrainMem     = envLine(text, "TR_BITBRAIN_MEM")
+  result.envBitbrainGains   = envLine(text, "TR_BITBRAIN_GAINS")
+  result.envMovement        = envLine(text, "TR_MOVEMENT")
+  result.rackActiveMelee    = envLine(text, "rack active melee")
+  result.rackActive1v1      = envLine(text, "rack active 1v1")
+
+# ── one run ──────────────────────────────────────────────────────────────────
+
+proc runOne(repo, outdir, bin, armName: string, env: seq[string],
+            run, rounds, timeout: int, commit, binsha: string): bool =
+  let armDir = outdir / armName
+  createDir(armDir)
+  let work = outdir / "work" / armName / ("run" & $run)
+  let botDir = work / "ModularBot"
+  let outLog = armDir / ("run" & $run & ".bot.stdout.log")
+  let errLog = armDir / ("run" & $run & ".bot.stderr.log")
+  let runLog = armDir / ("run" & $run & ".runner.log")
+  let jsonPath = armDir / ("run" & $run & ".json")
+  if fileExists(outLog): removeFile(outLog)
+  if fileExists(errLog): removeFile(errLog)
+  writeWrapper(botDir, env, bin, outLog, errLog)
+
+  var advDirs: seq[string]
+  for a in fieldDirs(repo): advDirs.add a
+
+  var ok = true
+  var raw = ""
+  try:
+    raw = runBattleRunner(getServerUrl(), @[botDir] & advDirs, rounds, timeout, true)
+  except CatchableError as e:
+    ok = false
+    raw = "EXCEPTION: " & e.msg
+  writeFile(runLog, raw)
+
+  if not ok:
+    writeFile(jsonPath, $(%*{"arm": armName, "run": run, "rounds": rounds,
+                              "commit": commit, "binary_sha256": binsha,
+                              "ok": false, "error": raw}))
+    stderr.writeLine(fmt"[melee] {armName} run {run}: FAILED")
+    return false
+
+  let r = parseServerOutput(raw)
+  var perRound: seq[tuple[round, score, rank: int, survived: bool]]
+  var botRd: seq[JsonNode]
+  for rr in r.rounds:
+    for br in rr.results:
+      if br.name == "ModularBot":
+        perRound.add (rr.round, br.score, br.rank, br.survived)
+
+  var modJ = %*{"name": "ModularBot", "found": false}
+  for b in r.results:
+    if b.name == "ModularBot":
+      modJ = %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank,
+                "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount,
+                "found": true}
+  var field = newJArray()
+  for b in r.results:
+    if b.name == "ModularBot": continue
+    field.add %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank,
+                 "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount}
+
+  for pr in perRound:
+    botRd.add %*{"round": pr.round, "score": pr.score, "rank": pr.rank,
+                 "survived": pr.survived}
+
+  var live = Live()
+  if fileExists(outLog):
+    live = parseLive(readFile(outLog))
+
+  let row = %*{
+    "arm": armName, "run": run, "rounds": rounds,
+    "commit": commit, "binary_sha256": binsha,
+    "ok": true,
+    "modularbot": modJ,
+    "field": field,
+    "perRound": botRd,
+    "live": %*{
+      "env_TR_RACK_PATTERN": live.envRackPattern,
+      "env_TR_RACK_BITBRAIN": live.envRackBitbrain,
+      "env_TR_BITBRAIN_MEM": live.envBitbrainMem,
+      "env_TR_BITBRAIN_GAINS": live.envBitbrainGains,
+      "env_TR_MOVEMENT": live.envMovement,
+      "rack_active_melee": live.rackActiveMelee,
+      "rack_active_1v1": live.rackActive1v1,
+      "rack_melee_lines": live.rackMeleeLines,
+      "config_lines": live.configLines,
+      "distinct_targets": live.distinctTargets,
+      "target_changes": live.targetChanges,
+      "bb_reset_target_change": live.bbResetTargetChange,
+      "bb_lines": live.bbLines,
+      "result_end_won": live.resultEndWon
+    }
+  }
+  writeFile(jsonPath, $(row))
+  let mb = r.results.filterIt(it.name == "ModularBot")
+  let wins = if mb.len > 0: mb[0].firstPlaces else: 0
+  let rk = if mb.len > 0: mb[0].rank else: 0
+  let sc = if mb.len > 0: mb[0].totalScore else: 0
+  echo fmt"[melee] {armName} run {run}: wins={wins}/{rounds} rank={rk} score={sc} " &
+       fmt"targets={live.distinctTargets} changes={live.targetChanges} " &
+       fmt"rackMelee={live.rackActiveMelee} bbreset={live.bbResetTargetChange}"
+  result = true
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+proc main() =
+  let repo = repoRoot()
+  let args = params()
+  if hasFlag(args, "--list-arms") or hasFlag(args, "--help") or args.len == 0:
+    echo "field:"
+    for f in fieldDirs(repo): echo "  " & f
+    echo "arms:"
+    for a in Arms:
+      echo a.name & "\t" & a.env.join(" ") & "\t" & a.note
+    return
+  let outdir = findParam(args, "--outdir", "/tmp/melee_strong_field")
+  createDir(outdir)
+  let commit = headCommit(repo)
+  let bin = buildFrozen(repo, outdir)
+  let binsha = binarySha(bin)
+  echo "[melee] commit=", commit, " binary_sha256=", binsha
+  if hasFlag(args, "--build-only"): return
+
+  let armName = findParam(args, "--arm")
+  if armName.len == 0:
+    stderr.writeLine("--arm is required (or --list-arms / --build-only)")
+    quit(1)
+  var arm: Arm
+  var found = false
+  for a in Arms:
+    if a.name == armName: arm = a; found = true
+  if not found:
+    stderr.writeLine("unknown arm: " & armName)
+    quit(1)
+
+  let runStart = parseInt(findParam(args, "--run-start", "1"))
+  let runCount = parseInt(findParam(args, "--run-count", "15"))
+  let rounds = parseInt(findParam(args, "--rounds", "7"))
+  let timeout = parseInt(findParam(args, "--timeout", "900000"))
+  let retries = parseInt(findParam(args, "--retries", "2"))
+
+  # `--runs 1,3,5` overrides the contiguous range (used to fill failed runs).
+  var runList: seq[int]
+  let runsSpec = findParam(args, "--runs", "")
+  if runsSpec.len > 0:
+    for t in runsSpec.split(','):
+      let s = t.strip()
+      if s.len > 0: runList.add parseInt(s)
+  else:
+    for run in runStart ..< runStart + runCount: runList.add run
+
+  ensureServer()
+  echo "[melee] server=", getServerUrl(), " arm=", armName,
+       " runs=", runList.len, " rounds=", rounds, " retries=", retries
+
+  var ok = 0
+  for run in runList:
+    var done = false
+    for attempt in 1 .. retries + 1:
+      if runOne(repo, outdir, bin, armName, arm.env, run, rounds, timeout, commit, binsha):
+        done = true
+        break
+      if attempt <= retries:
+        stderr.writeLine(fmt"[melee] {armName} run {run}: retry {attempt}/{retries} in 8s")
+        sleep(8000)
+    if done: inc ok
+  echo fmt"[melee] {armName}: {ok}/{runList.len} runs ok"
+
+when isMainModule:
+  main()
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# run_melee_strong_field.sh — drive the STRONG-FIELD melee A/B
+# (see common_libs/tests/measure_melee_strong_field.nim).
+#
+# Builds ONE frozen ModularBot (from `git archive HEAD`), then runs one process
+# per arm in parallel; each process owns its own Tank Royale server on an
+# ephemeral port. Output: <outdir>/<arm>/run<N>.json (+ .bot.stdout.log).
+#
+#   common_libs/tests/run_melee_strong_field.sh [outdir]
+#
+# Env:
+#   MELEE_RUNS    (default 12)
+#   MELEE_ROUNDS  (default 5)
+#   MELEE_ARMS    (default "pattern bb_ret bb_decay")
+#   MELEE_FIELD   (default "/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd")
+#   WAIT_ARENA    minutes to wait for foreign battles (default 45)
+#
+# ARENA SERIALIZATION: refuses to start (or waits) while another job's battles
+# are running; never pkills a foreign process. Kills only its own process groups
+# on EXIT/INT/TERM.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="$(cd "$HERE/../.." && pwd)"
+OUTDIR="${1:-/tmp/melee_strong_field}"
+RUNS="${MELEE_RUNS:-12}"
+ROUNDS="${MELEE_ROUNDS:-5}"
+ARMS_STR="${MELEE_ARMS:-pattern bb_ret bb_decay}"
+WAIT_ARENA="${WAIT_ARENA:-45}"
+export MELEE_FIELD="${MELEE_FIELD:-/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd}"
+BIN="$HERE/measure_melee_strong_field"
+
+[[ -x "$BIN" ]] || { echo "ERROR: build the runner first: nim c --nimcache:/tmp/nc_j124 --path:common_libs $HERE/measure_melee_strong_field.nim" >&2; exit 1; }
+
+# ── arena serialization (bracketed patterns: pgrep must not match itself) ────
+FOREIGN='[r]un_bridge_battle.sh|[r]obocode_shim.TrBattleCapture|[r]obocode_shim.LegacyBotBridge|[M]odularBot_bin|[m]easure_melee'
+foreign() { pgrep -af "$FOREIGN" 2>/dev/null | grep -v "$OUTDIR" || true; }
+if [[ -n "$(foreign)" ]]; then
+  if (( WAIT_ARENA <= 0 )); then
+    echo "[driver] ERROR: arena busy with another job's battles:" >&2
+    foreign | sed 's/^/    /' >&2
+    exit 3
+  fi
+  echo "[driver] arena busy — waiting up to ${WAIT_ARENA} min:"
+  foreign | sed 's/^/    /'
+  deadline=$(( $(date +%s) + WAIT_ARENA * 60 ))
+  while :; do
+    sleep 60
+    if [[ -z "$(foreign)" ]]; then echo "[driver] arena free."; break; fi
+    if (( $(date +%s) >= deadline )); then
+      echo "[driver] ERROR: arena still busy after ${WAIT_ARENA} min; NOT starting." >&2
+      exit 3
+    fi
+    echo "[driver] still busy at $(date +%H:%M:%S)"
+  done
+fi
+
+mkdir -p "$OUTDIR"
+echo "[driver] building frozen ModularBot…"
+"$BIN" --outdir "$OUTDIR" --build-only
+
+read -r -a ARMS <<< "$ARMS_STR"
+PIDS=()
+cleanup() {
+  local rc=$?
+  trap - EXIT INT TERM
+  for p in "${PIDS[@]:-}"; do
+    [[ -n "$p" ]] || continue
+    kill -TERM -- "-$p" 2>/dev/null || true
+  done
+  sleep 1
+  for p in "${PIDS[@]:-}"; do
+    [[ -n "$p" ]] || continue
+    kill -KILL -- "-$p" 2>/dev/null || true
+  done
+  pkill -f "[m]easure_melee_strong_field.*$OUTDIR" 2>/dev/null || true
+  exit "$rc"
+}
+trap cleanup EXIT INT TERM
+
+for arm in "${ARMS[@]}"; do
+  echo "[driver] launching arm $arm (runs=$RUNS rounds=$ROUNDS) field=$MELEE_FIELD"
+  setsid "$BIN" --outdir "$OUTDIR" --arm "$arm" \
+      --run-start 1 --run-count "$RUNS" --rounds "$ROUNDS" \
+      > "$OUTDIR/$arm.driver.log" 2>&1 &
+  PIDS+=("$!")
+  sleep 3   # stagger so concurrent servers do not race for the same free port
+done
+
+echo "[driver] all arms launched; waiting…"
+FAIL=0
+for p in "${PIDS[@]}"; do
+  wait "$p" || FAIL=$((FAIL + 1))
+done
+PIDS=()
+echo "[driver] done (failed arm processes: $FAIL)"
+exit "$FAIL"