From 6c21dc4c58caaaaf68dc670a28a7c0d6948e3f3e Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Sat, 26 Sep 2026 04:08:44 +0200 Subject: [PATCH] j124 fair melee: add strong-field melee harness + driver (legacy champion field) --- .../tests/measure_melee_strong_field.nim | 414 ++++++++++++++++++ common_libs/tests/run_melee_strong_field.sh | 97 ++++ 2 files changed, 511 insertions(+) create mode 100644 common_libs/tests/measure_melee_strong_field.nim create mode 100755 common_libs/tests/run_melee_strong_field.sh diff --git a/common_libs/tests/measure_melee_strong_field.nim b/common_libs/tests/measure_melee_strong_field.nim new file mode 100644 index 0000000..f10df16 --- /dev/null +++ b/common_libs/tests/measure_melee_strong_field.nim @@ -0,0 +1,414 @@ +## MELEE A/B — STRONG FIELD: is BitBrain actually better than Pattern in melee? +## +## This is the FAIR-FIELD rerun of the repo's first melee measurement +## (`measure_melee_bitbrain_ab.nim` / docs/melee_bitbrain_ab.md, j116). That test +## used a weak in-repo field (WaveSurfer + PatternMover + RandomMover) against +## which ModularBot won 97-100% of rounds REGARDLESS of gun — a ceiling effect +## that made the gun question untestable. Here the field is three +## battle-validated legacy champions (default Diamond + Dookious + GresSuffurd) +## that can actually punish a bad gun. +## +## Runs a 4-bot Free-For-All (ModularBot + 3 legacy champions) for each arm, +## where the ONLY thing that differs between arms is ModularBot's gun-rack +## environment (`TR_RACK_*` / `TR_BITBRAIN_*`). Every other bot is held fixed. +## +## Field override: `MELEE_FIELD=/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,...` +## (absolute dirs; a bare name is resolved under the in-repo adversaries dir). +## +## Design (see docs/spinner_and_melee_claims.md): +## * arms differ ONLY in the ModularBot wrapper script's exported env; +## * each run gets its own ModularBot bot-dir + stdout/stderr log, so liveness +## (`[env] rack active melee = ...`) and target-switching evidence +## (`[config] ... target=#N`, `[bb-reset] reason=target_change`) are captured; +## * the runner is driven in one process per arm from a shell driver; each +## process gets its own Tank Royale server (ephemeral port). +## +## Usage: +## nim c -r --nimcache:/tmp/nc_j124 --path:common_libs \ +## common_libs/tests/measure_melee_strong_field.nim --list-arms +## +## MELEE_FIELD=/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd \ +## common_libs/tests/measure_melee_strong_field.nim \ +## --outdir /tmp/melee_strong --build-only +## +## Arm env (verbatim, from the arms table below) is written into the per-run +## wrapper script; the runner process' own environment is NOT mutated for the +## bot, so parallel arm processes cannot leak into each other. + +import std/[os, osproc, strutils, strformat, json, sequtils] + +import test_framework/server_manager +import test_framework/runner_process +import test_framework/battle_result + +# ── the field (fixed across arms) ──────────────────────────────────────────── +const + repoSuffix = "common_libs" / "test_framework" / "adversaries" + DefaultField = @["/tmp/tr_bots/Diamond", "/tmp/tr_bots/Dookious", + "/tmp/tr_bots/GresSuffurd"] + ## Three battle-validated legacy champions: wave-surfing dodgers with real + ## guns (tools/robocode_shim/robots.json). They punish a bad gun, unlike the + ## j116 in-repo field where ModularBot won regardless of the rack. + +proc fieldDirs(repo: string): seq[string] = + ## `MELEE_FIELD` (comma-separated) overrides the default field. A bare name is + ## resolved under the in-repo adversaries dir; an absolute path is used as-is. + let spec = getEnv("MELEE_FIELD") + let raw = if spec.len > 0: spec.split(',') else: DefaultField + for item in raw: + let t = item.strip() + if t.len == 0: continue + result.add(if t.startsWith("/"): t else: repo / repoSuffix / t) + +# ── the arms (ONLY ModularBot's gun rack differs) ──────────────────────────── +type + Arm = object + name*: string + env*: seq[string] ## "KEY=VALUE", exported in the per-run wrapper script + note*: string + +const Arms*: array[3, Arm] = [ + Arm(name: "pattern", env: @[], + note: "shipped default: Pattern-only rack, no env"), + Arm(name: "bb_ret", + env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee", + "TR_BITBRAIN_MEM=retained", "TR_BITBRAIN_LOG=1"], + note: "BitBrain-only melee rack, retained memory"), + Arm(name: "bb_decay", + env: @["TR_RACK_PATTERN=off", "TR_RACK_BITBRAIN=melee", + "TR_BITBRAIN_MEM=decay", "TR_BITBRAIN_GAINS=1.0,1.25,1.5,2.0", + "TR_BITBRAIN_LOG=1"], + note: "BitBrain-only melee rack, decay memory + learned gains"), +] + +const + ModularBotJson = """{ + "name": "ModularBot", + "version": "0.1.0", + "authors": ["Davide Cappellini"], + "description": "frozen melee A/B build of ModularBot", + "homepage": "", + "countryCodes": ["IT"], + "gameTypes": ["classic", "1v1"], + "platform": "Nim", + "programmingLang": "Nim" +} +""" + +# ANSI SGR sequences (the bot's config line color-highlights changed fields). +const AnsiRe = "\x1b\\[[0-9;]*m" + +proc stripAnsi(s: string): string = + result = newStringOfCap(s.len) + var i = 0 + while i < s.len: + if s[i] == '\x1b' and i + 1 < s.len and s[i+1] == '[': + i += 2 + while i < s.len and s[i] notin {'m'}: inc i + if i < s.len: inc i # skip the terminating 'm' + else: + result.add s[i] + inc i + +proc repoRoot(): string = + currentSourcePath().parentDir.parentDir.parentDir + +proc quote(s: string): string = quoteShell(s) + +proc params(): seq[string] = + for i in 1 .. paramCount(): result.add paramStr(i) + +proc findParam(args: seq[string], name: string, default = ""): string = + for i in 0 ..< args.len: + if args[i] == name and i + 1 < args.len: return args[i+1] + default + +proc hasFlag(args: seq[string], name: string): bool = + for a in args: + if a == name: return true + false + +# ── frozen ModularBot build (from `git archive HEAD`, like tools/ab/ab_run.sh) ─ + +proc binarySha(bin: string): string = + let (outp, rc) = execCmdEx("sha256sum " & quote(bin)) + if rc == 0: outp.strip().split(' ')[0] else: "unknown" + +proc headCommit(repo: string): string = + let (outp, rc) = execCmdEx("git -C " & quote(repo) & " rev-parse HEAD") + if rc == 0: outp.strip() else: "unknown" + +proc buildFrozen(repo, outdir: string): string = + ## Compile ONE frozen ModularBot from the committed HEAD snapshot and return + ## its path. A dirty tree cannot leak into the measurement. + let bin = outdir / "frozen" / "ModularBot_bin" + if fileExists(bin): return bin + createDir(outdir / "frozen") + let snapshot = outdir / "frozen" / "head" + createDir(snapshot) + let (arcOut, arcRc) = execCmdEx( + "git -C " & quote(repo) & " archive HEAD | tar -x -C " & quote(snapshot)) + if arcRc != 0: + raise newException(IOError, "git archive HEAD failed:\n" & arcOut) + let garage = snapshot / "ModularBot_garage" + if not dirExists(garage): + raise newException(IOError, "no ModularBot_garage in HEAD snapshot") + let nimcache = "/tmp/nc_j124" + let cmd = "nim c -d:release --nimcache:" & quote(nimcache) & + " --out:" & quote(bin) & " src/ModularBot.nim" + let (outp, rc) = execCmdEx(cmd, workingDir = garage) + if rc != 0 or not fileExists(bin): + raise newException(OSError, "frozen ModularBot build failed:\n" & outp) + result = bin + +# ── per-run wrapper bot dir ────────────────────────────────────────────────── + +proc writeWrapper(botDir: string, env: seq[string], bin, stdoutLog, stderrLog: string) = + createDir(botDir) + writeFile(botDir / "ModularBot.json", ModularBotJson) + var sh = "#!/bin/sh\ncd \"$(dirname \"$0\")\"\n" + for kv in env: + let p = kv.split('=', 1) + sh.add("export " & p[0] & "=\"" & p[1] & "\"\n") + sh.add("exec ./ModularBot_bin > " & quote(stdoutLog) & + " 2> " & quote(stderrLog) & "\n") + writeFile(botDir / "ModularBot.sh", sh) + discard execCmdEx("chmod +x " & quote(botDir / "ModularBot.sh")) + if fileExists(botDir / "ModularBot_bin"): removeFile(botDir / "ModularBot_bin") + createSymlink(bin, botDir / "ModularBot_bin") + +# ── liveness / target-switching extraction from the bot's stdout ───────────── + +type Live = object + envRackPattern: string + envRackBitbrain: string + envBitbrainMem: string + envBitbrainGains: string + envMovement: string + rackActiveMelee: string + rackActive1v1: string + rackMeleeLines: int + configLines: int + distinctTargets: int + targetChanges: int + bbResetTargetChange: int + bbLines: int + resultEndWon: string + +proc envLine(text, key: string): string = + ## Value of an `[env] KEY = VALUE` (or `[env] KEY=VALUE`) boot-report line, + ## with the `(source: ...)` annotation stripped. Empty when absent. + let prefix = "[env] " & key + for line in text.splitLines(): + let s = line.strip() + if not s.startsWith(prefix): continue + var rest = s[prefix.len .. ^1].strip() + if rest.len == 0 or rest[0] != '=': continue + rest = rest[1 .. ^1].strip() + let si = rest.find("(source:") + if si >= 0: rest = rest[0 ..< si].strip() + return rest + +proc parseLive(text: string): Live = + var lastTarget = -1 + var seen: seq[int] + for raw in text.splitLines(): + let s = stripAnsi(raw).strip() + if s.startsWith("[config] "): + inc result.configLines + let i = s.find("target=#") + if i >= 0: + var j = i + "target=#".len + var neg = false + if j < s.len and s[j] == '-': neg = true; inc j + var num = 0 + var any = false + while j < s.len and s[j] in {'0'..'9'}: + num = num * 10 + (ord(s[j]) - ord('0')); inc j; any = true + if any and not neg: + if lastTarget >= 0 and num != lastTarget: inc result.targetChanges + lastTarget = num + if num notin seen: seen.add num + elif s.startsWith("[rack] "): + if " mode=melee " in s or s.endsWith(" mode=melee"): + inc result.rackMeleeLines + elif s.startsWith("[bb-reset] ") and "reason=target_change" in s: + inc result.bbResetTargetChange + elif s.startsWith("[bb] "): + inc result.bbLines + elif s.startsWith("[result] battle END:"): + let i = s.find("rounds won ") + if i >= 0: result.resultEndWon = s[i + "rounds won ".len .. ^1].strip() + result.distinctTargets = seen.len + result.envRackPattern = envLine(text, "TR_RACK_PATTERN") + result.envRackBitbrain = envLine(text, "TR_RACK_BITBRAIN") + result.envBitbrainMem = envLine(text, "TR_BITBRAIN_MEM") + result.envBitbrainGains = envLine(text, "TR_BITBRAIN_GAINS") + result.envMovement = envLine(text, "TR_MOVEMENT") + result.rackActiveMelee = envLine(text, "rack active melee") + result.rackActive1v1 = envLine(text, "rack active 1v1") + +# ── one run ────────────────────────────────────────────────────────────────── + +proc runOne(repo, outdir, bin, armName: string, env: seq[string], + run, rounds, timeout: int, commit, binsha: string): bool = + let armDir = outdir / armName + createDir(armDir) + let work = outdir / "work" / armName / ("run" & $run) + let botDir = work / "ModularBot" + let outLog = armDir / ("run" & $run & ".bot.stdout.log") + let errLog = armDir / ("run" & $run & ".bot.stderr.log") + let runLog = armDir / ("run" & $run & ".runner.log") + let jsonPath = armDir / ("run" & $run & ".json") + if fileExists(outLog): removeFile(outLog) + if fileExists(errLog): removeFile(errLog) + writeWrapper(botDir, env, bin, outLog, errLog) + + var advDirs: seq[string] + for a in fieldDirs(repo): advDirs.add a + + var ok = true + var raw = "" + try: + raw = runBattleRunner(getServerUrl(), @[botDir] & advDirs, rounds, timeout, true) + except CatchableError as e: + ok = false + raw = "EXCEPTION: " & e.msg + writeFile(runLog, raw) + + if not ok: + writeFile(jsonPath, $(%*{"arm": armName, "run": run, "rounds": rounds, + "commit": commit, "binary_sha256": binsha, + "ok": false, "error": raw})) + stderr.writeLine(fmt"[melee] {armName} run {run}: FAILED") + return false + + let r = parseServerOutput(raw) + var perRound: seq[tuple[round, score, rank: int, survived: bool]] + var botRd: seq[JsonNode] + for rr in r.rounds: + for br in rr.results: + if br.name == "ModularBot": + perRound.add (rr.round, br.score, br.rank, br.survived) + + var modJ = %*{"name": "ModularBot", "found": false} + for b in r.results: + if b.name == "ModularBot": + modJ = %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank, + "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount, + "found": true} + var field = newJArray() + for b in r.results: + if b.name == "ModularBot": continue + field.add %*{"name": b.name, "totalScore": b.totalScore, "rank": b.rank, + "firstPlaces": b.firstPlaces, "survivalCount": b.survivalCount} + + for pr in perRound: + botRd.add %*{"round": pr.round, "score": pr.score, "rank": pr.rank, + "survived": pr.survived} + + var live = Live() + if fileExists(outLog): + live = parseLive(readFile(outLog)) + + let row = %*{ + "arm": armName, "run": run, "rounds": rounds, + "commit": commit, "binary_sha256": binsha, + "ok": true, + "modularbot": modJ, + "field": field, + "perRound": botRd, + "live": %*{ + "env_TR_RACK_PATTERN": live.envRackPattern, + "env_TR_RACK_BITBRAIN": live.envRackBitbrain, + "env_TR_BITBRAIN_MEM": live.envBitbrainMem, + "env_TR_BITBRAIN_GAINS": live.envBitbrainGains, + "env_TR_MOVEMENT": live.envMovement, + "rack_active_melee": live.rackActiveMelee, + "rack_active_1v1": live.rackActive1v1, + "rack_melee_lines": live.rackMeleeLines, + "config_lines": live.configLines, + "distinct_targets": live.distinctTargets, + "target_changes": live.targetChanges, + "bb_reset_target_change": live.bbResetTargetChange, + "bb_lines": live.bbLines, + "result_end_won": live.resultEndWon + } + } + writeFile(jsonPath, $(row)) + let mb = r.results.filterIt(it.name == "ModularBot") + let wins = if mb.len > 0: mb[0].firstPlaces else: 0 + let rk = if mb.len > 0: mb[0].rank else: 0 + let sc = if mb.len > 0: mb[0].totalScore else: 0 + echo fmt"[melee] {armName} run {run}: wins={wins}/{rounds} rank={rk} score={sc} " & + fmt"targets={live.distinctTargets} changes={live.targetChanges} " & + fmt"rackMelee={live.rackActiveMelee} bbreset={live.bbResetTargetChange}" + result = true + +# ── main ───────────────────────────────────────────────────────────────────── + +proc main() = + let repo = repoRoot() + let args = params() + if hasFlag(args, "--list-arms") or hasFlag(args, "--help") or args.len == 0: + echo "field:" + for f in fieldDirs(repo): echo " " & f + echo "arms:" + for a in Arms: + echo a.name & "\t" & a.env.join(" ") & "\t" & a.note + return + let outdir = findParam(args, "--outdir", "/tmp/melee_strong_field") + createDir(outdir) + let commit = headCommit(repo) + let bin = buildFrozen(repo, outdir) + let binsha = binarySha(bin) + echo "[melee] commit=", commit, " binary_sha256=", binsha + if hasFlag(args, "--build-only"): return + + let armName = findParam(args, "--arm") + if armName.len == 0: + stderr.writeLine("--arm is required (or --list-arms / --build-only)") + quit(1) + var arm: Arm + var found = false + for a in Arms: + if a.name == armName: arm = a; found = true + if not found: + stderr.writeLine("unknown arm: " & armName) + quit(1) + + let runStart = parseInt(findParam(args, "--run-start", "1")) + let runCount = parseInt(findParam(args, "--run-count", "15")) + let rounds = parseInt(findParam(args, "--rounds", "7")) + let timeout = parseInt(findParam(args, "--timeout", "900000")) + let retries = parseInt(findParam(args, "--retries", "2")) + + # `--runs 1,3,5` overrides the contiguous range (used to fill failed runs). + var runList: seq[int] + let runsSpec = findParam(args, "--runs", "") + if runsSpec.len > 0: + for t in runsSpec.split(','): + let s = t.strip() + if s.len > 0: runList.add parseInt(s) + else: + for run in runStart ..< runStart + runCount: runList.add run + + ensureServer() + echo "[melee] server=", getServerUrl(), " arm=", armName, + " runs=", runList.len, " rounds=", rounds, " retries=", retries + + var ok = 0 + for run in runList: + var done = false + for attempt in 1 .. retries + 1: + if runOne(repo, outdir, bin, armName, arm.env, run, rounds, timeout, commit, binsha): + done = true + break + if attempt <= retries: + stderr.writeLine(fmt"[melee] {armName} run {run}: retry {attempt}/{retries} in 8s") + sleep(8000) + if done: inc ok + echo fmt"[melee] {armName}: {ok}/{runList.len} runs ok" + +when isMainModule: + main() diff --git a/common_libs/tests/run_melee_strong_field.sh b/common_libs/tests/run_melee_strong_field.sh new file mode 100755 index 0000000..1cb3efe --- /dev/null +++ b/common_libs/tests/run_melee_strong_field.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# run_melee_strong_field.sh — drive the STRONG-FIELD melee A/B +# (see common_libs/tests/measure_melee_strong_field.nim). +# +# Builds ONE frozen ModularBot (from `git archive HEAD`), then runs one process +# per arm in parallel; each process owns its own Tank Royale server on an +# ephemeral port. Output: //run.json (+ .bot.stdout.log). +# +# common_libs/tests/run_melee_strong_field.sh [outdir] +# +# Env: +# MELEE_RUNS (default 12) +# MELEE_ROUNDS (default 5) +# MELEE_ARMS (default "pattern bb_ret bb_decay") +# MELEE_FIELD (default "/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd") +# WAIT_ARENA minutes to wait for foreign battles (default 45) +# +# ARENA SERIALIZATION: refuses to start (or waits) while another job's battles +# are running; never pkills a foreign process. Kills only its own process groups +# on EXIT/INT/TERM. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +OUTDIR="${1:-/tmp/melee_strong_field}" +RUNS="${MELEE_RUNS:-12}" +ROUNDS="${MELEE_ROUNDS:-5}" +ARMS_STR="${MELEE_ARMS:-pattern bb_ret bb_decay}" +WAIT_ARENA="${WAIT_ARENA:-45}" +export MELEE_FIELD="${MELEE_FIELD:-/tmp/tr_bots/Diamond,/tmp/tr_bots/Dookious,/tmp/tr_bots/GresSuffurd}" +BIN="$HERE/measure_melee_strong_field" + +[[ -x "$BIN" ]] || { echo "ERROR: build the runner first: nim c --nimcache:/tmp/nc_j124 --path:common_libs $HERE/measure_melee_strong_field.nim" >&2; exit 1; } + +# ── arena serialization (bracketed patterns: pgrep must not match itself) ──── +FOREIGN='[r]un_bridge_battle.sh|[r]obocode_shim.TrBattleCapture|[r]obocode_shim.LegacyBotBridge|[M]odularBot_bin|[m]easure_melee' +foreign() { pgrep -af "$FOREIGN" 2>/dev/null | grep -v "$OUTDIR" || true; } +if [[ -n "$(foreign)" ]]; then + if (( WAIT_ARENA <= 0 )); then + echo "[driver] ERROR: arena busy with another job's battles:" >&2 + foreign | sed 's/^/ /' >&2 + exit 3 + fi + echo "[driver] arena busy — waiting up to ${WAIT_ARENA} min:" + foreign | sed 's/^/ /' + deadline=$(( $(date +%s) + WAIT_ARENA * 60 )) + while :; do + sleep 60 + if [[ -z "$(foreign)" ]]; then echo "[driver] arena free."; break; fi + if (( $(date +%s) >= deadline )); then + echo "[driver] ERROR: arena still busy after ${WAIT_ARENA} min; NOT starting." >&2 + exit 3 + fi + echo "[driver] still busy at $(date +%H:%M:%S)" + done +fi + +mkdir -p "$OUTDIR" +echo "[driver] building frozen ModularBot…" +"$BIN" --outdir "$OUTDIR" --build-only + +read -r -a ARMS <<< "$ARMS_STR" +PIDS=() +cleanup() { + local rc=$? + trap - EXIT INT TERM + for p in "${PIDS[@]:-}"; do + [[ -n "$p" ]] || continue + kill -TERM -- "-$p" 2>/dev/null || true + done + sleep 1 + for p in "${PIDS[@]:-}"; do + [[ -n "$p" ]] || continue + kill -KILL -- "-$p" 2>/dev/null || true + done + pkill -f "[m]easure_melee_strong_field.*$OUTDIR" 2>/dev/null || true + exit "$rc" +} +trap cleanup EXIT INT TERM + +for arm in "${ARMS[@]}"; do + echo "[driver] launching arm $arm (runs=$RUNS rounds=$ROUNDS) field=$MELEE_FIELD" + setsid "$BIN" --outdir "$OUTDIR" --arm "$arm" \ + --run-start 1 --run-count "$RUNS" --rounds "$ROUNDS" \ + > "$OUTDIR/$arm.driver.log" 2>&1 & + PIDS+=("$!") + sleep 3 # stagger so concurrent servers do not race for the same free port +done + +echo "[driver] all arms launched; waiting…" +FAIL=0 +for p in "${PIDS[@]}"; do + wait "$p" || FAIL=$((FAIL + 1)) +done +PIDS=() +echo "[driver] done (failed arm processes: $FAIL)" +exit "$FAIL"