#!/usr/bin/env bash # ───────────────────────────────────────────────────────────────────────────── # tournament_run.sh — THE MULTI-OPPONENT TOURNAMENT HARNESS of the movement # campaign: play OUR bot (one FROZEN binary from `git archive HEAD`) against a # FIXED PANEL of opponents, once per (opponent × arm × run), and leave a session # dir that `tournament_analyze.py` turns into a paired per-opponent ranking. # # tools/ab/tournament_run.sh \ # --arms tools/ab/arms_movement_b1.txt \ # --panel tools/ab/panel_movement.txt \ # --runs 3 --rounds 3 --conc 6 --wait-arena 45 \ # --outdir /tmp/ab/j118_b1 # # Why it exists next to ab_run.sh: ab_run.sh hard-wires exactly ONE adversary # (DrussGT). Here the SUBJECT is our frozen bot and the ADVERSARY iterates over # a frozen panel, so the unit of evidence becomes the NUMBER OF OPPONENTS. # The arm file format is identical to ab_run.sh / gauntlet_run.sh. # # Design contract (all four are load-bearing for a valid measurement): # 1. ONE binary for every arm. The binary is built once from `git archive # HEAD` (a dirty tree cannot leak in) and every arm differs ONLY by the env # dict it is launched with. No per-arm rebuild, ever. # 2. Every run is isolated: own bot dir, own classic data dir, own output # files, its own process group; the runner picks ephemeral ports. # 3. A run that never really started is retried, not recorded. A battle that # did not boot ("Only 1 of 2 bots started") would otherwise show up as a # 0-0 phantom and poison the paired deltas. # 4. ARENA SERIALIZATION. Other campaign jobs may be fighting at the same # time; two concurrent battle fleets on one 16-core box distort each other. # --wait-arena N polls for foreign bridge processes (bracketed pgrep so it # cannot match itself) and WAITS until they are gone, up to N minutes. # With N=0 the session REFUSES to start while the arena is busy. # # Arm file format: name | ENV=value ENV=value | optional label # Panel file format: /tmp/tr_bots/Name | style | why (a bare Name works) # # Output layout (what tournament_analyze.py reads): # /session.json commit, sha, arms, panel # ///run.jsonl tick capture # ///run.jsonl.rounds.json # ///run.jsonl.results.json # ///run.events.jsonl fire/hit/death sidecar # ///run.battle.log capture stdout + stats # ///run.bot.stdout.log our bot's boot env report # # Cleanup on EXIT/INT/TERM: only THIS session's process groups, and only # processes whose command line references THIS outdir. Never a broad pkill. # ───────────────────────────────────────────────────────────────────────────── set -euo pipefail REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" RUN_BRIDGE="$REPO/tools/robocode_shim/run_bridge_battle.sh" ARMS_FILE="" PANEL_FILE="" RUNS=3 ROUNDS=3 CONC=6 OUTDIR="" WAIT_ARENA=0 FORCE_BUSY=0 REFERENCE="" # A bot that fails to connect inside the booter's window is a FALSE NEGATIVE. # Retry until the capture shows real rows AND the subject counter line. MAX_ATTEMPTS="${TOURNAMENT_MAX_ATTEMPTS:-4}" # 800x600 default arena; recorded for the ledger. GAME_TYPE="${TOURNAMENT_GAME_TYPE:-1v1}" usage() { sed -n '2,45p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-0}" } while [[ $# -gt 0 ]]; do case "$1" in --arms) ARMS_FILE="$2"; shift 2;; --panel|--opponents) PANEL_FILE="$2"; shift 2;; --runs) RUNS="$2"; shift 2;; --rounds) ROUNDS="$2"; shift 2;; --conc) CONC="$2"; shift 2;; --outdir) OUTDIR="$2"; shift 2;; --wait-arena) WAIT_ARENA="$2"; shift 2;; --force-when-busy) FORCE_BUSY=1; shift;; --reference) REFERENCE="$2"; shift 2;; -h|--help) usage 0;; *) echo "unknown argument: $1" >&2; usage 1;; esac done [[ -n "$ARMS_FILE" ]] || { echo "ERROR: --arms is required" >&2; usage 1; } [[ -n "$PANEL_FILE" ]] || { echo "ERROR: --panel is required" >&2; usage 1; } [[ -f "$ARMS_FILE" ]] || { echo "ERROR: arm file not found: $ARMS_FILE" >&2; exit 1; } [[ -f "$PANEL_FILE" ]] || { echo "ERROR: panel file not found: $PANEL_FILE" >&2; exit 1; } [[ -n "$OUTDIR" ]] || { echo "ERROR: --outdir is required" >&2; exit 1; } [[ "$OUTDIR" != "/" && "$OUTDIR" != "" ]] || { echo "ERROR: refusing outdir '$OUTDIR'" >&2; exit 1; } mkdir -p "$OUTDIR" OUTDIR="$(cd "$OUTDIR" && pwd)" # never nuke a directory that is not one of ours (non-empty without a session.json) if [[ -n "$(ls -A "$OUTDIR" 2>/dev/null)" && ! -f "$OUTDIR/session.json" ]]; then echo "ERROR: refusing to clear non-empty, non-session dir: $OUTDIR" >&2 echo " (delete it by hand or pick another --outdir)" >&2 exit 1 fi # ── prerequisites ──────────────────────────────────────────────────────────── ROBOCODE_JAR="${ROBOCODE_JAR:-/tmp/robocode/install/libs/robocode.jar}" RUNNER_JAR="${TR_RUNNER_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}" BOT_API_JAR="${TR_BOT_API_JAR:-$HOME/Downloads/sample-bots-java-1.0.2/lib/robocode-tankroyale-bot-api-1.0.2.jar}" missing=0 for f in "$ROBOCODE_JAR" "$RUNNER_JAR" "$BOT_API_JAR"; do [[ -f "$f" ]] || { echo "ERROR: missing prerequisite: $f" >&2; missing=1; } done command -v nim >/dev/null 2>&1 || { echo "ERROR: nim not on PATH" >&2; missing=1; } command -v git >/dev/null 2>&1 || { echo "ERROR: git not on PATH" >&2; missing=1; } [[ -f "$RUN_BRIDGE" ]] || { echo "ERROR: missing $RUN_BRIDGE" >&2; missing=1; } (( missing == 0 )) || { echo "ERROR: prerequisites missing — not starting a broken session" >&2; exit 1; } # ── ARENA SERIALIZATION ────────────────────────────────────────────────────── # Bracketed patterns: pgrep -f must not match this script's own command line. FOREIGN_PATTERN='[r]un_bridge_battle.sh|[r]obocode_shim.TrBattleCapture|[r]obocode_shim.LegacyBotBridge|[M]odularBot_bin' foreign_battles() { pgrep -af "$FOREIGN_PATTERN" 2>/dev/null | grep -v "$OUTDIR" || true } check_arena() { local found found="$(foreign_battles)" [[ -z "$found" ]] } if ! check_arena; then if (( FORCE_BUSY == 1 )); then echo "[tournament] WARNING: arena busy but --force-when-busy was given:" >&2 foreign_battles | sed 's/^/ /' >&2 elif (( WAIT_ARENA > 0 )); then echo "[tournament] arena busy — waiting up to ${WAIT_ARENA} min for foreign battles:" foreign_battles | sed 's/^/ /' deadline=$(( $(date +%s) + WAIT_ARENA * 60 )) while :; do sleep 120 if check_arena; then echo "[tournament] arena free."; break; fi if (( $(date +%s) >= deadline )); then echo "[tournament] ERROR: arena still busy after ${WAIT_ARENA} min; NOT starting." >&2 foreign_battles | sed 's/^/ /' >&2 exit 3 fi echo "[tournament] still busy at $(date +%H:%M:%S); $(foreign_battles | wc -l) foreign process(es)" done else echo "[tournament] ERROR: the arena is busy with another job's battles:" >&2 foreign_battles | sed 's/^/ /' >&2 echo "[tournament] rerun with --wait-arena to wait for it, or --force-when-busy." >&2 exit 3 fi fi # ── parse the arm file ─────────────────────────────────────────────────────── trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; } ARM_NAMES=(); ARM_ENVS=(); ARM_LABELS=() while IFS= read -r line || [[ -n "$line" ]]; do [[ -z "${line//[[:space:]]/}" ]] && continue [[ "$line" =~ ^[[:space:]]*# ]] && continue name="$(trim "${line%%|*}")" rest="${line#*|}" if [[ "$line" == *"|"* ]]; then envspec="$(trim "${rest%%|*}")" label="$(trim "${rest#*|}")" else envspec=""; label="" fi [[ -n "$name" ]] || { echo "ERROR: arm with empty name in $ARMS_FILE" >&2; exit 1; } # every env token must be VAR=value (a typo'd arm must fail loudly, not quietly) for tok in $envspec; do [[ "$tok" =~ ^[A-Za-z_][A-Za-z0-9_]*=.*$ ]] \ || { echo "ERROR: arm '$name' has a malformed env token: '$tok'" >&2; exit 1; } done ARM_NAMES+=("$name"); ARM_ENVS+=("$envspec"); ARM_LABELS+=("$label") done < "$ARMS_FILE" (( ${#ARM_NAMES[@]} > 0 )) || { echo "ERROR: no arms parsed from $ARMS_FILE" >&2; exit 1; } dupes="$(printf '%s\n' "${ARM_NAMES[@]}" | sort | uniq -d)" [[ -z "$dupes" ]] || { echo "ERROR: duplicate arm name(s): $dupes" >&2; exit 1; } if [[ -n "$REFERENCE" ]]; then printf '%s\n' "${ARM_NAMES[@]}" | grep -qx "$REFERENCE" \ || { echo "ERROR: --reference '$REFERENCE' is not one of the arms" >&2; exit 1; } fi # ── parse the panel file ───────────────────────────────────────────────────── OPP_DIRS=(); OPP_NAMES=(); OPP_STYLES=() while IFS= read -r line || [[ -n "$line" ]]; do [[ -z "${line//[[:space:]]/}" ]] && continue [[ "$line" =~ ^[[:space:]]*# ]] && continue dir="$(trim "${line%%|*}")" style=""; note="" if [[ "$line" == *"|"* ]]; then rest="${line#*|}" if [[ "$rest" == *"|"* ]]; then style="$(trim "${rest%%|*}")"; note="$(trim "${rest#*|}")" else style="$(trim "$rest")" fi fi [[ "$dir" == */* ]] || dir="/tmp/tr_bots/$dir" [[ -d "$dir" ]] || { echo "ERROR: opponent bot dir not found: $dir" >&2; exit 1; } OPP_DIRS+=("$dir"); OPP_NAMES+=("$(basename "$dir")"); OPP_STYLES+=("$style") done < "$PANEL_FILE" (( ${#OPP_NAMES[@]} > 0 )) || { echo "ERROR: no opponents parsed from $PANEL_FILE" >&2; exit 1; } dupes="$(printf '%s\n' "${OPP_NAMES[@]}" | sort | uniq -d)" [[ -z "$dupes" ]] || { echo "ERROR: duplicate opponent name(s) in the panel: $dupes" >&2; exit 1; } (( ${#OPP_NAMES[@]} >= 4 )) || { echo "ERROR: a panel below 4 opponents cannot carry a sign test" >&2; exit 1; } # ── fresh session dir ──────────────────────────────────────────────────────── rm -rf "$OUTDIR" mkdir -p "$OUTDIR" WORK="$OUTDIR/.work"; mkdir -p "$WORK" # ── build ONE frozen bot from HEAD (single binary for every arm) ───────────── COMMIT="$(git -C "$REPO" rev-parse HEAD)" FROZEN_DIR="$OUTDIR/frozen/ModularBot" FROZEN_BIN="$FROZEN_DIR/ModularBot_bin" FROZEN_JSON="$FROZEN_DIR/ModularBot.json" mkdir -p "$FROZEN_DIR" BUILDDIR="$WORK/head"; mkdir -p "$BUILDDIR" echo "[tournament] exporting HEAD ($COMMIT) -> $BUILDDIR" git -C "$REPO" archive HEAD | tar -x -C "$BUILDDIR" NIMCACHE="${TOURNAMENT_NIMCACHE:-/tmp/nc_j118}" echo "[tournament] building the frozen ModularBot (nim c -d:release, nimcache=$NIMCACHE)…" ( cd "$BUILDDIR/ModularBot_garage" && \ nim c -d:release --nimcache:"$NIMCACHE" --out:"$FROZEN_BIN" src/ModularBot.nim ) \ > "$OUTDIR/frozen/build.log" 2>&1 \ || { echo "ERROR: frozen build failed — see $OUTDIR/frozen/build.log" >&2; tail -30 "$OUTDIR/frozen/build.log" >&2; exit 1; } [[ -x "$FROZEN_BIN" ]] || { echo "ERROR: frozen build produced no binary" >&2; exit 1; } BINSHA="$(sha256sum "$FROZEN_BIN" | awk '{print $1}')" echo "[tournament] frozen binary sha256=$BINSHA" cat > "$FROZEN_JSON" <<'JSON' { "name": "ModularBot", "version": "0.1.0", "authors": ["Davide Cappellini"], "description": "frozen tournament build of ModularBot (tournament_run.sh)", "homepage": "", "countryCodes": ["IT"], "gameTypes": ["classic", "1v1"], "platform": "Nim", "programmingLang": "Nim" } JSON # ── session.json ───────────────────────────────────────────────────────────── json_esc() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } { printf '{\n' printf ' "commit": "%s",\n' "$COMMIT" printf ' "binary_sha256": "%s",\n' "$BINSHA" printf ' "binary": "frozen/ModularBot/ModularBot_bin",\n' printf ' "rounds": %d,\n' "$ROUNDS" printf ' "runs": %d,\n' "$RUNS" printf ' "conc": %d,\n' "$CONC" printf ' "game_type": "%s",\n' "$GAME_TYPE" printf ' "reference": "%s",\n' "$(json_esc "$REFERENCE")" printf ' "timestamp": "%s",\n' "$(date -Is)" printf ' "outdir": "%s",\n' "$(json_esc "$OUTDIR")" printf ' "arms_file": "%s",\n' "$(json_esc "$ARMS_FILE")" printf ' "panel_file": "%s",\n' "$(json_esc "$PANEL_FILE")" printf ' "arms": [\n' for i in "${!ARM_NAMES[@]}"; do printf ' {"name": "%s", "env": "%s", "label": "%s"}' \ "$(json_esc "${ARM_NAMES[$i]}")" "$(json_esc "${ARM_ENVS[$i]}")" "$(json_esc "${ARM_LABELS[$i]}")" (( i + 1 < ${#ARM_NAMES[@]} )) && printf ',' || true printf '\n' done printf ' ],\n' printf ' "opponents": [\n' for i in "${!OPP_NAMES[@]}"; do printf ' {"name": "%s", "dir": "%s", "style": "%s"}' \ "$(json_esc "${OPP_NAMES[$i]}")" "$(json_esc "${OPP_DIRS[$i]}")" "$(json_esc "${OPP_STYLES[$i]}")" (( i + 1 < ${#OPP_NAMES[@]} )) && printf ',' || true printf '\n' done printf ' ]\n}\n' } > "$OUTDIR/session.json" # ── one job ────────────────────────────────────────────────────────────────── export REPO RUN_BRIDGE ROUNDS OUTDIR WORK FROZEN_BIN FROZEN_JSON MAX_ATTEMPTS tournament_run_one() { local opponent_dir="$1" opponent="$2" arm="$3" run="$4" envspec="$5" local dir="$OUTDIR/$opponent/$arm" local work="$WORK/$opponent/$arm/run$run" local botdir="$work/bots/ModularBot" mkdir -p "$dir" "$botdir" "$work/data" [[ -n "$envspec" ]] && printf '%s\n' "$envspec" > "$dir/run$run.arm.env" cp "$FROZEN_JSON" "$botdir/ModularBot.json" cat > "$botdir/ModularBot.sh" < "$dir/run$run.bot.stdout.log" 2> "$dir/run$run.bot.stderr.log" SH chmod +x "$botdir/ModularBot.sh" ln -sf "$FROZEN_BIN" "$botdir/ModularBot_bin" local attempt=0 while :; do attempt=$((attempt + 1)) rm -f "$dir/run$run.jsonl" "$dir/run$run.jsonl.rounds.json" \ "$dir/run$run.jsonl.results.json" "$dir/run$run.events.jsonl" \ "$dir/run$run.status" "$dir/run$run.bot.stdout.log" "$dir/run$run.bot.stderr.log" local rc=0 # shellcheck disable=SC2086 # envspec is intentionally word-split env $envspec \ SHIM_BOTDIR="$botdir" \ SHIM_BOT_NAME="ModularBot" \ SHIM_DATA="$work/data" \ TR_EVENTS_OUT="$dir/run$run.events.jsonl" \ timeout 900 "$RUN_BRIDGE" "$opponent_dir" "$ROUNDS" "$dir/run$run.jsonl" \ > "$dir/run$run.battle.log" 2>&1 || rc=$? echo "$rc" > "$dir/run$run.status" # liveness: a real battle has captured rows AND the subject counter line if grep -q "subject event counts" "$dir/run$run.battle.log" 2>/dev/null \ && grep -Eq "rows=[1-9]" "$dir/run$run.battle.log" 2>/dev/null; then break fi if (( attempt >= MAX_ATTEMPTS )); then echo "[tournament] WARNING: ${opponent}/${arm} run${run} never started after ${attempt} attempts" >&2 break fi sleep $((attempt * 2)) done echo "$attempt" > "$dir/run$run.attempts" return 0 } export -f tournament_run_one # ── cleanup: only this session's process groups, only this outdir ──────────── JOB_PIDS=() cleanup() { local rc=$? trap - EXIT INT TERM for p in "${JOB_PIDS[@]:-}"; do [[ -n "$p" ]] || continue kill -TERM -- "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null || true done sleep 0.5 for p in "${JOB_PIDS[@]:-}"; do [[ -n "$p" ]] || continue kill -KILL -- "-$p" 2>/dev/null || true done pkill -f "[r]un_bridge_battle.sh.*$OUTDIR" 2>/dev/null || true pkill -f "[r]obocode_shim.TrBattleCapture.*$OUTDIR" 2>/dev/null || true exit "$rc" } trap cleanup EXIT INT TERM # ── launch, K at a time ────────────────────────────────────────────────────── NJOBS=$(( ${#OPP_NAMES[@]} * ${#ARM_NAMES[@]} * RUNS )) SECONDS=0 echo "[tournament] session: ${#OPP_NAMES[@]} opponents × ${#ARM_NAMES[@]} arms × $RUNS runs = $NJOBS battles, rounds=$ROUNDS, conc=$CONC" echo "[tournament] outdir: $OUTDIR" RUNNING=0; DONE=0 for oi in "${!OPP_NAMES[@]}"; do for ai in "${!ARM_NAMES[@]}"; do for (( r=1; r<=RUNS; r++ )); do setsid bash -c 'tournament_run_one "$1" "$2" "$3" "$4" "$5"' _ \ "${OPP_DIRS[$oi]}" "${OPP_NAMES[$oi]}" "${ARM_NAMES[$ai]}" "$r" "${ARM_ENVS[$ai]}" & JOB_PIDS+=("$!") RUNNING=$((RUNNING + 1)) if (( RUNNING >= CONC )); then wait -n || true RUNNING=$((RUNNING - 1)); DONE=$((DONE + 1)) printf '[tournament] %3d/%3d done (%ds)\n' "$DONE" "$NJOBS" "$SECONDS" fi done done done wait || true DONE=$NJOBS printf '[tournament] %3d/%3d done (%ds)\n' "$DONE" "$NJOBS" "$SECONDS" JOB_PIDS=() # ── aggregate status ───────────────────────────────────────────────────────── FAILED=0; STARTFAIL=0 for oi in "${!OPP_NAMES[@]}"; do for ai in "${!ARM_NAMES[@]}"; do for (( r=1; r<=RUNS; r++ )); do st="$(cat "$OUTDIR/${OPP_NAMES[$oi]}/${ARM_NAMES[$ai]}/run$r.status" 2>/dev/null || echo 999)" if [[ "$st" != "0" ]]; then echo "[tournament] FAIL ${OPP_NAMES[$oi]}/${ARM_NAMES[$ai]} run$r (rc=$st)" >&2 FAILED=$((FAILED + 1)) fi if ! grep -q "subject event counts" \ "$OUTDIR/${OPP_NAMES[$oi]}/${ARM_NAMES[$ai]}/run$r.battle.log" 2>/dev/null; then STARTFAIL=$((STARTFAIL + 1)) fi done done done echo "[tournament] done: $(( NJOBS - FAILED )) ok, $FAILED failed, $STARTFAIL never started ($(( SECONDS ))s)" echo "[tournament] wall clock: ${SECONDS}s for $NJOBS battles" echo "[tournament] analyze: python3 $REPO/tools/ab/tournament_analyze.py $OUTDIR${REFERENCE:+ --reference $REFERENCE}" (( FAILED == 0 )) || exit 1