diff --git a/SAC_LSTM_Bot/SAC_LSTM_Bot.json b/SAC_LSTM_Bot/SAC_LSTM_Bot.json new file mode 100644 index 0000000..ffef785 --- /dev/null +++ b/SAC_LSTM_Bot/SAC_LSTM_Bot.json @@ -0,0 +1,11 @@ +{ + "name": "SAC_LSTM_Bot", + "version": "0.1.0", + "authors": ["Davide Cappellini"], + "description": "SAC+LSTM-trained Tank Royale bot (#37) — training/eval launch config", + "homepage": "", + "countryCodes": ["IT"], + "gameTypes": ["classic", "melee", "1v1"], + "platform": "Nim", + "programmingLang": "Nim" +} diff --git a/SAC_LSTM_Bot/SAC_LSTM_Bot.sh b/SAC_LSTM_Bot/SAC_LSTM_Bot.sh new file mode 100755 index 0000000..e9b2b34 --- /dev/null +++ b/SAC_LSTM_Bot/SAC_LSTM_Bot.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Launch config for tools/training_runner/RunTraining.java (#49): the runner +# executes .sh inside the bot dir (sample-bots convention). +exec "$(dirname "$0")/SAC_LSTM_Bot" diff --git a/SAC_LSTM_Bot/sac_train.sh b/SAC_LSTM_Bot/sac_train.sh new file mode 100755 index 0000000..4bcd2e1 --- /dev/null +++ b/SAC_LSTM_Bot/sac_train.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# sac_train.sh — #49 training orchestration for SAC_LSTM_Bot. +# +# Drives chunked self-play via tools/training_runner/RunTraining.java (which +# owns server lifecycle, opponent connection and dead-bot liveness detection +# through weights/round_counter.txt), samples opponents by weight per chunk, +# runs deterministic evaluation (SACLSTM_EVAL_MODE=1) every N chunks, and keeps +# the best checkpoint (weights/sac_best.zip) by eval win rate. +# +# Config (env vars): +# SAC_OPPONENTS "Name:weight,Name:weight,..." (default below) +# SAC_TOTAL_ROUNDS total training-round budget (default 100) +# SAC_CHUNK_SIZE rounds per RunTraining battle (default 10) +# SAC_EVAL_INTERVAL eval every N chunks (default 2) +# SAC_EVAL_ROUNDS rounds per evaluation battle (default 10) +# SAC_EVAL_OPPONENT fixed eval opponent (default first opponent) +# SAC_MAX_CRASHES consecutive crashes before abort (default 5) +# SAC_LOG_FILE / SAC_EVAL_LOG_FILE (JSON-lines logs) +# SACLSTM_* passed through to the bot (UTD_RATIO, BATCH_SIZE, ...) +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RUNNER_DIR="$REPO_ROOT/tools/training_runner" +JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}" + +export PPO_BOT_DIR="$SCRIPT_DIR" # runner launches THIS bot dir +export BOT_NAME="${BOT_NAME:-SAC_LSTM_Bot}" # RunTraining result matching +export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}" +# Liveness contract (#49): RunTraining.java watches $BOT_DIR/weights/round_counter.txt +# and integration.bumpRoundCounter() writes it next to the weights — so the bot's +# weights path is pinned here, NOT env-overridable. +export SACLSTM_WEIGHTS_PATH="$SCRIPT_DIR/weights/sac_latest.zip" +WEIGHTS_DIR="$(dirname "$SACLSTM_WEIGHTS_PATH")" + +OPPONENTS="${SAC_OPPONENTS:-Corners:3,Crazy:2,RamFire:1,Target:1}" +TOTAL_ROUNDS="${SAC_TOTAL_ROUNDS:-100}" +CHUNK_SIZE="${SAC_CHUNK_SIZE:-10}" +EVAL_INTERVAL="${SAC_EVAL_INTERVAL:-2}" +EVAL_ROUNDS="${SAC_EVAL_ROUNDS:-10}" +EVAL_OPPONENT="${SAC_EVAL_OPPONENT:-${OPPONENTS%%:*}}" +MAX_CRASHES="${SAC_MAX_CRASHES:-5}" +LOG_FILE="${SAC_LOG_FILE:-$SCRIPT_DIR/training_log.jsonl}" +EVAL_LOG_FILE="${SAC_EVAL_LOG_FILE:-$SCRIPT_DIR/eval_log.jsonl}" +CLASSES_DIR="/tmp/opencode/sac_train_classes" + +echo "=== SAC_LSTM_Bot training harness ===" +echo "Opponents: $OPPONENTS | budget: $TOTAL_ROUNDS rounds in chunks of $CHUNK_SIZE" +echo "Eval: every $EVAL_INTERVAL chunks, $EVAL_ROUNDS rounds vs $EVAL_OPPONENT" +echo "Weights: $SACLSTM_WEIGHTS_PATH" + +# ── compile bot + java runner ───────────────────────────────────────────────── +(cd "$SCRIPT_DIR" && nimble build -d:release) || { echo ">>> bot build failed"; exit 1; } +mkdir -p "$WEIGHTS_DIR" "$CLASSES_DIR" +javac -cp "$JAR" -d "$CLASSES_DIR" "$RUNNER_DIR/RunTraining.java" || { echo ">>> javac failed"; exit 1; } + +# ── weighted opponent pick over "Name:w,Name:w" ─────────────────────────────── +pick_opponent() { + local total=0 p name w r + local pairs + IFS=',' read -ra pairs <<< "$OPPONENTS" + for p in "${pairs[@]}"; do total=$(( total + ${p##*:} )); done + r=$(( RANDOM % total )) + for p in "${pairs[@]}"; do + name="${p%%:*}"; w="${p##*:}" + if (( r < w )); then echo "$name"; return; fi + r=$(( r - w )) + done + echo "${pairs[0]%%:*}" +} + +run_battle() { # $1=opponent $2=rounds $3=log file + PPOB_LOG_FILE="$3" java -cp "$CLASSES_DIR:$JAR" RunTraining "$1" "$2" +} + +eval_checkpoint() { + local tmp="$EVAL_LOG_FILE.tmp" wins rounds wr best + : > "$tmp" + echo ">>> [eval] $EVAL_ROUNDS deterministic rounds vs $EVAL_OPPONENT" + if ! SACLSTM_EVAL_MODE=1 run_battle "$EVAL_OPPONENT" "$EVAL_ROUNDS" "$tmp"; then + rm -f "$tmp" + echo ">>> [eval] crashed — keeping previous best" + return 0 + fi + mv "$tmp" "$EVAL_LOG_FILE" + wins=$(grep -c '"win":true' "$EVAL_LOG_FILE" || true) + rounds=$(grep -c '"type":"game"' "$EVAL_LOG_FILE" || true) + (( rounds == 0 )) && { echo ">>> [eval] no results"; return 0; } + wr=$(( 100 * wins / rounds )) + echo ">>> [eval] win rate: $wins/$rounds ($wr%) vs $EVAL_OPPONENT" + # ponytail: best-score state is a plain file next to the checkpoint; survives + # harness restarts, no lock needed (single harness instance assumed). + best=-1 + [ -f "$WEIGHTS_DIR/best_score.txt" ] && best=$(cat "$WEIGHTS_DIR/best_score.txt") + if (( wr > best )) && [ -f "$SACLSTM_WEIGHTS_PATH" ]; then + echo "$wr" > "$WEIGHTS_DIR/best_score.txt" + cp "$SACLSTM_WEIGHTS_PATH" "$WEIGHTS_DIR/sac_best.zip" + echo ">>> [eval] new best ($wr%) -> sac_best.zip" + fi +} + +NUM_CHUNKS=$(( (TOTAL_ROUNDS + CHUNK_SIZE - 1) / CHUNK_SIZE )) +fails=0 +for chunk in $(seq 1 "$NUM_CHUNKS"); do + ROUNDS=$CHUNK_SIZE + (( TOTAL_ROUNDS - (chunk - 1) * CHUNK_SIZE < CHUNK_SIZE )) && \ + ROUNDS=$(( TOTAL_ROUNDS - (chunk - 1) * CHUNK_SIZE )) + OPP=$(pick_opponent) + echo "=== Chunk $chunk/$NUM_CHUNKS: $ROUNDS rounds vs $OPP ===" + if ! run_battle "$OPP" "$ROUNDS" "$LOG_FILE"; then + fails=$(( fails + 1 )) + if (( fails >= MAX_CRASHES )); then + echo ">>> aborted: $fails consecutive crashes (bot process dying?)" + exit 1 + fi + # Crash recovery: RunTraining's liveness detection exited; the bot reloads + # its latest checkpoint on restart, so just rerun this chunk. + echo ">>> crash #$fails — restarting chunk from latest checkpoint" + (( chunk-- )); continue + fi + fails=0 + (( chunk % EVAL_INTERVAL == 0 )) && eval_checkpoint +done + +echo ">>> training complete: $NUM_CHUNKS chunks. Logs:" +echo " training: $LOG_FILE" +echo " eval: $EVAL_LOG_FILE" +[ -f "$WEIGHTS_DIR/sac_best.zip" ] && \ + echo " best: $WEIGHTS_DIR/sac_best.zip ($(cat "$WEIGHTS_DIR/best_score.txt")%)" +exit 0 diff --git a/SAC_LSTM_Bot/src/SAC_LSTM_Bot.json b/SAC_LSTM_Bot/src/SAC_LSTM_Bot.json index 166cd4e..eaeada0 100644 --- a/SAC_LSTM_Bot/src/SAC_LSTM_Bot.json +++ b/SAC_LSTM_Bot/src/SAC_LSTM_Bot.json @@ -1,8 +1,8 @@ { - "name": "Recurrent Royalty", + "name": "SAC_LSTM_Bot", "version": "0.1.0", "authors": ["Davide Cappellini"], - "description": "SAC+LSTM Tank Royale bot — skeleton with radar lock", + "description": "SAC+LSTM Tank Royale bot — self-reported identity; MUST match the name in ../SAC_LSTM_Bot.json (booter identity) or the training runner never sees this bot join", "homepage": "", "countryCodes": ["IT"], "gameTypes": ["classic", "melee", "1v1"], diff --git a/SAC_LSTM_Bot/src/SAC_LSTM_Bot.nim b/SAC_LSTM_Bot/src/SAC_LSTM_Bot.nim index bb16eac..2cb5dc7 100644 --- a/SAC_LSTM_Bot/src/SAC_LSTM_Bot.nim +++ b/SAC_LSTM_Bot/src/SAC_LSTM_Bot.nim @@ -144,8 +144,9 @@ method onScannedBot*(bot: SacBot, e: ScannedBotEvent) = bot.enemy.energy = e.energy bot.hasContact = true bot.ticksSinceScan = 0 - # Q12b/Q14: one NewBattle per battle, numeric scannedBotId - # ponytail: name-based opponent identity deferred to #49 (protocol has no names). + # Q12b/Q14+#49: one NewBattle per battle, numeric scannedBotId. Identity is + # keyed on getBotName(id) training-side (integration.opponentKey), numeric + # fallback in the pre-BotListUpdate window. if not bot.newBattleSent: bot.newBattleSent = true discard sendTrainingMsg(TrainingMsg(kind: tmkNewBattle, enemyId: e.scannedBotId)) @@ -169,6 +170,11 @@ method onGameAborted*(bot: SacBot) = bot.hasLastTrans = false bot.dmgDealt = 0; bot.dmgTaken = 0; bot.wastedPower = 0; bot.wallHits = 0 +method onRoundEnded*(bot: SacBot, e: RoundEndedEventForBot) = + ## Harness liveness signal (#49): RunTraining.java watches round_counter.txt + ## and aborts the battle if it freezes (dead bot process). Main thread. + bumpRoundCounter() + method onGameEnded*(bot: SacBot, e: GameEndedEventForBot) = ## Battle end -> terminal transition with done=true. Main thread; the API has ## joined the bot thread before this fires, so these plain fields are quiescent. diff --git a/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim b/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim index e8628ac..8d3c3d6 100644 --- a/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim +++ b/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim @@ -13,6 +13,7 @@ import arraymancer except Linear import std/[locks, os, math, random, strutils] +import tankroyale_botapi # getBotName (#49 name-based opponent identity) import SAC_LSTM_Bot/network import SAC_LSTM_Bot/state # STATE_DIM import SAC_LSTM_Bot/actions # ACTION_DIM @@ -38,6 +39,31 @@ proc getWeightsPath*(): string = getEnv("SACLSTM_WEIGHTS_PATH", currentSourcePath().parentDir / "weights" / "sac_latest.zip") +proc opponentKey*(enemyId: int): string = + ## Q14 follow-up (#49): name-based opponent identity from the v1.0.1 + ## BotListUpdate table; numeric-id fallback for the window before the first + ## update arrives (getBotName still returns "" then). Called on the training + ## thread — the lookup is lock-guarded in the API, no cross-thread refs. + let name = getBotName(enemyId) + if name.len > 0: name else: $enemyId + +proc bumpRoundCounter*() = + ## Liveness signal for tools/training_runner/RunTraining.java (#49): one + ## increment per round end; the runner aborts when it freezes (dead bot). + # ponytail: non-atomic read-modify-write; single writer (main thread) and the + # runner re-polls every 500ms with multi-round tolerance, torn reads self-heal. + let p = getWeightsPath().parentDir / "round_counter.txt" + var n = 0 + try: + n = parseInt(readFile(p).strip()) + except CatchableError: + discard # absent/garbage -> start at 1 + try: + createDir(p.parentDir) + writeFile(p, $(n + 1)) + except CatchableError: + discard # counter is best-effort liveness; never kill an event handler + # ── Channel message (plain data only) ──────────────────────────────────────── type @@ -200,7 +226,7 @@ type TrainState* = object trainer*: SACTrainer buf*: ReplayBuffer - lastEnemyId*: int + lastEnemyKey*: string # opponent identity key (#49 name-based, Q14) stepCount*: int nextSave*: int @@ -227,7 +253,7 @@ proc trainerFromFull*(initial: FullSnap): SACTrainer = proc initTrainState*(initial: FullSnap): TrainState = result.trainer = trainerFromFull(initial) result.buf = newReplayBuffer(getBufferCapacity(), STATE_DIM, ACTION_DIM) - result.lastEnemyId = -1 + result.lastEnemyKey = "" result.nextSave = getSaveInterval() proc handleTrainingMsg*(st: var TrainState; msg: TrainingMsg): bool = @@ -243,9 +269,11 @@ proc handleTrainingMsg*(st: var TrainState; msg: TrainingMsg): bool = done: msg.done)) of tmkNewBattle: # Q12a: keep the buffer if the opponent is unchanged, clear otherwise. - if msg.enemyId != st.lastEnemyId: + # Identity keyed on NAME (#49); numeric-id fallback pre-BotListUpdate. + let key = opponentKey(msg.enemyId) + if key != st.lastEnemyKey: st.buf.clear() - st.lastEnemyId = msg.enemyId + st.lastEnemyKey = key of tmkShutdown: return false true diff --git a/SAC_LSTM_Bot/tests/test_integration.nim b/SAC_LSTM_Bot/tests/test_integration.nim index fc11276..5d9c11e 100644 --- a/SAC_LSTM_Bot/tests/test_integration.nim +++ b/SAC_LSTM_Bot/tests/test_integration.nim @@ -3,7 +3,8 @@ ## drain-then-train NewBattle/Shutdown handling, trainPass safety below canSample. import arraymancer except Linear -import std/locks +import std/[locks, json, os] +import tankroyale_botapi # updateBotNames: seed the vendored id->name table import SAC_LSTM_Bot/integration import SAC_LSTM_Bot/state # STATE_DIM import SAC_LSTM_Bot/actions # ACTION_DIM @@ -46,14 +47,20 @@ block: assert not ok4, "closed channel must report dataAvailable=false" echo "PASS TrainingMsg channel round-trip" -# ── 2. NewBattle clears only when the opponent changes; Shutdown stops ──────── +# ── 2. NewBattle clears only when the opponent NAME changes; Shutdown stops ─── block: + # Seed the API's id->name table (v1.0.1) for name-based identity (#49). + updateBotNames(parseJson( + """{"bots":[{"id":7,"name":"Corners"},{"id":8,"name":"Crazy"}]}""")) + assert opponentKey(7) == "Corners", "known id resolves to name" + assert opponentKey(99) == "99", "unknown id falls back to numeric string" + var st: TrainState st.trainer = initSACTrainer(STATE_DIM, ACTION_DIM) st.buf = newReplayBuffer(64, STATE_DIM, ACTION_DIM, burnIn = 2, trainWindow = 3) assert handleTrainingMsg(st, TrainingMsg(kind: tmkNewBattle, enemyId: 7)) - assert st.lastEnemyId == 7 + assert st.lastEnemyKey == "Corners" for i in 0 ..< 5: var m = TrainingMsg(kind: tmkTransition) @@ -68,11 +75,24 @@ block: # Opponent changed -> clear. assert handleTrainingMsg(st, TrainingMsg(kind: tmkNewBattle, enemyId: 8)) assert st.buf.len == 0, "opponent change must clear" - assert st.lastEnemyId == 8 + assert st.lastEnemyKey == "Crazy" + + # Nameless window (pre-BotListUpdate) is a distinct key. + assert handleTrainingMsg(st, TrainingMsg(kind: tmkNewBattle, enemyId: 99)) + assert st.lastEnemyKey == "99" and st.buf.len == 0 # Shutdown stops the caller's loop. assert not handleTrainingMsg(st, TrainingMsg(kind: tmkShutdown)) - echo "PASS drain-then-train NewBattle/Shutdown" + echo "PASS name-keyed NewBattle clear / Shutdown" + +# ── 2b. bumpRoundCounter increments per call, cold-starts at 1 ──────────────── +block: + let tmp = getTempDir() / "sac_test_weights_" & $getCurrentProcessId() + putEnv("SACLSTM_WEIGHTS_PATH", tmp / "sac_latest.zip") + bumpRoundCounter() + bumpRoundCounter() + assert readFile(tmp / "round_counter.txt") == "2", "counter increments per round" + echo "PASS bumpRoundCounter" # ── 3. trainPass is a safe no-op below canSample (no steps, no publish) ─────── block: diff --git a/tools/training_runner/RunTraining.java b/tools/training_runner/RunTraining.java index c976d16..b86a06b 100644 --- a/tools/training_runner/RunTraining.java +++ b/tools/training_runner/RunTraining.java @@ -18,9 +18,10 @@ import java.util.logging.Logger; * to the same log file via PPOB_LOG_FILE — the shell wrapper stitches them. * * Usage (env vars): - * PPO_BOT_DIR — path to PPO_Bot dir + * PPO_BOT_DIR — path to the bot dir (any Tank Royale bot) * SAMPLE_BOTS_DIR — path to sample bots archive * PPOB_LOG_FILE — path to training_log.jsonl (appended) + * BOT_NAME — bot name to match in round results (default: PPO_Bot) * TRAINING_OPPONENT — opponent bot name (default: Target) * TRAINING_ROUNDS — number of rounds to run (CLI arg or env var) * @@ -39,8 +40,9 @@ public class RunTraining { : System.getenv().getOrDefault("TRAINING_OPPONENT", "Target"); int totalRounds = args.length > 1 ? Integer.parseInt(args[1]) : Integer.parseInt(System.getenv().getOrDefault("TRAINING_ROUNDS", "100")); + String botName = System.getenv().getOrDefault("BOT_NAME", "PPO_Bot"); - System.out.printf("Training: PPO_Bot vs %s for %d rounds%n", opponent, totalRounds); + System.out.printf("Training: %s vs %s for %d rounds%n", botName, opponent, totalRounds); System.out.printf("Log: %s%n", logFile); // Dead-bot guard: the runner keeps listing a crashed PPO_Bot in the @@ -81,7 +83,7 @@ public class RunTraining { boolean win = false; boolean found = false; for (var r : event.getResults()) { - if (r.getName().equals("PPO_Bot")) { + if (r.getName().equals(botName)) { found = true; totalScore = r.getTotalScore(); win = r.getRank() == 1; @@ -99,9 +101,9 @@ public class RunTraining { frozenRounds[0]++; long frozenMs = System.currentTimeMillis() - lastAdvanceMs[0]; if (frozenRounds[0] >= 10 && frozenMs >= 10_000) { - System.err.printf("PPO_Bot round_counter frozen at %d for %d " + System.err.printf("%s round_counter frozen at %d for %d " + "harness rounds / %.0fs — process dead, aborting for restart%n", - ctr, frozenRounds[0], frozenMs / 1000.0); + botName, ctr, frozenRounds[0], frozenMs / 1000.0); System.exit(1); } } else { @@ -110,7 +112,7 @@ public class RunTraining { lastAdvanceMs[0] = System.currentTimeMillis(); } if (!found) { - System.err.println("PPO_Bot missing from round " + round + System.err.println(botName + " missing from round " + round + " results — process died, aborting battle for restart"); System.exit(1); } @@ -147,10 +149,10 @@ public class RunTraining { endCounter = readCounter(counterPath); } if (endCounter < expectedEnd) { - System.err.printf("PPO_Bot round_counter %d < expected %d (start+%d) at battle " + System.err.printf("%s round_counter %d < expected %d (start+%d) at battle " + "end (waited 60s) — %d rounds never trained (corpse?), aborting for " + "restart%n", - endCounter, expectedEnd, totalRounds, expectedEnd - endCounter); + botName, endCounter, expectedEnd, totalRounds, expectedEnd - endCounter); System.exit(1); } System.out.printf("Counter check passed: %d == expected %d%n", endCounter, expectedEnd);