fix(botapi): static event queue storage + end-of-battle train wait
The event queue's heap seq was the last GC'd block surviving across rounds: each round runs on a freshly spawned bot thread, so the N+1 thread realloc'd a block grown by dead thread N's allocator mid-round (at the next capacity doubling, ~turn 104) -> rawDealloc SIGSEGV in addEvent (7 gdb-confirmed coredumps). Replace with a static array[MAX_QUEUE_SIZE, BotEvent] + eventsLen: no heap block crosses threads, realloc can never happen. Also fix the harness aborting the final round mid-train: PPO_Bot's onRoundEnded trains synchronously after the runner's RoundEndedEvent, so the counter read right after awaitResults() is the stale pre-train value and System.exit killed the bot inside ppoUpdate. Poll up to 60s for the counter to catch up before declaring the battle incomplete. Verified: 72 consecutive rounds vs Fire, 100% wins, all rounds trained (counter advanced 1:1), zero coredumps since the fix.
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
import dev.robocode.tankroyale.runner.*;
|
||||
import dev.robocode.tankroyale.client.model.*;
|
||||
import java.io.*;
|
||||
import java.nio.file.*;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Training runner: PPO_Bot vs opponent for N rounds, all in one battle.
|
||||
*
|
||||
* All rounds run in a single battle so PPO_Bot's process (and its training
|
||||
* thread) survives across rounds — that is how checkpoints + round_counter
|
||||
* advance. Per-round battles killed the bot's training thread at round end.
|
||||
*
|
||||
* Writes one JSON line per round to the log file for game-level outcomes.
|
||||
* PPO_Bot itself writes training health fields (actorLoss, valueLoss, etc.)
|
||||
* to the same log file via PPOB_LOG_FILE — the shell wrapper stitches them.
|
||||
*
|
||||
* Usage (env vars):
|
||||
* PPO_BOT_DIR — path to PPO_Bot dir
|
||||
* SAMPLE_BOTS_DIR — path to sample bots archive
|
||||
* PPOB_LOG_FILE — path to training_log.jsonl (appended)
|
||||
* TRAINING_OPPONENT — opponent bot name (default: Target)
|
||||
* TRAINING_ROUNDS — number of rounds to run (CLI arg or env var)
|
||||
*
|
||||
* CLI: java RunTraining [opponent] [rounds]
|
||||
*/
|
||||
public class RunTraining {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Logger.getLogger("dev.robocode.tankroyale").setLevel(Level.WARNING);
|
||||
|
||||
String ppoBotDir = requireEnv("PPO_BOT_DIR");
|
||||
String sampleBots = requireEnv("SAMPLE_BOTS_DIR");
|
||||
String logFile = requireEnv("PPOB_LOG_FILE");
|
||||
|
||||
String opponent = args.length > 0 ? args[0]
|
||||
: System.getenv().getOrDefault("TRAINING_OPPONENT", "Target");
|
||||
int totalRounds = args.length > 1 ? Integer.parseInt(args[1])
|
||||
: Integer.parseInt(System.getenv().getOrDefault("TRAINING_ROUNDS", "100"));
|
||||
|
||||
System.out.printf("Training: PPO_Bot vs %s for %d rounds%n", opponent, totalRounds);
|
||||
System.out.printf("Log: %s%n", logFile);
|
||||
|
||||
// Dead-bot guard: the runner keeps listing a crashed PPO_Bot in the
|
||||
// round results (name stays present, rank collapses), so `!found`
|
||||
// never fires. Instead watch the counter PPO_Bot writes at round
|
||||
// end (weights/round_counter.txt, BEFORE the runner's round event —
|
||||
// verified empirically). Aware that the counter write lags the
|
||||
// harness: sync training + checkpoint save take up to ~7s while
|
||||
// server rounds complete in 1-4s, so requiring only "no advance for
|
||||
// 2 harness rounds" false-positives on healthy bots (observed). A
|
||||
// truly dead process stays frozen for minutes, so abort only when
|
||||
// the counter has not advanced for BOTH >=10 harness rounds AND
|
||||
// >=10s wall time: exit(1) so run.sh's crash-restart loop resumes
|
||||
// from the counter. (Hoisted above the try so the end-of-battle
|
||||
// completeness check below can reuse them.)
|
||||
Path counterPath = Paths.get(ppoBotDir, "weights", "round_counter.txt");
|
||||
long[] lastCounter = { readCounter(counterPath) };
|
||||
long startCounter = lastCounter[0];
|
||||
int[] frozenRounds = { 0 };
|
||||
long[] lastAdvanceMs = { System.currentTimeMillis() };
|
||||
|
||||
try (var runner = BattleRunner.create(b -> b.embeddedServer().suppressServerOutput())) {
|
||||
var setup = BattleSetup.classic(s -> s.setNumberOfRounds(totalRounds));
|
||||
var bots = List.of(
|
||||
BotEntry.of(ppoBotDir),
|
||||
BotEntry.of(sampleBots + "/" + opponent)
|
||||
);
|
||||
|
||||
var owner = new Object();
|
||||
|
||||
try (var handle = runner.startBattleAsync(setup, bots)) {
|
||||
|
||||
handle.getOnRoundEnded().on(owner, event -> {
|
||||
int round = event.getRoundNumber();
|
||||
int ticks = event.getTurnNumber();
|
||||
int score = 0;
|
||||
boolean win = false;
|
||||
boolean found = false;
|
||||
for (var r : event.getResults()) {
|
||||
if (r.getName().equals("PPO_Bot")) {
|
||||
found = true;
|
||||
score = r.getTotalScore();
|
||||
win = r.getRank() == 1;
|
||||
}
|
||||
}
|
||||
// PPO_Bot's process died mid-battle: abort so run.sh's crash-restart
|
||||
// loop resumes from round_counter instead of grinding dummy rounds.
|
||||
// Frozen for a few harness rounds can be a healthy-but-lagging counter
|
||||
// (sync training + checkpoint save vs fast server rounds), so also
|
||||
// require 10s of wall time before declaring death.
|
||||
long ctr = readCounter(counterPath);
|
||||
if (ctr == lastCounter[0]) {
|
||||
frozenRounds[0]++;
|
||||
long frozenMs = System.currentTimeMillis() - lastAdvanceMs[0];
|
||||
if (frozenRounds[0] >= 10 && frozenMs >= 10_000) {
|
||||
System.err.printf("PPO_Bot round_counter frozen at %d for %d "
|
||||
+ "harness rounds / %.0fs — process dead, aborting for restart%n",
|
||||
ctr, frozenRounds[0], frozenMs / 1000.0);
|
||||
System.exit(1);
|
||||
}
|
||||
} else {
|
||||
lastCounter[0] = ctr;
|
||||
frozenRounds[0] = 0;
|
||||
lastAdvanceMs[0] = System.currentTimeMillis();
|
||||
}
|
||||
if (!found) {
|
||||
System.err.println("PPO_Bot missing from round " + round
|
||||
+ " results — process died, aborting battle for restart");
|
||||
System.exit(1);
|
||||
}
|
||||
// Append game-outcome JSON line
|
||||
String line = String.format(
|
||||
"{\"type\":\"game\",\"round\":%d,\"ticks\":%d,\"score\":%d,\"win\":%b,\"opponent\":\"%s\"}",
|
||||
round, ticks, score, win, opponent
|
||||
);
|
||||
try {
|
||||
appendLine(logFile, line);
|
||||
} catch (IOException e) {
|
||||
System.err.println("Failed to append log line: " + e);
|
||||
}
|
||||
System.out.printf("Round %d/%d — ticks:%d score:%d win:%b%n",
|
||||
round, totalRounds, ticks, score, win);
|
||||
});
|
||||
|
||||
handle.awaitResults();
|
||||
|
||||
// The final round's train lags the battle end: PPO_Bot's
|
||||
// onRoundEnded runs ppoUpdate + checkpoint save synchronously
|
||||
// on its own main thread, which can take longer than the
|
||||
// runner takes to deliver the last RoundEnded/GameEnded
|
||||
// events. So the counter right after awaitResults() is often
|
||||
// the stale pre-train value of the LAST round. Poll for it to
|
||||
// catch up before declaring the battle incomplete (the
|
||||
// mid-battle guard already catches truly frozen processes,
|
||||
// and a timeout still aborts for restart).
|
||||
long endCounter = readCounter(counterPath);
|
||||
long expectedEnd = startCounter + totalRounds; // every round must have trained
|
||||
long waitUntil = System.currentTimeMillis() + 60_000;
|
||||
while (endCounter < expectedEnd && System.currentTimeMillis() < waitUntil) {
|
||||
Thread.sleep(500);
|
||||
endCounter = readCounter(counterPath);
|
||||
}
|
||||
if (endCounter < expectedEnd) {
|
||||
System.err.printf("PPO_Bot round_counter %d < expected %d (start+%d) at battle "
|
||||
+ "end (waited 60s) — %d rounds never trained (corpse?), aborting for "
|
||||
+ "restart%n",
|
||||
endCounter, expectedEnd, totalRounds, expectedEnd - endCounter);
|
||||
System.exit(1);
|
||||
}
|
||||
System.out.printf("Counter check passed: %d == expected %d%n", endCounter, expectedEnd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void appendLine(String path, String line) throws IOException {
|
||||
try (var w = Files.newBufferedWriter(Paths.get(path),
|
||||
StandardOpenOption.CREATE,
|
||||
StandardOpenOption.APPEND)) {
|
||||
w.write(line);
|
||||
w.newLine();
|
||||
}
|
||||
}
|
||||
|
||||
/** Read PPO_Bot's persisted round counter; -1 when absent/unreadable. */
|
||||
static long readCounter(Path p) {
|
||||
try {
|
||||
return Long.parseLong(Files.readString(p).trim());
|
||||
} catch (Exception e) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
static String requireEnv(String name) {
|
||||
var v = System.getenv(name);
|
||||
if (v == null || v.isBlank()) {
|
||||
System.err.println("Error: " + name + " env var not set");
|
||||
System.exit(1);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
}
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# Training runner: compile PPO_Bot, then loop the Java battle runner with crash recovery.
|
||||
#
|
||||
# Usage:
|
||||
# ./run.sh [opponent] [rounds]
|
||||
#
|
||||
# Arguments (also settable via env vars):
|
||||
# opponent — bot name under SAMPLE_BOTS_DIR (default: Target)
|
||||
# rounds — total rounds to run (default: 100)
|
||||
#
|
||||
# Env vars:
|
||||
# TANK_ROYALE_JAR — path to robocode-tankroyale-runner.jar
|
||||
# SAMPLE_BOTS_DIR — path to sample bots archive
|
||||
# PPOB_LOG_FILE — JSON lines log file (default: training_log.jsonl in repo root)
|
||||
# PPOB_LR, PPOB_CLIP_EPSILON, PPOB_ENTROPY_COEFF, PPOB_VALUE_LOSS_COEFF,
|
||||
# PPOB_MAX_GRAD_NORM, PPOB_GAMMA, PPOB_LAM, PPOB_EPOCHS, PPOB_MINI_BATCH_SIZE
|
||||
# — hyperparameters (all optional, PPO_Bot uses sensible defaults)
|
||||
set -uo pipefail
|
||||
|
||||
ENV_FILE="$(cd "$(dirname "$0")" && pwd)/training.env"
|
||||
[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
OPPONENT="${1:-${TRAINING_OPPONENT:-Target}}"
|
||||
TOTAL_ROUNDS="${2:-${TRAINING_ROUNDS:-100}}"
|
||||
|
||||
JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}"
|
||||
PPO_BOT_SRC="$REPO_ROOT/PPO_Bot"
|
||||
|
||||
export PPO_BOT_DIR="$PPO_BOT_SRC"
|
||||
export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}"
|
||||
export PPOB_LOG_FILE="${PPOB_LOG_FILE:-$REPO_ROOT/training_log.jsonl}"
|
||||
|
||||
echo "=== Training runner ==="
|
||||
echo "Opponent: $OPPONENT"
|
||||
echo "Total rounds: $TOTAL_ROUNDS"
|
||||
echo "Log file: $PPOB_LOG_FILE"
|
||||
echo ""
|
||||
|
||||
# ── 1. Compile PPO_Bot ────────────────────────────────────────────────────────
|
||||
echo ">>> Compiling PPO_Bot..."
|
||||
(cd "$PPO_BOT_SRC" && nimble build -d:release)
|
||||
echo ">>> PPO_Bot compiled."
|
||||
|
||||
# ── 2. Compile RunTraining.java ───────────────────────────────────────────────
|
||||
cd "$SCRIPT_DIR"
|
||||
echo ">>> Compiling RunTraining.java..."
|
||||
javac -cp "$JAR" RunTraining.java
|
||||
echo ">>> RunTraining compiled."
|
||||
|
||||
# ── 3. Run with crash-restart loop ───────────────────────────────────────────
|
||||
# We pass total rounds to the Java runner. On crash, restart from where we left
|
||||
# off by reading the round counter PPO_Bot persists to weights/round_counter.txt.
|
||||
# ponytail: simple restart from persisted counter; no partial-round recovery needed
|
||||
|
||||
ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt"
|
||||
|
||||
remaining_rounds() {
|
||||
local done=0
|
||||
if [ -f "$ROUND_COUNTER_FILE" ]; then
|
||||
done=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0)
|
||||
fi
|
||||
echo $(( TOTAL_ROUNDS - done ))
|
||||
}
|
||||
|
||||
attempts=0
|
||||
while true; do
|
||||
remaining=$(remaining_rounds)
|
||||
if [ "$remaining" -le 0 ]; then
|
||||
echo ">>> All $TOTAL_ROUNDS rounds complete."
|
||||
break
|
||||
fi
|
||||
|
||||
attempts=$(( attempts + 1 ))
|
||||
if [ "$attempts" -gt 1 ]; then
|
||||
echo ">>> Restart #$attempts — $remaining rounds remaining..."
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
echo ">>> Running $remaining rounds (opponent: $OPPONENT)..."
|
||||
java -cp ".:$JAR" RunTraining "$OPPONENT" "$remaining" && break || {
|
||||
exit_code=$?
|
||||
echo ">>> Java runner exited with code $exit_code — checking if done..."
|
||||
remaining=$(remaining_rounds)
|
||||
if [ "$remaining" -le 0 ]; then
|
||||
echo ">>> All rounds complete despite non-zero exit."
|
||||
break
|
||||
fi
|
||||
echo ">>> Crash detected ($remaining rounds remain). Restarting..."
|
||||
}
|
||||
done
|
||||
|
||||
echo ">>> Training done. Log at $PPOB_LOG_FILE"
|
||||
@@ -0,0 +1,16 @@
|
||||
TRAINING_OPPONENT=Fire
|
||||
TRAINING_ROUNDS=3000
|
||||
PPOB_LOG_FILE=/home/davide/Projects/SirRoboGarage/tools/training_runner/logs/fire_training.jsonl
|
||||
PPOB_LR=0.0003
|
||||
PPOB_CLIP_EPSILON=0.2
|
||||
PPOB_ENTROPY_COEFF=0.01
|
||||
PPOB_VALUE_LOSS_COEFF=0.5
|
||||
PPOB_MAX_GRAD_NORM=0.5
|
||||
PPOB_GAMMA=0.99
|
||||
PPOB_LAM=0.95
|
||||
PPOB_EPOCHS=4
|
||||
PPOB_MINI_BATCH_SIZE=64
|
||||
PPOB_LOG_STD_FLOOR=-3.0
|
||||
PPOB_INITIAL_LOG_STD=0.0
|
||||
|
||||
# PPOB_EVAL_ONLY=1 → freeze training (pure evaluation)
|
||||
Reference in New Issue
Block a user