From 12624d3069ee2b57924da8c84c71424db4270054 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Thu, 20 Aug 2026 14:27:28 +0200 Subject: [PATCH] feat(PPO_Bot): bot-relative bullets + scan staleness (STATE_DIM=57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bullet state (indices 44-55): enemy-relative → bot-relative frame (bot needs threat vectors to itself for dodging, not to enemy) - New index 56: scan staleness = min(ticksSinceLastScan / 30, 1.0) (gives policy a confidence signal for enemy data freshness) - warm_start.py updated: 44→57 dim expansion, TARGET_DIM variable - Tests updated for new state layout Co-Authored-By: Claude Sonnet 4.6 --- PPO_Bot/network.nim | 2 +- PPO_Bot/state_vector.nim | 18 +- PPO_Bot/tests/test_state.nim | 16 +- tools/training_runner/curriculum_retrain.sh | 126 +++++++ tools/training_runner/generalist_train.sh | 351 ++++++++++++++++++++ tools/training_runner/mixed_train.sh | 155 +++++++++ tools/warm_start.py | 66 ++++ 7 files changed, 718 insertions(+), 16 deletions(-) create mode 100755 tools/training_runner/curriculum_retrain.sh create mode 100755 tools/training_runner/generalist_train.sh create mode 100755 tools/training_runner/mixed_train.sh create mode 100644 tools/warm_start.py diff --git a/PPO_Bot/network.nim b/PPO_Bot/network.nim index 9da2083..4b21fd4 100644 --- a/PPO_Bot/network.nim +++ b/PPO_Bot/network.nim @@ -4,7 +4,7 @@ import arraymancer import std/[math, random] const - STATE_DIM* = 56 + STATE_DIM* = 57 ACTION_DIM* = 6 var diff --git a/PPO_Bot/state_vector.nim b/PPO_Bot/state_vector.nim index 861a017..7c6515b 100644 --- a/PPO_Bot/state_vector.nim +++ b/PPO_Bot/state_vector.nim @@ -1,4 +1,4 @@ -## State vector builder — produces 56-float normalized tensor for PPO policy. +## State vector builder — produces 57-float normalized tensor for PPO policy. ## No bot API imports; takes plain BotState + EnemyTracker structs. import std/math @@ -26,10 +26,11 @@ proc buildStateVector*(bot: BotStateData; enemy: EnemyTracker; remainingGunAngle: float64 = 0.0; bullets: openArray[BulletData] = []; bulletCount: int = 0): Tensor[float32] = - ## Build the 56-float normalized state tensor. + ## Build the 57-float normalized state tensor. ## Indices 0-43: existing features. Indices 44-55: up to 3 bullet slots (4 floats each). + ## Index 56: scan staleness (ticksSinceLastScan / 30, clamped to 1). ## All values clipped to roughly [-1, 1] via division by physical maxima. - result = zeros[float32](56) + result = zeros[float32](57) let aW = bot.arenaWidth let aH = bot.arenaHeight @@ -102,14 +103,12 @@ proc buildStateVector*(bot: BotStateData; enemy: EnemyTracker; # --- Bullet tracking (indices 44-55): up to 3 enemy bullets, 4 floats each --- # Per bullet: relX/aW, relY/aH, speed/20, ticksToImpact/diag - # Positions are relative to enemy (threat vector). Slots beyond bulletCount stay 0. - let ex = if enemy.hasContact: enemy.current.x else: bot.arenaWidth / 2.0 - let ey = if enemy.hasContact: enemy.current.y else: bot.arenaHeight / 2.0 + # Positions are relative to bot (useful for dodging). Slots beyond bulletCount stay 0. for i in 0 ..< min(bulletCount, 3): let b = bullets[i] let bSpeed = 20.0 - 3.0 * b.power # Tank Royale bullet speed formula - let bdx = b.x - ex - let bdy = b.y - ey + let bdx = b.x - bot.x + let bdy = b.y - bot.y let bdist = sqrt(bdx * bdx + bdy * bdy) let ticks = if bSpeed > 0.0: bdist / bSpeed else: 0.0 let base = 44 + i * 4 @@ -117,3 +116,6 @@ proc buildStateVector*(bot: BotStateData; enemy: EnemyTracker; result[base + 1] = float32(bdy / bot.arenaHeight) result[base + 2] = float32(bSpeed / 20.0) result[base + 3] = float32(ticks / diag) + + # --- Scan staleness (index 56) --- + result[56] = float32(min(enemy.current.ticksSinceLastScan.float64 / 30.0, 1.0)) diff --git a/PPO_Bot/tests/test_state.nim b/PPO_Bot/tests/test_state.nim index 8cc8282..cf538dd 100644 --- a/PPO_Bot/tests/test_state.nim +++ b/PPO_Bot/tests/test_state.nim @@ -84,7 +84,7 @@ block testStateVectorLength: arenaWidth: 800.0, arenaHeight: 600.0, ) let sv = buildStateVector(bot, t) - check sv.shape == [56], "state vector has 56 elements" + check sv.shape == [57], "state vector has 57 elements" block testStateVectorRange: var t = initEnemyTracker() @@ -95,7 +95,7 @@ block testStateVectorRange: arenaWidth: 800.0, arenaHeight: 600.0, ) let sv = buildStateVector(bot, t) - for i in 0 ..< 56: + for i in 0 ..< 57: check sv[i] >= -2.0f32 and sv[i] <= 2.0f32, &"sv[{i}]={sv[i]} out of [-2,2] range" @@ -161,6 +161,8 @@ block testHistoryPaddedWhenEmpty: # indices 44-55 (bullet slots) default to 0 when no bullets provided for i in 44 ..< 56: check sv[i] == 0.0f32, &"bullet slot {i} should be 0 when no bullets" + # index 56 (staleness): no contact so ticksSinceLastScan=0 → 0/30 = 0 + check sv[56] == 0.0f32, "sv[56] staleness should be 0 when no contact" block testBulletSlots: var t = initEnemyTracker() @@ -171,15 +173,15 @@ block testBulletSlots: arenaWidth: 800.0, arenaHeight: 600.0, ) # Bullet at (300,250), power 1.0 → speed = 20-3 = 17 - # relX = 300-400 = -100, relY = 250-300 = -50 - # dist to enemy = sqrt(100^2+50^2) ≈ 111.8, ticks ≈ 111.8/17 ≈ 6.58 + # relX = 300-200 = 100, relY = 250-200 = 50 (relative to bot, not enemy) + # dist to bot = sqrt(100^2+50^2) ≈ 111.8, ticks ≈ 111.8/17 ≈ 6.58 let b = BulletData(x: 300.0, y: 250.0, power: 1.0) let sv = buildStateVector(bot, t, 0.0, 0.0, [b], 1) - check abs(sv[44] - (-100.0/800.0).float32) < 0.001f32, "bullet relX" - check abs(sv[45] - (-50.0/600.0).float32) < 0.001f32, "bullet relY" + check abs(sv[44] - (100.0/800.0).float32) < 0.001f32, "bullet relX" + check abs(sv[45] - (50.0/600.0).float32) < 0.001f32, "bullet relY" check abs(sv[46] - (17.0/20.0).float32) < 0.001f32, "bullet speed norm" # second slot should be zero-padded - for i in 48 ..< 56: + for i in 48 ..< 57: check sv[i] == 0.0f32, &"unused bullet slot {i} should be 0" echo "All tests passed" diff --git a/tools/training_runner/curriculum_retrain.sh b/tools/training_runner/curriculum_retrain.sh new file mode 100755 index 0000000..3383e20 --- /dev/null +++ b/tools/training_runner/curriculum_retrain.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# curriculum_retrain.sh — sequential curriculum retraining for PPO_Bot v2. +# Bots in order: Fire → MyFirstDroid → Target → MyFirstLeader → Crazy → Corners → PaintingBot → MyFirstBot +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PPO_BOT_SRC="$REPO_ROOT/PPO_Bot" + +# Source hyperparams (but we'll override opponent/rounds/log per phase) +ENV_FILE="$SCRIPT_DIR/training.env" +[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a + +JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}" +export PPO_BOT_DIR="$PPO_BOT_SRC" +export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}" + +LOGS_DIR="$SCRIPT_DIR/logs" +SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots" +SUMMARY="$LOGS_DIR/curriculum_retrain_summary.txt" +mkdir -p "$LOGS_DIR" "$SNAPSHOTS_DIR" + +ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt" + +echo "=== PPO_Bot v2 curriculum retrain — $(date) ===" | tee -a "$SUMMARY" + +# ── compile once up front (run.sh does this; curriculum skipped it) ─────────── +echo ">>> Compiling PPO_Bot..." +(cd "$PPO_BOT_SRC" && nimble build -d:release) +echo ">>> Compiling RunTraining.java..." +(cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java) +echo ">>> Compilation done." + +# ── helpers ────────────────────────────────────────────────────────────────── + +run_java() { + # run_java + local opp="$1" rounds="$2" + local remaining="$rounds" + local attempts=0 + while [ "$remaining" -gt 0 ]; do + attempts=$(( attempts + 1 )) + [ "$attempts" -gt 1 ] && sleep 2 + java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$remaining" || \ + echo ">>> Java exited non-zero (crash/restart expected)" + local done=0 + [ -f "$ROUND_COUNTER_FILE" ] && done=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0) + remaining=$(( rounds - done )) + # Guard: if counter didn't advance at all on this attempt, something is + # fundamentally broken (bad binary, missing bots dir, etc.) — abort rather + # than loop forever. + if [ "$attempts" -ge 3 ] && [ "$done" -eq 0 ]; then + echo "ERROR: round_counter still 0 after $attempts attempts — aborting" >&2 + exit 1 + fi + done +} + +count_wins() { + # count_wins → prints win count + # grep -c exits 1 on zero matches (not an error) which was triggering || echo 0 + # making WINS="0\n0". Use awk which always exits 0. + local f="$1" + [ -f "$f" ] || { echo 0; return; } + awk '/"win":true/{c++} END{print c+0}' "$f" +} + +snap_weights() { + local label="$1" + local counter + counter=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0) + local dst="$SNAPSHOTS_DIR/v2_${label}_r${counter}" + cp -a "$PPO_BOT_SRC/weights/latest" "$dst" + echo " snapshot → $dst" +} + +# ── curriculum ──────────────────────────────────────────────────────────────── + +declare -A TRAIN_ROUNDS=( [Fire]=200 [MyFirstDroid]=200 [Target]=200 [MyFirstLeader]=200 [Crazy]=200 + [Corners]=400 [PaintingBot]=400 [MyFirstBot]=1000 ) +BOTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot) +GATE=150 +GATE_PASS=148 + +for BOT in "${BOTS[@]}"; do + TR="${TRAIN_ROUNDS[$BOT]}" + echo "" | tee -a "$SUMMARY" + echo "══ $BOT (train=${TR} gate=${GATE}) ══" | tee -a "$SUMMARY" + + # Reset round counter so run_java terminates correctly for each phase + echo 0 > "$ROUND_COUNTER_FILE" + + # ── training phase ────────────────────────────────────────────────────────── + TRAIN_LOG="$LOGS_DIR/${BOT}_train.jsonl" + export TRAINING_OPPONENT="$BOT" + export TRAINING_ROUNDS="$TR" + export PPOB_LOG_FILE="$TRAIN_LOG" + unset PPOB_EVAL_ONLY + > "$TRAIN_LOG" + echo " [train] $TR rounds..." | tee -a "$SUMMARY" + run_java "$BOT" "$TR" + echo " [train] done" | tee -a "$SUMMARY" + + # ── gate eval ─────────────────────────────────────────────────────────────── + echo 0 > "$ROUND_COUNTER_FILE" + EVAL_LOG="$LOGS_DIR/${BOT}_eval.jsonl" + export TRAINING_ROUNDS="$GATE" + export PPOB_LOG_FILE="$EVAL_LOG" + export PPOB_EVAL_ONLY=1 + > "$EVAL_LOG" + echo " [eval] $GATE rounds..." | tee -a "$SUMMARY" + run_java "$BOT" "$GATE" + + WINS=$(count_wins "$EVAL_LOG") + if [ "$WINS" -ge "$GATE_PASS" ]; then + STATUS="PASS ($WINS/$GATE)" + snap_weights "$BOT" + else + STATUS="FAIL ($WINS/$GATE)" + fi + echo " [gate] $STATUS" | tee -a "$SUMMARY" + unset PPOB_EVAL_ONLY +done + +echo "" | tee -a "$SUMMARY" +echo "=== curriculum complete — $(date) ===" | tee -a "$SUMMARY" diff --git a/tools/training_runner/generalist_train.sh b/tools/training_runner/generalist_train.sh new file mode 100755 index 0000000..6aa834e --- /dev/null +++ b/tools/training_runner/generalist_train.sh @@ -0,0 +1,351 @@ +#!/usr/bin/env bash +# generalist_train.sh — adaptive mixed-opponent training for a generalist PPO_Bot. +# +# Problem it solves: the sequential curriculum killed earlier skills (catastrophic +# forgetting). Instead we sample ONE opponent per chunk, weighted by how weak the +# bot currently is against it (max(FLOOR, 1 - winrate)), and keep the best +# generalist checkpoint by mean eval winrate across all 8 opponents. +# +# State / logs (all plain files, resumable): +# /tmp/generalist_wr.txt — one line per opponent: "Name winrate" +# /tmp/generalist_seed.txt — optional starting winrates (same format); +# wins over wr.txt when both exist +# /tmp/generalist_best.txt — "mean rounds" of the best generalist eval so far +# /tmp/generalist_train.log — full run log (tee'd) +# /tmp/generalist_eval_.jsonl — per-opponent eval battles (fresh per eval) +# /tmp/generalist_train_.jsonl — per-opponent training battles (accumulated) +# +# Env-overridable config: TOTAL_ROUNDS CHUNK_SIZE EVAL_INTERVAL EVAL_ROUNDS +# PASS_RATE FLOOR ALPHA (all declared in the config block below). +# +# Usage: +# ./generalist_train.sh — run the full training loop +# ./generalist_train.sh --selftest — verify the weighted sampler against the +# expected distribution (no battles, no compilation) +# +# Dead-bot resilience is inherited from the harness: RunTraining.java aborts +# (exit 1) when PPO_Bot's round counter is frozen or short at battle end, and +# this script retries; a chunk that makes no counter progress across 3 attempts +# aborts loudly instead of silently burning rounds. +set -uo pipefail +export LC_ALL=C + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PPO_BOT_SRC="$REPO_ROOT/PPO_Bot" + +ENV_FILE="$SCRIPT_DIR/training.env" +[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a + +JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}" +export PPO_BOT_DIR="$PPO_BOT_SRC" +export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}" + +SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots" +mkdir -p "$SNAPSHOTS_DIR" + +ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt" +LOG="/tmp/generalist_train.log" +WR_STATE_FILE="/tmp/generalist_wr.txt" +WR_SEED_FILE="/tmp/generalist_seed.txt" +BEST_FILE="/tmp/generalist_best.txt" +SELFTEST_LOG="/tmp/generalist_sampling_selftest.log" + +# ── config (env-overridable) ────────────────────────────────────────────────── +TOTAL_ROUNDS="${TOTAL_ROUNDS:-40000}" +CHUNK_SIZE="${CHUNK_SIZE:-60}" +EVAL_INTERVAL="${EVAL_INTERVAL:-1000}" +EVAL_ROUNDS="${EVAL_ROUNDS:-30}" +PASS_RATE="${PASS_RATE:-0.90}" +FLOOR="${FLOOR:-0.05}" # minimum sampling weight — keeps maintenance sampling alive +ALPHA="${ALPHA:-0.3}" # winrate EMA smoothing: new = ALPHA*old + (1-ALPHA)*eval + +OPPONENTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot) + +# ── helpers ─────────────────────────────────────────────────────────────────── + +# read_counter — PPO_Bot's persisted round counter; robust to a mid-write read +# (partial/empty file → 0 instead of a garbage/arithmetic error). +read_counter() { + local v=0 + [ -f "$ROUND_COUNTER_FILE" ] || { echo 0; return; } + v=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0) + case "$v" in + ''|*[!0-9]*) echo 0 ;; + *) echo "$v" ;; + esac +} + +count_wins() { + local f="$1" + [ -f "$f" ] || { echo 0; return; } + awk '/"win":true/{c++} END{print c+0}' "$f" +} + +# sample_opponent — weighted draw: P(opp) ∝ max(FLOOR, 1 - winrate). +# Cumulative distribution built in awk; random draw via awk's rand() seeded from +# bash $RANDOM. Deterministic when $RANDOM is pre-seeded (the self-test relies on +# this). Pass an explicit seed as $1 to override. +sample_opponent() { + local seed="${1:-$RANDOM}" + awk -v seed="$seed" -v floor="$FLOOR" \ + -v names="${OPPONENTS[*]}" -v rates="${WR[*]}" ' + BEGIN { + srand(seed); + n = split(names, N, " "); + split(rates, R, " "); + total = 0; + for (i = 1; i <= n; i++) { + w = 1.0 - R[i] + 0.0; + if (w < floor) w = floor; + W[i] = w; total += w; + } + r = rand() * total; + acc = 0; + for (i = 1; i <= n; i++) { + acc += W[i]; + if (r < acc) { print N[i]; exit } + } + print N[n]; + }' +} + +# run_chunk [train|eval] — one battle chunk, with the +# 3-attempt retry / loud-abort guard from curriculum_retrain.sh. Prints the +# JSONL path on success; exits the script on a zero-progress chunk. +run_chunk() { + local opp="$1" want="$2" mode="${3:-train}" + local logfile + if [ "$mode" = "eval" ]; then + logfile="/tmp/generalist_eval_${opp}.jsonl" + > "$logfile" # fresh snapshot per eval + export PPOB_EVAL_ONLY=1 # freeze training (pure evaluation) + else + logfile="/tmp/generalist_train_${opp}.jsonl" + unset PPOB_EVAL_ONLY + fi + export TRAINING_OPPONENT="$opp" + export TRAINING_ROUNDS="$want" + export PPOB_LOG_FILE="$logfile" + + # Reset the counter so RunTraining's expected-end check (start+rounds) is exact. + echo 0 > "$ROUND_COUNTER_FILE" + + local remaining="$want" attempts=0 done=0 + while [ "$remaining" -gt 0 ]; do + attempts=$(( attempts + 1 )) + [ "$attempts" -gt 1 ] && sleep 2 + # RunTraining.java exits 1 on its own when the counter is frozen ("process + # dead") or short at battle end ("N rounds never trained ... aborting for + # restart") — treat that as a retry signal, then re-check the counter. + java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$remaining" \ + || echo ">>> Java exited non-zero (attempt $attempts/3) — crash/restart expected" + done=$(read_counter) + remaining=$(( want - done )) + # Zero counter progress across 3 attempts = dead bot / broken harness. + # The bot crashed constantly before; a crash must fail loudly now. + if [ "$attempts" -ge 3 ] && [ "$done" -eq 0 ]; then + echo "ERROR: round_counter still 0 after $attempts attempts vs $opp — aborting for restart" >&2 + exit 1 + fi + done + unset PPOB_EVAL_ONLY + echo "$logfile" +} + +persist_wr() { + local i opp + : > "$WR_STATE_FILE" + for i in "${!OPPONENTS[@]}"; do + printf "%s %.4f\n" "${OPPONENTS[$i]}" "${WR[$i]}" >> "$WR_STATE_FILE" + done +} + +load_wr() { + local src="" + if [ -f "$WR_SEED_FILE" ]; then + src="$WR_SEED_FILE" + echo ">>> Seeding winrates from $WR_SEED_FILE" + elif [ -f "$WR_STATE_FILE" ]; then + src="$WR_STATE_FILE" + echo ">>> Loading winrates from $WR_STATE_FILE" + else + echo ">>> No winrate state — starting all at 1.0 (uniform floor sampling until first eval)" + fi + local -A from_file=() + if [ -n "$src" ]; then + local name val + while read -r name val _; do + [ -n "$name" ] || continue + from_file[$name]="${val:-1.0}" + done < "$src" + fi + local i opp + for i in "${!OPPONENTS[@]}"; do + opp="${OPPONENTS[$i]}" + WR[$i]="${from_file[$opp]:-1.0}" + done +} + +load_best() { + BEST_MEAN=0.0 + BEST_ROUNDS=0 + if [ -f "$BEST_FILE" ]; then + read -r BEST_MEAN BEST_ROUNDS < "$BEST_FILE" || : + [ -n "$BEST_MEAN" ] || BEST_MEAN=0.0 + [ -n "$BEST_ROUNDS" ] || BEST_ROUNDS=0 + echo ">>> Best-so-far: mean=$BEST_MEAN at round $BEST_ROUNDS" + fi +} + +snap_weights() { + local rounds="$1" + local dst="$SNAPSHOTS_DIR/generalist_best_r${rounds}" + cp -a "$PPO_BOT_SRC/weights/latest" "$dst" # fresh dir per save (rounds strictly increase) + echo " saved checkpoint -> $dst" +} + +# do_eval — fresh battle per opponent, PPOB_EVAL_ONLY=1, +# EVAL_ROUNDS each. Updates winrates (eval-only!) and the best generalist +# checkpoint by mean winrate. +do_eval() { + local trained_rounds="$1" + local i opp w logfile oldwr evwr newwr alert + local total_wins=0 + local total_possible=$(( ${#OPPONENTS[@]} * EVAL_ROUNDS )) + local mean + echo "" + echo "=== EVAL @ round $trained_rounds ===" + for i in "${!OPPONENTS[@]}"; do + opp="${OPPONENTS[$i]}" + logfile=$(run_chunk "$opp" "$EVAL_ROUNDS" eval) || { echo " $opp: ERR"; continue; } + w=$(count_wins "$logfile") + total_wins=$(( total_wins + w )) + evwr=$(awk -v w="$w" -v n="$EVAL_ROUNDS" 'BEGIN{printf "%.4f", w/n}') + oldwr="${WR[$i]}" + newwr=$(awk -v a="$ALPHA" -v o="$oldwr" -v e="$evwr" 'BEGIN{printf "%.4f", a*o + (1-a)*e}') + WR[$i]="$newwr" + persist_wr # state survives a crash mid-eval + alert="" + if awk -v e="$evwr" -v p="$PASS_RATE" 'BEGIN{exit !(e < p)}'; then + alert=" <-- BELOW PASS_RATE $PASS_RATE" + fi + echo " $opp: $w/$EVAL_ROUNDS (wr $oldwr -> $newwr)$alert" + done + mean=$(awk -v n="${#OPPONENTS[@]}" -v rates="${WR[*]}" ' + BEGIN { split(rates, R, " "); s = 0; for (i = 1; i <= n; i++) s += R[i]; printf "%.4f", s / n }') + echo " mean winrate: $mean (total $total_wins/$total_possible)" + if awk -v m="$mean" -v b="$BEST_MEAN" 'BEGIN{exit !(m > b)}'; then + BEST_MEAN="$mean" + BEST_ROUNDS="$trained_rounds" + snap_weights "$trained_rounds" + printf '%s %s\n' "$BEST_MEAN" "$BEST_ROUNDS" > "$BEST_FILE" + echo " >>> NEW BEST generalist: mean=$mean at $trained_rounds rounds" + else + echo " best remains: $BEST_MEAN at $BEST_ROUNDS rounds" + fi +} + +# ── self-test ───────────────────────────────────────────────────────────────── +# Mock winrates: MyFirstBot 0.0, PaintingBot 0.1, 6 strong bots 1.0. +# Weights: 1.0, 0.9, and 6x FLOOR 0.05 => total 2.2. +# Expected: MyFirstBot 1.0/2.2=45.5%, PaintingBot 40.9%, each strong 2.27%. +selftest() { + local rc=0 i opp d frac + local -A draws=() + WR=(1.0 1.0 1.0 1.0 1.0 1.0 0.1 0.0) # order matches OPPONENTS + RANDOM=12345 # fixed seed -> reproducible draws + for ((i = 0; i < 10000; i++)); do + opp=$(sample_opponent) + draws[$opp]=$(( ${draws[$opp]:-0} + 1 )) + done + echo "=== sampling self-test: 10000 draws, seed RANDOM=12345 ===" + echo "mock winrates: MyFirstBot 0.0, PaintingBot 0.1, strong x6 1.0" + echo "weights -> MyFirstBot 1.0, PaintingBot 0.9, strong 0.05 each (total 2.2)" + echo "" + echo "actual draw distribution:" + for opp in "${OPPONENTS[@]}"; do + d=${draws[$opp]:-0} + frac=$(awk -v d="$d" 'BEGIN{printf "%.4f", d/10000}') + printf " %-14s %5d draws (share %.4f)\n" "$opp" "$d" "$frac" + done + echo "" + local mfb + mfb=$(awk -v d="${draws[MyFirstBot]:-0}" 'BEGIN{printf "%.4f", d/10000}') + assert_range() { + # assert_range + if awk -v f="$2" -v lo="$3" -v hi="$4" 'BEGIN{exit !(f >= lo && f <= hi)}'; then + echo " OK $1 share $2 in [$3,$4]" + else + echo " FAIL $1 share $2 not in [$3,$4]"; rc=1 + fi + } + assert_range "MyFirstBot (weak focus)" "$mfb" 0.35 0.50 + for opp in Fire MyFirstDroid Target MyFirstLeader Crazy Corners; do + frac=$(awk -v d="${draws[$opp]:-0}" 'BEGIN{printf "%.4f", d/10000}') + assert_range "$opp maintenance" "$frac" 0.015 0.05 + done + echo "" + [ "$rc" -eq 0 ] && echo "SELF-TEST PASS" || echo "SELF-TEST FAIL" + exit "$rc" +} + +# ── main training loop ──────────────────────────────────────────────────────── +main() { + echo "=== PPO_Bot generalist mixed training — $(date) ===" + echo "Config: TOTAL_ROUNDS=$TOTAL_ROUNDS CHUNK_SIZE=$CHUNK_SIZE EVAL_INTERVAL=$EVAL_INTERVAL EVAL_ROUNDS=$EVAL_ROUNDS PASS_RATE=$PASS_RATE FLOOR=$FLOOR ALPHA=$ALPHA" + echo "Opponents: ${OPPONENTS[*]}" + + echo ">>> Compiling PPO_Bot..." + (cd "$PPO_BOT_SRC" && nimble build -d:release) || { echo "ERROR: PPO_Bot build failed" >&2; exit 1; } + echo ">>> Compiling RunTraining.java..." + (cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java) || { echo "ERROR: RunTraining compile failed" >&2; exit 1; } + echo ">>> Compilation done." + + load_best + load_wr + persist_wr # materialize state now so an early crash still leaves a readable file + + # fresh per-run train JSONLs; cross-chunk state lives in the wr/best files + local opp + for opp in "${OPPONENTS[@]}"; do + : > "/tmp/generalist_train_${opp}.jsonl" + done + + local trained_rounds=0 chunk_rounds=0 + while [ "$trained_rounds" -lt "$TOTAL_ROUNDS" ]; do + chunk_rounds=$CHUNK_SIZE + if [ $(( trained_rounds + CHUNK_SIZE )) -gt "$TOTAL_ROUNDS" ]; then + chunk_rounds=$(( TOTAL_ROUNDS - trained_rounds )) + fi + opp=$(sample_opponent) + echo "" + echo "Chunk $trained_rounds: $opp ($chunk_rounds rounds expected)" + run_chunk "$opp" "$chunk_rounds" train || : + trained_rounds=$(( trained_rounds + chunk_rounds )) + + if [ $(( trained_rounds % EVAL_INTERVAL )) -eq 0 ]; then + do_eval "$trained_rounds" + fi + done + + # final eval at the very end (skip when the loop already evaluated at TOTAL_ROUNDS) + if [ $(( TOTAL_ROUNDS % EVAL_INTERVAL )) -ne 0 ]; then + echo "" + echo "=== FINAL eval — $(date) ===" + do_eval "$trained_rounds" + fi + + echo "" + echo "=== generalist training complete — $(date) ===" + echo "Final winrates in $WR_STATE_FILE" + echo "Best generalist checkpoint: $SNAPSHOTS_DIR/generalist_best_r${BEST_ROUNDS} (mean=$BEST_MEAN at $BEST_ROUNDS rounds)" +} + +if [ "${1:-}" = "--selftest" ]; then + selftest 2>&1 | tee "$SELFTEST_LOG" + exit "${PIPESTATUS[0]}" +fi + +main "$@" 2>&1 | tee -a "$LOG" +exit "${PIPESTATUS[0]}" \ No newline at end of file diff --git a/tools/training_runner/mixed_train.sh b/tools/training_runner/mixed_train.sh new file mode 100755 index 0000000..abc621e --- /dev/null +++ b/tools/training_runner/mixed_train.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# mixed_train.sh — mixed-opponent training for generalist PPO_Bot policy. +# Randomly picks an opponent each chunk to prevent catastrophic forgetting. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PPO_BOT_SRC="$REPO_ROOT/PPO_Bot" + +ENV_FILE="$SCRIPT_DIR/training.env" +[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a + +JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}" +export PPO_BOT_DIR="$PPO_BOT_SRC" +export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}" + +SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots" +mkdir -p "$SNAPSHOTS_DIR" + +ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt" + +# ── config ──────────────────────────────────────────────────────────────────── +OPPONENTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot) +TOTAL_ROUNDS=10000 +CHUNK_SIZE=60 +EVAL_INTERVAL=500 +EVAL_ROUNDS=50 +PASS_THRESHOLD=45 + +# ── compile once ────────────────────────────────────────────────────────────── +echo "=== PPO_Bot mixed training — $(date) ===" +echo ">>> Compiling PPO_Bot..." +(cd "$PPO_BOT_SRC" && nimble build -d:release) +echo ">>> Compiling RunTraining.java..." +(cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java) +echo ">>> Compilation done." + +# ── helpers ─────────────────────────────────────────────────────────────────── + +count_wins() { + local f="$1" + [ -f "$f" ] || { echo 0; return; } + awk '/"win":true/{c++} END{print c+0}' "$f" +} + +# run_chunk [eval] +# Sets PPOB_EVAL_ONLY if third arg is "eval". Retries up to 3 times on failure. +run_chunk() { + local opp="$1" rounds="$2" mode="${3:-train}" + local logfile + if [ "$mode" = "eval" ]; then + logfile="/tmp/${opp}_eval.jsonl" + export PPOB_EVAL_ONLY=1 + else + logfile="/tmp/${opp}_mixed_train.jsonl" + unset PPOB_EVAL_ONLY + fi + export TRAINING_OPPONENT="$opp" + export TRAINING_ROUNDS="$rounds" + export PPOB_LOG_FILE="$logfile" + > "$logfile" + + # Reset counter so java knows when it's done + echo 0 > "$ROUND_COUNTER_FILE" + + local attempt=0 + while [ "$attempt" -lt 3 ]; do + attempt=$(( attempt + 1 )) + [ "$attempt" -gt 1 ] && sleep 2 + java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$rounds" && break + echo ">>> Java exited non-zero (attempt $attempt/3)" + done + + if [ "$attempt" -ge 3 ]; then + local done=0 + [ -f "$ROUND_COUNTER_FILE" ] && done=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0) + if [ "$done" -eq 0 ]; then + echo "WARNING: chunk $opp made no progress after 3 attempts — skipping" + unset PPOB_EVAL_ONLY + return 1 + fi + fi + + unset PPOB_EVAL_ONLY + echo "$logfile" +} + +evaluate_all() { + local trained_rounds="$1" + local total_wins=0 + local total_possible=$(( ${#OPPONENTS[@]} * EVAL_ROUNDS )) + local scoreboard="[EVAL @ round $trained_rounds]" + + for opp in "${OPPONENTS[@]}"; do + local logfile + logfile=$(run_chunk "$opp" "$EVAL_ROUNDS" eval) || { scoreboard="$scoreboard ${opp}:ERR"; continue; } + local wins + wins=$(count_wins "$logfile") + total_wins=$(( total_wins + wins )) + scoreboard="$scoreboard ${opp}:${wins}/${EVAL_ROUNDS}" + done + + scoreboard="$scoreboard Total:${total_wins}/${total_possible}" + echo "$scoreboard" + echo "$total_wins" +} + +snap_weights() { + local trained_rounds="$1" + local dst="$SNAPSHOTS_DIR/mixed_r${trained_rounds}" + cp -a "$PPO_BOT_SRC/weights/latest" "$dst" + echo " snapshot → $dst" +} + +random_opponent() { + local idx=$(( RANDOM % ${#OPPONENTS[@]} )) + echo "${OPPONENTS[$idx]}" +} + +# ── training loop ───────────────────────────────────────────────────────────── +trained_rounds=0 +best_total_wins=-1 +n_opponents=${#OPPONENTS[@]} +total_possible=$(( n_opponents * EVAL_ROUNDS )) + +echo "Config: TOTAL_ROUNDS=$TOTAL_ROUNDS CHUNK_SIZE=$CHUNK_SIZE EVAL_INTERVAL=$EVAL_INTERVAL EVAL_ROUNDS=$EVAL_ROUNDS PASS_THRESHOLD=$PASS_THRESHOLD" +echo "Opponents: ${OPPONENTS[*]}" + +while [ "$trained_rounds" -lt "$TOTAL_ROUNDS" ]; do + opp=$(random_opponent) + echo "--- [round $trained_rounds] training vs $opp ($CHUNK_SIZE rounds)" + run_chunk "$opp" "$CHUNK_SIZE" train || true + trained_rounds=$(( trained_rounds + CHUNK_SIZE )) + + if [ $(( trained_rounds % EVAL_INTERVAL )) -eq 0 ]; then + # evaluate_all prints scoreboard then total_wins on separate lines + mapfile -t eval_out < <(evaluate_all "$trained_rounds") + echo "${eval_out[0]}" + total_wins="${eval_out[1]:-0}" + if [ "$total_wins" -gt "$best_total_wins" ]; then + best_total_wins="$total_wins" + snap_weights "$trained_rounds" + fi + fi +done + +echo "" +echo "=== Final evaluation — $(date) ===" +mapfile -t eval_out < <(evaluate_all "$trained_rounds") +echo "${eval_out[0]}" +total_wins="${eval_out[1]:-0}" +if [ "$total_wins" -gt "$best_total_wins" ]; then + snap_weights "$trained_rounds" +fi +echo "=== mixed training complete — $(date) ===" diff --git a/tools/warm_start.py b/tools/warm_start.py new file mode 100644 index 0000000..7cd4172 --- /dev/null +++ b/tools/warm_start.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Warm-start TARGET_DIM weights from old 44-dim trained weights.""" +import numpy as np +from pathlib import Path + +SRC = Path("tools/training_runner/snapshots/best_post_maint_r38003") +DST = Path("PPO_Bot/weights/latest") +OLD_DIM = 44 +TARGET_DIM = 57 # change this to expand to a different input dimension + +# Pad w1 [64, OLD_DIM] -> [64, TARGET_DIM] with zeros (weights and Adam moments) +for prefix in ("actor", "critic"): + # Weight + old = np.load(SRC / f"{prefix}_w1.npy") + assert old.shape == (64, OLD_DIM), f"unexpected shape {old.shape}" + new = np.zeros((64, TARGET_DIM), dtype=old.dtype) + new[:, :OLD_DIM] = old + np.save(DST / f"{prefix}_w1.npy", new) + print(f" {prefix}_w1: {old.shape} -> {new.shape}") + + # Adam moments for w1: pad same way + adam_base = f"adam_{'a' if prefix == 'actor' else 'c'}w1" + for moment in ("_m", "_v"): + old_m = np.load(SRC / f"{adam_base}{moment}.npy") + new_m = np.zeros((64, TARGET_DIM), dtype=old_m.dtype) + new_m[:, :OLD_DIM] = old_m + np.save(DST / f"{adam_base}{moment}.npy", new_m) + print(f" {adam_base}{moment}: {old_m.shape} -> {new_m.shape}") + +# Copy unchanged weight files as-is +unchanged = [ + "actor_w2", "actor_w3", "actor_b1", "actor_b2", "actor_b3", + "critic_w2", "critic_w3", "critic_b1", "critic_b2", "critic_b3", + "log_std", +] +for name in unchanged: + data = np.load(SRC / f"{name}.npy") + np.save(DST / f"{name}.npy", data) + print(f" {name}: {data.shape} copied") + +# Copy unchanged Adam moments (all except w1, which were handled above) +unchanged_adam = [ + "adam_aw2", "adam_cw2", + "adam_aw3", "adam_cw3", + "adam_ab1", "adam_cb1", + "adam_ab2", "adam_cb2", + "adam_ab3", "adam_cb3", + "adam_logstd", +] +for base in unchanged_adam: + for moment in ("_m", "_v"): + data = np.load(SRC / f"{base}{moment}.npy") + np.save(DST / f"{base}{moment}.npy", data) + print(f" {base}{moment}: {data.shape} copied") + +(DST / "adam_t.txt").write_text("1\n") +(DST.parent / "round_counter.txt").write_text("0\n") +print(" adam_t.txt -> 1, round_counter.txt -> 0") + +# Verify +w1 = np.load(DST / "actor_w1.npy") +old_w1 = np.load(SRC / "actor_w1.npy") +assert w1.shape == (64, TARGET_DIM), f"bad shape {w1.shape}" +assert np.allclose(w1[:, :OLD_DIM], old_w1), "old columns don't match" +assert np.all(w1[:, OLD_DIM:] == 0), "new columns not zero" +print(f"\nOK: actor_w1 shape={w1.shape}, cols 0-{OLD_DIM-1} match old, cols {OLD_DIM}-{TARGET_DIM-1} are zero")