feat(PPO_Bot): bot-relative bullets + scan staleness (STATE_DIM=57)
- Bullet state (indices 44-55): enemy-relative → bot-relative frame (bot needs threat vectors to itself for dodging, not to enemy) - New index 56: scan staleness = min(ticksSinceLastScan / 30, 1.0) (gives policy a confidence signal for enemy data freshness) - warm_start.py updated: 44→57 dim expansion, TARGET_DIM variable - Tests updated for new state layout Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# curriculum_retrain.sh — sequential curriculum retraining for PPO_Bot v2.
|
||||
# Bots in order: Fire → MyFirstDroid → Target → MyFirstLeader → Crazy → Corners → PaintingBot → MyFirstBot
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
PPO_BOT_SRC="$REPO_ROOT/PPO_Bot"
|
||||
|
||||
# Source hyperparams (but we'll override opponent/rounds/log per phase)
|
||||
ENV_FILE="$SCRIPT_DIR/training.env"
|
||||
[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}"
|
||||
export PPO_BOT_DIR="$PPO_BOT_SRC"
|
||||
export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}"
|
||||
|
||||
LOGS_DIR="$SCRIPT_DIR/logs"
|
||||
SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots"
|
||||
SUMMARY="$LOGS_DIR/curriculum_retrain_summary.txt"
|
||||
mkdir -p "$LOGS_DIR" "$SNAPSHOTS_DIR"
|
||||
|
||||
ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt"
|
||||
|
||||
echo "=== PPO_Bot v2 curriculum retrain — $(date) ===" | tee -a "$SUMMARY"
|
||||
|
||||
# ── compile once up front (run.sh does this; curriculum skipped it) ───────────
|
||||
echo ">>> Compiling PPO_Bot..."
|
||||
(cd "$PPO_BOT_SRC" && nimble build -d:release)
|
||||
echo ">>> Compiling RunTraining.java..."
|
||||
(cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java)
|
||||
echo ">>> Compilation done."
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
run_java() {
|
||||
# run_java <opponent> <rounds>
|
||||
local opp="$1" rounds="$2"
|
||||
local remaining="$rounds"
|
||||
local attempts=0
|
||||
while [ "$remaining" -gt 0 ]; do
|
||||
attempts=$(( attempts + 1 ))
|
||||
[ "$attempts" -gt 1 ] && sleep 2
|
||||
java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$remaining" || \
|
||||
echo ">>> Java exited non-zero (crash/restart expected)"
|
||||
local done=0
|
||||
[ -f "$ROUND_COUNTER_FILE" ] && done=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0)
|
||||
remaining=$(( rounds - done ))
|
||||
# Guard: if counter didn't advance at all on this attempt, something is
|
||||
# fundamentally broken (bad binary, missing bots dir, etc.) — abort rather
|
||||
# than loop forever.
|
||||
if [ "$attempts" -ge 3 ] && [ "$done" -eq 0 ]; then
|
||||
echo "ERROR: round_counter still 0 after $attempts attempts — aborting" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
count_wins() {
|
||||
# count_wins <jsonl_file> → prints win count
|
||||
# grep -c exits 1 on zero matches (not an error) which was triggering || echo 0
|
||||
# making WINS="0\n0". Use awk which always exits 0.
|
||||
local f="$1"
|
||||
[ -f "$f" ] || { echo 0; return; }
|
||||
awk '/"win":true/{c++} END{print c+0}' "$f"
|
||||
}
|
||||
|
||||
snap_weights() {
|
||||
local label="$1"
|
||||
local counter
|
||||
counter=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0)
|
||||
local dst="$SNAPSHOTS_DIR/v2_${label}_r${counter}"
|
||||
cp -a "$PPO_BOT_SRC/weights/latest" "$dst"
|
||||
echo " snapshot → $dst"
|
||||
}
|
||||
|
||||
# ── curriculum ────────────────────────────────────────────────────────────────
|
||||
|
||||
declare -A TRAIN_ROUNDS=( [Fire]=200 [MyFirstDroid]=200 [Target]=200 [MyFirstLeader]=200 [Crazy]=200
|
||||
[Corners]=400 [PaintingBot]=400 [MyFirstBot]=1000 )
|
||||
BOTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot)
|
||||
GATE=150
|
||||
GATE_PASS=148
|
||||
|
||||
for BOT in "${BOTS[@]}"; do
|
||||
TR="${TRAIN_ROUNDS[$BOT]}"
|
||||
echo "" | tee -a "$SUMMARY"
|
||||
echo "══ $BOT (train=${TR} gate=${GATE}) ══" | tee -a "$SUMMARY"
|
||||
|
||||
# Reset round counter so run_java terminates correctly for each phase
|
||||
echo 0 > "$ROUND_COUNTER_FILE"
|
||||
|
||||
# ── training phase ──────────────────────────────────────────────────────────
|
||||
TRAIN_LOG="$LOGS_DIR/${BOT}_train.jsonl"
|
||||
export TRAINING_OPPONENT="$BOT"
|
||||
export TRAINING_ROUNDS="$TR"
|
||||
export PPOB_LOG_FILE="$TRAIN_LOG"
|
||||
unset PPOB_EVAL_ONLY
|
||||
> "$TRAIN_LOG"
|
||||
echo " [train] $TR rounds..." | tee -a "$SUMMARY"
|
||||
run_java "$BOT" "$TR"
|
||||
echo " [train] done" | tee -a "$SUMMARY"
|
||||
|
||||
# ── gate eval ───────────────────────────────────────────────────────────────
|
||||
echo 0 > "$ROUND_COUNTER_FILE"
|
||||
EVAL_LOG="$LOGS_DIR/${BOT}_eval.jsonl"
|
||||
export TRAINING_ROUNDS="$GATE"
|
||||
export PPOB_LOG_FILE="$EVAL_LOG"
|
||||
export PPOB_EVAL_ONLY=1
|
||||
> "$EVAL_LOG"
|
||||
echo " [eval] $GATE rounds..." | tee -a "$SUMMARY"
|
||||
run_java "$BOT" "$GATE"
|
||||
|
||||
WINS=$(count_wins "$EVAL_LOG")
|
||||
if [ "$WINS" -ge "$GATE_PASS" ]; then
|
||||
STATUS="PASS ($WINS/$GATE)"
|
||||
snap_weights "$BOT"
|
||||
else
|
||||
STATUS="FAIL ($WINS/$GATE)"
|
||||
fi
|
||||
echo " [gate] $STATUS" | tee -a "$SUMMARY"
|
||||
unset PPOB_EVAL_ONLY
|
||||
done
|
||||
|
||||
echo "" | tee -a "$SUMMARY"
|
||||
echo "=== curriculum complete — $(date) ===" | tee -a "$SUMMARY"
|
||||
Executable
+351
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env bash
|
||||
# generalist_train.sh — adaptive mixed-opponent training for a generalist PPO_Bot.
|
||||
#
|
||||
# Problem it solves: the sequential curriculum killed earlier skills (catastrophic
|
||||
# forgetting). Instead we sample ONE opponent per chunk, weighted by how weak the
|
||||
# bot currently is against it (max(FLOOR, 1 - winrate)), and keep the best
|
||||
# generalist checkpoint by mean eval winrate across all 8 opponents.
|
||||
#
|
||||
# State / logs (all plain files, resumable):
|
||||
# /tmp/generalist_wr.txt — one line per opponent: "Name winrate"
|
||||
# /tmp/generalist_seed.txt — optional starting winrates (same format);
|
||||
# wins over wr.txt when both exist
|
||||
# /tmp/generalist_best.txt — "mean rounds" of the best generalist eval so far
|
||||
# /tmp/generalist_train.log — full run log (tee'd)
|
||||
# /tmp/generalist_eval_<Opp>.jsonl — per-opponent eval battles (fresh per eval)
|
||||
# /tmp/generalist_train_<Opp>.jsonl — per-opponent training battles (accumulated)
|
||||
#
|
||||
# Env-overridable config: TOTAL_ROUNDS CHUNK_SIZE EVAL_INTERVAL EVAL_ROUNDS
|
||||
# PASS_RATE FLOOR ALPHA (all declared in the config block below).
|
||||
#
|
||||
# Usage:
|
||||
# ./generalist_train.sh — run the full training loop
|
||||
# ./generalist_train.sh --selftest — verify the weighted sampler against the
|
||||
# expected distribution (no battles, no compilation)
|
||||
#
|
||||
# Dead-bot resilience is inherited from the harness: RunTraining.java aborts
|
||||
# (exit 1) when PPO_Bot's round counter is frozen or short at battle end, and
|
||||
# this script retries; a chunk that makes no counter progress across 3 attempts
|
||||
# aborts loudly instead of silently burning rounds.
|
||||
set -uo pipefail
|
||||
export LC_ALL=C
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
PPO_BOT_SRC="$REPO_ROOT/PPO_Bot"
|
||||
|
||||
ENV_FILE="$SCRIPT_DIR/training.env"
|
||||
[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}"
|
||||
export PPO_BOT_DIR="$PPO_BOT_SRC"
|
||||
export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}"
|
||||
|
||||
SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots"
|
||||
mkdir -p "$SNAPSHOTS_DIR"
|
||||
|
||||
ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt"
|
||||
LOG="/tmp/generalist_train.log"
|
||||
WR_STATE_FILE="/tmp/generalist_wr.txt"
|
||||
WR_SEED_FILE="/tmp/generalist_seed.txt"
|
||||
BEST_FILE="/tmp/generalist_best.txt"
|
||||
SELFTEST_LOG="/tmp/generalist_sampling_selftest.log"
|
||||
|
||||
# ── config (env-overridable) ──────────────────────────────────────────────────
|
||||
TOTAL_ROUNDS="${TOTAL_ROUNDS:-40000}"
|
||||
CHUNK_SIZE="${CHUNK_SIZE:-60}"
|
||||
EVAL_INTERVAL="${EVAL_INTERVAL:-1000}"
|
||||
EVAL_ROUNDS="${EVAL_ROUNDS:-30}"
|
||||
PASS_RATE="${PASS_RATE:-0.90}"
|
||||
FLOOR="${FLOOR:-0.05}" # minimum sampling weight — keeps maintenance sampling alive
|
||||
ALPHA="${ALPHA:-0.3}" # winrate EMA smoothing: new = ALPHA*old + (1-ALPHA)*eval
|
||||
|
||||
OPPONENTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot)
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
# read_counter — PPO_Bot's persisted round counter; robust to a mid-write read
|
||||
# (partial/empty file → 0 instead of a garbage/arithmetic error).
|
||||
read_counter() {
|
||||
local v=0
|
||||
[ -f "$ROUND_COUNTER_FILE" ] || { echo 0; return; }
|
||||
v=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0)
|
||||
case "$v" in
|
||||
''|*[!0-9]*) echo 0 ;;
|
||||
*) echo "$v" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
count_wins() {
|
||||
local f="$1"
|
||||
[ -f "$f" ] || { echo 0; return; }
|
||||
awk '/"win":true/{c++} END{print c+0}' "$f"
|
||||
}
|
||||
|
||||
# sample_opponent — weighted draw: P(opp) ∝ max(FLOOR, 1 - winrate).
|
||||
# Cumulative distribution built in awk; random draw via awk's rand() seeded from
|
||||
# bash $RANDOM. Deterministic when $RANDOM is pre-seeded (the self-test relies on
|
||||
# this). Pass an explicit seed as $1 to override.
|
||||
sample_opponent() {
|
||||
local seed="${1:-$RANDOM}"
|
||||
awk -v seed="$seed" -v floor="$FLOOR" \
|
||||
-v names="${OPPONENTS[*]}" -v rates="${WR[*]}" '
|
||||
BEGIN {
|
||||
srand(seed);
|
||||
n = split(names, N, " ");
|
||||
split(rates, R, " ");
|
||||
total = 0;
|
||||
for (i = 1; i <= n; i++) {
|
||||
w = 1.0 - R[i] + 0.0;
|
||||
if (w < floor) w = floor;
|
||||
W[i] = w; total += w;
|
||||
}
|
||||
r = rand() * total;
|
||||
acc = 0;
|
||||
for (i = 1; i <= n; i++) {
|
||||
acc += W[i];
|
||||
if (r < acc) { print N[i]; exit }
|
||||
}
|
||||
print N[n];
|
||||
}'
|
||||
}
|
||||
|
||||
# run_chunk <opponent> <rounds> [train|eval] — one battle chunk, with the
|
||||
# 3-attempt retry / loud-abort guard from curriculum_retrain.sh. Prints the
|
||||
# JSONL path on success; exits the script on a zero-progress chunk.
|
||||
run_chunk() {
|
||||
local opp="$1" want="$2" mode="${3:-train}"
|
||||
local logfile
|
||||
if [ "$mode" = "eval" ]; then
|
||||
logfile="/tmp/generalist_eval_${opp}.jsonl"
|
||||
> "$logfile" # fresh snapshot per eval
|
||||
export PPOB_EVAL_ONLY=1 # freeze training (pure evaluation)
|
||||
else
|
||||
logfile="/tmp/generalist_train_${opp}.jsonl"
|
||||
unset PPOB_EVAL_ONLY
|
||||
fi
|
||||
export TRAINING_OPPONENT="$opp"
|
||||
export TRAINING_ROUNDS="$want"
|
||||
export PPOB_LOG_FILE="$logfile"
|
||||
|
||||
# Reset the counter so RunTraining's expected-end check (start+rounds) is exact.
|
||||
echo 0 > "$ROUND_COUNTER_FILE"
|
||||
|
||||
local remaining="$want" attempts=0 done=0
|
||||
while [ "$remaining" -gt 0 ]; do
|
||||
attempts=$(( attempts + 1 ))
|
||||
[ "$attempts" -gt 1 ] && sleep 2
|
||||
# RunTraining.java exits 1 on its own when the counter is frozen ("process
|
||||
# dead") or short at battle end ("N rounds never trained ... aborting for
|
||||
# restart") — treat that as a retry signal, then re-check the counter.
|
||||
java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$remaining" \
|
||||
|| echo ">>> Java exited non-zero (attempt $attempts/3) — crash/restart expected"
|
||||
done=$(read_counter)
|
||||
remaining=$(( want - done ))
|
||||
# Zero counter progress across 3 attempts = dead bot / broken harness.
|
||||
# The bot crashed constantly before; a crash must fail loudly now.
|
||||
if [ "$attempts" -ge 3 ] && [ "$done" -eq 0 ]; then
|
||||
echo "ERROR: round_counter still 0 after $attempts attempts vs $opp — aborting for restart" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset PPOB_EVAL_ONLY
|
||||
echo "$logfile"
|
||||
}
|
||||
|
||||
persist_wr() {
|
||||
local i opp
|
||||
: > "$WR_STATE_FILE"
|
||||
for i in "${!OPPONENTS[@]}"; do
|
||||
printf "%s %.4f\n" "${OPPONENTS[$i]}" "${WR[$i]}" >> "$WR_STATE_FILE"
|
||||
done
|
||||
}
|
||||
|
||||
load_wr() {
|
||||
local src=""
|
||||
if [ -f "$WR_SEED_FILE" ]; then
|
||||
src="$WR_SEED_FILE"
|
||||
echo ">>> Seeding winrates from $WR_SEED_FILE"
|
||||
elif [ -f "$WR_STATE_FILE" ]; then
|
||||
src="$WR_STATE_FILE"
|
||||
echo ">>> Loading winrates from $WR_STATE_FILE"
|
||||
else
|
||||
echo ">>> No winrate state — starting all at 1.0 (uniform floor sampling until first eval)"
|
||||
fi
|
||||
local -A from_file=()
|
||||
if [ -n "$src" ]; then
|
||||
local name val
|
||||
while read -r name val _; do
|
||||
[ -n "$name" ] || continue
|
||||
from_file[$name]="${val:-1.0}"
|
||||
done < "$src"
|
||||
fi
|
||||
local i opp
|
||||
for i in "${!OPPONENTS[@]}"; do
|
||||
opp="${OPPONENTS[$i]}"
|
||||
WR[$i]="${from_file[$opp]:-1.0}"
|
||||
done
|
||||
}
|
||||
|
||||
load_best() {
|
||||
BEST_MEAN=0.0
|
||||
BEST_ROUNDS=0
|
||||
if [ -f "$BEST_FILE" ]; then
|
||||
read -r BEST_MEAN BEST_ROUNDS < "$BEST_FILE" || :
|
||||
[ -n "$BEST_MEAN" ] || BEST_MEAN=0.0
|
||||
[ -n "$BEST_ROUNDS" ] || BEST_ROUNDS=0
|
||||
echo ">>> Best-so-far: mean=$BEST_MEAN at round $BEST_ROUNDS"
|
||||
fi
|
||||
}
|
||||
|
||||
snap_weights() {
|
||||
local rounds="$1"
|
||||
local dst="$SNAPSHOTS_DIR/generalist_best_r${rounds}"
|
||||
cp -a "$PPO_BOT_SRC/weights/latest" "$dst" # fresh dir per save (rounds strictly increase)
|
||||
echo " saved checkpoint -> $dst"
|
||||
}
|
||||
|
||||
# do_eval <trained_rounds> — fresh battle per opponent, PPOB_EVAL_ONLY=1,
|
||||
# EVAL_ROUNDS each. Updates winrates (eval-only!) and the best generalist
|
||||
# checkpoint by mean winrate.
|
||||
do_eval() {
|
||||
local trained_rounds="$1"
|
||||
local i opp w logfile oldwr evwr newwr alert
|
||||
local total_wins=0
|
||||
local total_possible=$(( ${#OPPONENTS[@]} * EVAL_ROUNDS ))
|
||||
local mean
|
||||
echo ""
|
||||
echo "=== EVAL @ round $trained_rounds ==="
|
||||
for i in "${!OPPONENTS[@]}"; do
|
||||
opp="${OPPONENTS[$i]}"
|
||||
logfile=$(run_chunk "$opp" "$EVAL_ROUNDS" eval) || { echo " $opp: ERR"; continue; }
|
||||
w=$(count_wins "$logfile")
|
||||
total_wins=$(( total_wins + w ))
|
||||
evwr=$(awk -v w="$w" -v n="$EVAL_ROUNDS" 'BEGIN{printf "%.4f", w/n}')
|
||||
oldwr="${WR[$i]}"
|
||||
newwr=$(awk -v a="$ALPHA" -v o="$oldwr" -v e="$evwr" 'BEGIN{printf "%.4f", a*o + (1-a)*e}')
|
||||
WR[$i]="$newwr"
|
||||
persist_wr # state survives a crash mid-eval
|
||||
alert=""
|
||||
if awk -v e="$evwr" -v p="$PASS_RATE" 'BEGIN{exit !(e < p)}'; then
|
||||
alert=" <-- BELOW PASS_RATE $PASS_RATE"
|
||||
fi
|
||||
echo " $opp: $w/$EVAL_ROUNDS (wr $oldwr -> $newwr)$alert"
|
||||
done
|
||||
mean=$(awk -v n="${#OPPONENTS[@]}" -v rates="${WR[*]}" '
|
||||
BEGIN { split(rates, R, " "); s = 0; for (i = 1; i <= n; i++) s += R[i]; printf "%.4f", s / n }')
|
||||
echo " mean winrate: $mean (total $total_wins/$total_possible)"
|
||||
if awk -v m="$mean" -v b="$BEST_MEAN" 'BEGIN{exit !(m > b)}'; then
|
||||
BEST_MEAN="$mean"
|
||||
BEST_ROUNDS="$trained_rounds"
|
||||
snap_weights "$trained_rounds"
|
||||
printf '%s %s\n' "$BEST_MEAN" "$BEST_ROUNDS" > "$BEST_FILE"
|
||||
echo " >>> NEW BEST generalist: mean=$mean at $trained_rounds rounds"
|
||||
else
|
||||
echo " best remains: $BEST_MEAN at $BEST_ROUNDS rounds"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── self-test ─────────────────────────────────────────────────────────────────
|
||||
# Mock winrates: MyFirstBot 0.0, PaintingBot 0.1, 6 strong bots 1.0.
|
||||
# Weights: 1.0, 0.9, and 6x FLOOR 0.05 => total 2.2.
|
||||
# Expected: MyFirstBot 1.0/2.2=45.5%, PaintingBot 40.9%, each strong 2.27%.
|
||||
selftest() {
|
||||
local rc=0 i opp d frac
|
||||
local -A draws=()
|
||||
WR=(1.0 1.0 1.0 1.0 1.0 1.0 0.1 0.0) # order matches OPPONENTS
|
||||
RANDOM=12345 # fixed seed -> reproducible draws
|
||||
for ((i = 0; i < 10000; i++)); do
|
||||
opp=$(sample_opponent)
|
||||
draws[$opp]=$(( ${draws[$opp]:-0} + 1 ))
|
||||
done
|
||||
echo "=== sampling self-test: 10000 draws, seed RANDOM=12345 ==="
|
||||
echo "mock winrates: MyFirstBot 0.0, PaintingBot 0.1, strong x6 1.0"
|
||||
echo "weights -> MyFirstBot 1.0, PaintingBot 0.9, strong 0.05 each (total 2.2)"
|
||||
echo ""
|
||||
echo "actual draw distribution:"
|
||||
for opp in "${OPPONENTS[@]}"; do
|
||||
d=${draws[$opp]:-0}
|
||||
frac=$(awk -v d="$d" 'BEGIN{printf "%.4f", d/10000}')
|
||||
printf " %-14s %5d draws (share %.4f)\n" "$opp" "$d" "$frac"
|
||||
done
|
||||
echo ""
|
||||
local mfb
|
||||
mfb=$(awk -v d="${draws[MyFirstBot]:-0}" 'BEGIN{printf "%.4f", d/10000}')
|
||||
assert_range() {
|
||||
# assert_range <name> <frac> <lo> <hi>
|
||||
if awk -v f="$2" -v lo="$3" -v hi="$4" 'BEGIN{exit !(f >= lo && f <= hi)}'; then
|
||||
echo " OK $1 share $2 in [$3,$4]"
|
||||
else
|
||||
echo " FAIL $1 share $2 not in [$3,$4]"; rc=1
|
||||
fi
|
||||
}
|
||||
assert_range "MyFirstBot (weak focus)" "$mfb" 0.35 0.50
|
||||
for opp in Fire MyFirstDroid Target MyFirstLeader Crazy Corners; do
|
||||
frac=$(awk -v d="${draws[$opp]:-0}" 'BEGIN{printf "%.4f", d/10000}')
|
||||
assert_range "$opp maintenance" "$frac" 0.015 0.05
|
||||
done
|
||||
echo ""
|
||||
[ "$rc" -eq 0 ] && echo "SELF-TEST PASS" || echo "SELF-TEST FAIL"
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
# ── main training loop ────────────────────────────────────────────────────────
|
||||
main() {
|
||||
echo "=== PPO_Bot generalist mixed training — $(date) ==="
|
||||
echo "Config: TOTAL_ROUNDS=$TOTAL_ROUNDS CHUNK_SIZE=$CHUNK_SIZE EVAL_INTERVAL=$EVAL_INTERVAL EVAL_ROUNDS=$EVAL_ROUNDS PASS_RATE=$PASS_RATE FLOOR=$FLOOR ALPHA=$ALPHA"
|
||||
echo "Opponents: ${OPPONENTS[*]}"
|
||||
|
||||
echo ">>> Compiling PPO_Bot..."
|
||||
(cd "$PPO_BOT_SRC" && nimble build -d:release) || { echo "ERROR: PPO_Bot build failed" >&2; exit 1; }
|
||||
echo ">>> Compiling RunTraining.java..."
|
||||
(cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java) || { echo "ERROR: RunTraining compile failed" >&2; exit 1; }
|
||||
echo ">>> Compilation done."
|
||||
|
||||
load_best
|
||||
load_wr
|
||||
persist_wr # materialize state now so an early crash still leaves a readable file
|
||||
|
||||
# fresh per-run train JSONLs; cross-chunk state lives in the wr/best files
|
||||
local opp
|
||||
for opp in "${OPPONENTS[@]}"; do
|
||||
: > "/tmp/generalist_train_${opp}.jsonl"
|
||||
done
|
||||
|
||||
local trained_rounds=0 chunk_rounds=0
|
||||
while [ "$trained_rounds" -lt "$TOTAL_ROUNDS" ]; do
|
||||
chunk_rounds=$CHUNK_SIZE
|
||||
if [ $(( trained_rounds + CHUNK_SIZE )) -gt "$TOTAL_ROUNDS" ]; then
|
||||
chunk_rounds=$(( TOTAL_ROUNDS - trained_rounds ))
|
||||
fi
|
||||
opp=$(sample_opponent)
|
||||
echo ""
|
||||
echo "Chunk $trained_rounds: $opp ($chunk_rounds rounds expected)"
|
||||
run_chunk "$opp" "$chunk_rounds" train || :
|
||||
trained_rounds=$(( trained_rounds + chunk_rounds ))
|
||||
|
||||
if [ $(( trained_rounds % EVAL_INTERVAL )) -eq 0 ]; then
|
||||
do_eval "$trained_rounds"
|
||||
fi
|
||||
done
|
||||
|
||||
# final eval at the very end (skip when the loop already evaluated at TOTAL_ROUNDS)
|
||||
if [ $(( TOTAL_ROUNDS % EVAL_INTERVAL )) -ne 0 ]; then
|
||||
echo ""
|
||||
echo "=== FINAL eval — $(date) ==="
|
||||
do_eval "$trained_rounds"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== generalist training complete — $(date) ==="
|
||||
echo "Final winrates in $WR_STATE_FILE"
|
||||
echo "Best generalist checkpoint: $SNAPSHOTS_DIR/generalist_best_r${BEST_ROUNDS} (mean=$BEST_MEAN at $BEST_ROUNDS rounds)"
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--selftest" ]; then
|
||||
selftest 2>&1 | tee "$SELFTEST_LOG"
|
||||
exit "${PIPESTATUS[0]}"
|
||||
fi
|
||||
|
||||
main "$@" 2>&1 | tee -a "$LOG"
|
||||
exit "${PIPESTATUS[0]}"
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
# mixed_train.sh — mixed-opponent training for generalist PPO_Bot policy.
|
||||
# Randomly picks an opponent each chunk to prevent catastrophic forgetting.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
PPO_BOT_SRC="$REPO_ROOT/PPO_Bot"
|
||||
|
||||
ENV_FILE="$SCRIPT_DIR/training.env"
|
||||
[ -f "$ENV_FILE" ] && set -a && . "$ENV_FILE" && set +a
|
||||
|
||||
JAR="${TANK_ROYALE_JAR:-/home/davide/Projects/tank-royale/runner/examples/lib/robocode-tankroyale-runner.jar}"
|
||||
export PPO_BOT_DIR="$PPO_BOT_SRC"
|
||||
export SAMPLE_BOTS_DIR="${SAMPLE_BOTS_DIR:-/home/davide/Projects/tank-royale/sample-bots/java/build/archive}"
|
||||
|
||||
SNAPSHOTS_DIR="$SCRIPT_DIR/snapshots"
|
||||
mkdir -p "$SNAPSHOTS_DIR"
|
||||
|
||||
ROUND_COUNTER_FILE="$PPO_BOT_SRC/weights/round_counter.txt"
|
||||
|
||||
# ── config ────────────────────────────────────────────────────────────────────
|
||||
OPPONENTS=(Fire MyFirstDroid Target MyFirstLeader Crazy Corners PaintingBot MyFirstBot)
|
||||
TOTAL_ROUNDS=10000
|
||||
CHUNK_SIZE=60
|
||||
EVAL_INTERVAL=500
|
||||
EVAL_ROUNDS=50
|
||||
PASS_THRESHOLD=45
|
||||
|
||||
# ── compile once ──────────────────────────────────────────────────────────────
|
||||
echo "=== PPO_Bot mixed training — $(date) ==="
|
||||
echo ">>> Compiling PPO_Bot..."
|
||||
(cd "$PPO_BOT_SRC" && nimble build -d:release)
|
||||
echo ">>> Compiling RunTraining.java..."
|
||||
(cd "$SCRIPT_DIR" && javac -cp "$JAR" RunTraining.java)
|
||||
echo ">>> Compilation done."
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
count_wins() {
|
||||
local f="$1"
|
||||
[ -f "$f" ] || { echo 0; return; }
|
||||
awk '/"win":true/{c++} END{print c+0}' "$f"
|
||||
}
|
||||
|
||||
# run_chunk <opponent> <rounds> [eval]
|
||||
# Sets PPOB_EVAL_ONLY if third arg is "eval". Retries up to 3 times on failure.
|
||||
run_chunk() {
|
||||
local opp="$1" rounds="$2" mode="${3:-train}"
|
||||
local logfile
|
||||
if [ "$mode" = "eval" ]; then
|
||||
logfile="/tmp/${opp}_eval.jsonl"
|
||||
export PPOB_EVAL_ONLY=1
|
||||
else
|
||||
logfile="/tmp/${opp}_mixed_train.jsonl"
|
||||
unset PPOB_EVAL_ONLY
|
||||
fi
|
||||
export TRAINING_OPPONENT="$opp"
|
||||
export TRAINING_ROUNDS="$rounds"
|
||||
export PPOB_LOG_FILE="$logfile"
|
||||
> "$logfile"
|
||||
|
||||
# Reset counter so java knows when it's done
|
||||
echo 0 > "$ROUND_COUNTER_FILE"
|
||||
|
||||
local attempt=0
|
||||
while [ "$attempt" -lt 3 ]; do
|
||||
attempt=$(( attempt + 1 ))
|
||||
[ "$attempt" -gt 1 ] && sleep 2
|
||||
java -cp "$SCRIPT_DIR:$JAR" RunTraining "$opp" "$rounds" && break
|
||||
echo ">>> Java exited non-zero (attempt $attempt/3)"
|
||||
done
|
||||
|
||||
if [ "$attempt" -ge 3 ]; then
|
||||
local done=0
|
||||
[ -f "$ROUND_COUNTER_FILE" ] && done=$(cat "$ROUND_COUNTER_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$done" -eq 0 ]; then
|
||||
echo "WARNING: chunk $opp made no progress after 3 attempts — skipping"
|
||||
unset PPOB_EVAL_ONLY
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
unset PPOB_EVAL_ONLY
|
||||
echo "$logfile"
|
||||
}
|
||||
|
||||
evaluate_all() {
|
||||
local trained_rounds="$1"
|
||||
local total_wins=0
|
||||
local total_possible=$(( ${#OPPONENTS[@]} * EVAL_ROUNDS ))
|
||||
local scoreboard="[EVAL @ round $trained_rounds]"
|
||||
|
||||
for opp in "${OPPONENTS[@]}"; do
|
||||
local logfile
|
||||
logfile=$(run_chunk "$opp" "$EVAL_ROUNDS" eval) || { scoreboard="$scoreboard ${opp}:ERR"; continue; }
|
||||
local wins
|
||||
wins=$(count_wins "$logfile")
|
||||
total_wins=$(( total_wins + wins ))
|
||||
scoreboard="$scoreboard ${opp}:${wins}/${EVAL_ROUNDS}"
|
||||
done
|
||||
|
||||
scoreboard="$scoreboard Total:${total_wins}/${total_possible}"
|
||||
echo "$scoreboard"
|
||||
echo "$total_wins"
|
||||
}
|
||||
|
||||
snap_weights() {
|
||||
local trained_rounds="$1"
|
||||
local dst="$SNAPSHOTS_DIR/mixed_r${trained_rounds}"
|
||||
cp -a "$PPO_BOT_SRC/weights/latest" "$dst"
|
||||
echo " snapshot → $dst"
|
||||
}
|
||||
|
||||
random_opponent() {
|
||||
local idx=$(( RANDOM % ${#OPPONENTS[@]} ))
|
||||
echo "${OPPONENTS[$idx]}"
|
||||
}
|
||||
|
||||
# ── training loop ─────────────────────────────────────────────────────────────
|
||||
trained_rounds=0
|
||||
best_total_wins=-1
|
||||
n_opponents=${#OPPONENTS[@]}
|
||||
total_possible=$(( n_opponents * EVAL_ROUNDS ))
|
||||
|
||||
echo "Config: TOTAL_ROUNDS=$TOTAL_ROUNDS CHUNK_SIZE=$CHUNK_SIZE EVAL_INTERVAL=$EVAL_INTERVAL EVAL_ROUNDS=$EVAL_ROUNDS PASS_THRESHOLD=$PASS_THRESHOLD"
|
||||
echo "Opponents: ${OPPONENTS[*]}"
|
||||
|
||||
while [ "$trained_rounds" -lt "$TOTAL_ROUNDS" ]; do
|
||||
opp=$(random_opponent)
|
||||
echo "--- [round $trained_rounds] training vs $opp ($CHUNK_SIZE rounds)"
|
||||
run_chunk "$opp" "$CHUNK_SIZE" train || true
|
||||
trained_rounds=$(( trained_rounds + CHUNK_SIZE ))
|
||||
|
||||
if [ $(( trained_rounds % EVAL_INTERVAL )) -eq 0 ]; then
|
||||
# evaluate_all prints scoreboard then total_wins on separate lines
|
||||
mapfile -t eval_out < <(evaluate_all "$trained_rounds")
|
||||
echo "${eval_out[0]}"
|
||||
total_wins="${eval_out[1]:-0}"
|
||||
if [ "$total_wins" -gt "$best_total_wins" ]; then
|
||||
best_total_wins="$total_wins"
|
||||
snap_weights "$trained_rounds"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Final evaluation — $(date) ==="
|
||||
mapfile -t eval_out < <(evaluate_all "$trained_rounds")
|
||||
echo "${eval_out[0]}"
|
||||
total_wins="${eval_out[1]:-0}"
|
||||
if [ "$total_wins" -gt "$best_total_wins" ]; then
|
||||
snap_weights "$trained_rounds"
|
||||
fi
|
||||
echo "=== mixed training complete — $(date) ==="
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Warm-start TARGET_DIM weights from old 44-dim trained weights."""
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path("tools/training_runner/snapshots/best_post_maint_r38003")
|
||||
DST = Path("PPO_Bot/weights/latest")
|
||||
OLD_DIM = 44
|
||||
TARGET_DIM = 57 # change this to expand to a different input dimension
|
||||
|
||||
# Pad w1 [64, OLD_DIM] -> [64, TARGET_DIM] with zeros (weights and Adam moments)
|
||||
for prefix in ("actor", "critic"):
|
||||
# Weight
|
||||
old = np.load(SRC / f"{prefix}_w1.npy")
|
||||
assert old.shape == (64, OLD_DIM), f"unexpected shape {old.shape}"
|
||||
new = np.zeros((64, TARGET_DIM), dtype=old.dtype)
|
||||
new[:, :OLD_DIM] = old
|
||||
np.save(DST / f"{prefix}_w1.npy", new)
|
||||
print(f" {prefix}_w1: {old.shape} -> {new.shape}")
|
||||
|
||||
# Adam moments for w1: pad same way
|
||||
adam_base = f"adam_{'a' if prefix == 'actor' else 'c'}w1"
|
||||
for moment in ("_m", "_v"):
|
||||
old_m = np.load(SRC / f"{adam_base}{moment}.npy")
|
||||
new_m = np.zeros((64, TARGET_DIM), dtype=old_m.dtype)
|
||||
new_m[:, :OLD_DIM] = old_m
|
||||
np.save(DST / f"{adam_base}{moment}.npy", new_m)
|
||||
print(f" {adam_base}{moment}: {old_m.shape} -> {new_m.shape}")
|
||||
|
||||
# Copy unchanged weight files as-is
|
||||
unchanged = [
|
||||
"actor_w2", "actor_w3", "actor_b1", "actor_b2", "actor_b3",
|
||||
"critic_w2", "critic_w3", "critic_b1", "critic_b2", "critic_b3",
|
||||
"log_std",
|
||||
]
|
||||
for name in unchanged:
|
||||
data = np.load(SRC / f"{name}.npy")
|
||||
np.save(DST / f"{name}.npy", data)
|
||||
print(f" {name}: {data.shape} copied")
|
||||
|
||||
# Copy unchanged Adam moments (all except w1, which were handled above)
|
||||
unchanged_adam = [
|
||||
"adam_aw2", "adam_cw2",
|
||||
"adam_aw3", "adam_cw3",
|
||||
"adam_ab1", "adam_cb1",
|
||||
"adam_ab2", "adam_cb2",
|
||||
"adam_ab3", "adam_cb3",
|
||||
"adam_logstd",
|
||||
]
|
||||
for base in unchanged_adam:
|
||||
for moment in ("_m", "_v"):
|
||||
data = np.load(SRC / f"{base}{moment}.npy")
|
||||
np.save(DST / f"{base}{moment}.npy", data)
|
||||
print(f" {base}{moment}: {data.shape} copied")
|
||||
|
||||
(DST / "adam_t.txt").write_text("1\n")
|
||||
(DST.parent / "round_counter.txt").write_text("0\n")
|
||||
print(" adam_t.txt -> 1, round_counter.txt -> 0")
|
||||
|
||||
# Verify
|
||||
w1 = np.load(DST / "actor_w1.npy")
|
||||
old_w1 = np.load(SRC / "actor_w1.npy")
|
||||
assert w1.shape == (64, TARGET_DIM), f"bad shape {w1.shape}"
|
||||
assert np.allclose(w1[:, :OLD_DIM], old_w1), "old columns don't match"
|
||||
assert np.all(w1[:, OLD_DIM:] == 0), "new columns not zero"
|
||||
print(f"\nOK: actor_w1 shape={w1.shape}, cols 0-{OLD_DIM-1} match old, cols {OLD_DIM}-{TARGET_DIM-1} are zero")
|
||||
Reference in New Issue
Block a user