diff --git a/SAC_LSTM_Bot/sac_train.sh b/SAC_LSTM_Bot/sac_train.sh index 704b672..83f7331 100755 --- a/SAC_LSTM_Bot/sac_train.sh +++ b/SAC_LSTM_Bot/sac_train.sh @@ -5,7 +5,8 @@ # owns server lifecycle, opponent connection and dead-bot liveness detection # through weights/round_counter.txt), samples opponents by weight per chunk, # runs deterministic evaluation (SACLSTM_EVAL_MODE=1) every N chunks, and keeps -# the best checkpoint (weights/sac_best.zip) by eval win rate. +# the best checkpoint (weights/sac_best.zip) by a moving-average composite over +# the eval opponent set (campaign v2 lever 1, #59). # # Config (env vars): # SAC_OPPONENTS "Name:weight,Name:weight,..." (default below) @@ -13,7 +14,9 @@ # SAC_CHUNK_SIZE rounds per RunTraining battle (default 10) # SAC_EVAL_INTERVAL eval every N chunks (default 2) # SAC_EVAL_ROUNDS rounds per evaluation battle (default 10) -# SAC_EVAL_OPPONENT fixed eval opponent (default first opponent) +# SAC_EVAL_OPPONENTS comma-separated eval set (default Corners,Crazy,Target) +# — each cycle evaluates EVERY one; results all land in +# eval_log.jsonl (lines carry "opponent":"Name") # SAC_MAX_CRASHES consecutive crashes before abort (default 5) # SAC_LOG_FILE / SAC_EVAL_LOG_FILE (JSON-lines logs) # SACLSTM_* passed through to the bot (UTD_RATIO, BATCH_SIZE, ...) @@ -38,7 +41,8 @@ TOTAL_ROUNDS="${SAC_TOTAL_ROUNDS:-100}" CHUNK_SIZE="${SAC_CHUNK_SIZE:-10}" EVAL_INTERVAL="${SAC_EVAL_INTERVAL:-2}" EVAL_ROUNDS="${SAC_EVAL_ROUNDS:-10}" -EVAL_OPPONENT="${SAC_EVAL_OPPONENT:-${OPPONENTS%%:*}}" +EVAL_OPPONENTS="${SAC_EVAL_OPPONENTS:-Corners,Crazy,Target}" +MA_WINDOW=5 # lever 1 (#59): per-opponent moving average over last N evals MAX_CRASHES="${SAC_MAX_CRASHES:-5}" LOG_FILE="${SAC_LOG_FILE:-$SCRIPT_DIR/training_log.jsonl}" EVAL_LOG_FILE="${SAC_EVAL_LOG_FILE:-$SCRIPT_DIR/eval_log.jsonl}" @@ -46,7 +50,7 @@ CLASSES_DIR="/tmp/opencode/sac_train_classes" echo "=== SAC_LSTM_Bot training harness ===" echo "Opponents: $OPPONENTS | budget: $TOTAL_ROUNDS rounds in chunks of $CHUNK_SIZE" -echo "Eval: every $EVAL_INTERVAL chunks, $EVAL_ROUNDS rounds vs $EVAL_OPPONENT" +echo "Eval: every $EVAL_INTERVAL chunks, $EVAL_ROUNDS rounds vs [$EVAL_OPPONENTS], MA-$MA_WINDOW composite best-gating" echo "Weights: $SACLSTM_WEIGHTS_PATH" # ── compile bot + java runner ───────────────────────────────────────────────── @@ -76,29 +80,60 @@ run_battle() { # $1=opponent $2=rounds $3=log file PPOB_LOG_FILE="$3" java -cp "$CLASSES_DIR:$JAR" RunTraining "$1" "$2" } +# ── Lever 1 (#59): eval rotation + MA best-gating ───────────────────────────── +# best_score.txt FORMAT CHANGE: it used to store the single-opponent integer +# win rate (%); that semantics is retired. It now stores the COMPOSITE score — +# the mean over SAC_EVAL_OPPONENTS of each opponent's moving average (last +# MA_WINDOW eval win rates, %). sac_best.zip is rewritten only when the +# composite strictly improves. +ma_hist_file() { echo "$WEIGHTS_DIR/ma_history_$1.txt"; } + +composite_of() { # reads one "w w w ..." history line per opponent on stdin + awk -v W="$MA_WINDOW" ' + NF > 0 { n=NF; k=(n>W)?W:n; s=0; for(j=n-k+1;j<=n;j++) s+=$j; tot+=s/k; c++ } + END { if (c>0) printf "%.4f", tot/c; else print "-1" }' +} + eval_checkpoint() { - local tmp="$EVAL_LOG_FILE.tmp" wins rounds wr best + # ponytail: opponent names are split by whitespace — fine for Tank Royale bot + # names (no spaces); switch to a mapfile IFS=',\n' read if that ever changes. + local opps=(${EVAL_OPPONENTS//,/ }) + local tmp="$EVAL_LOG_FILE.tmp" otmp opp wins rounds wr composite best : > "$tmp" - echo ">>> [eval] $EVAL_ROUNDS deterministic rounds vs $EVAL_OPPONENT" - if ! SACLSTM_EVAL_MODE=1 run_battle "$EVAL_OPPONENT" "$EVAL_ROUNDS" "$tmp"; then - rm -f "$tmp" - echo ">>> [eval] crashed — keeping previous best" - return 0 - fi + for opp in "${opps[@]}"; do + otmp="$EVAL_LOG_FILE.$opp.tmp" + : > "$otmp" + echo ">>> [eval] $EVAL_ROUNDS deterministic rounds vs $opp" + if ! SACLSTM_EVAL_MODE=1 run_battle "$opp" "$EVAL_ROUNDS" "$otmp"; then + rm -f "$otmp" "$tmp" + echo ">>> [eval] crashed vs $opp — keeping previous best" + return 0 + fi + wins=$(grep -c '"win":true' "$otmp" || true) + rounds=$(grep -c '"type":"game"' "$otmp" || true) + if (( rounds == 0 )); then + rm -f "$otmp" "$tmp" + echo ">>> [eval] no results vs $opp — keeping previous best" + return 0 + fi + wr=$(( 100 * wins / rounds )) + echo ">>> [eval] win rate: $wins/$rounds ($wr%) vs $opp" + cat "$otmp" >> "$tmp"; rm -f "$otmp" + # Per-opponent history: append this cycle's win rate, keep last MA_WINDOW. + printf '%s\n' "$(cat "$(ma_hist_file "$opp")" 2>/dev/null)" "$wr" \ + | tail -n "$MA_WINDOW" | tr '\n' ' ' > "$(ma_hist_file "$opp")" + done mv "$tmp" "$EVAL_LOG_FILE" - wins=$(grep -c '"win":true' "$EVAL_LOG_FILE" || true) - rounds=$(grep -c '"type":"game"' "$EVAL_LOG_FILE" || true) - (( rounds == 0 )) && { echo ">>> [eval] no results"; return 0; } - wr=$(( 100 * wins / rounds )) - echo ">>> [eval] win rate: $wins/$rounds ($wr%) vs $EVAL_OPPONENT" + composite=$(for opp in "${opps[@]}"; do cat "$(ma_hist_file "$opp")"; echo; done | composite_of) # ponytail: best-score state is a plain file next to the checkpoint; survives # harness restarts, no lock needed (single harness instance assumed). - best=-1 - [ -f "$WEIGHTS_DIR/best_score.txt" ] && best=$(cat "$WEIGHTS_DIR/best_score.txt") - if (( wr > best )) && [ -f "$SACLSTM_WEIGHTS_PATH" ]; then - echo "$wr" > "$WEIGHTS_DIR/best_score.txt" + best=$(cat "$WEIGHTS_DIR/best_score.txt" 2>/dev/null) + [ -z "$best" ] && best=-1 + if awk -v a="$composite" -v b="$best" 'BEGIN{exit !(a+0 > b+0)}' \ + && [ -f "$SACLSTM_WEIGHTS_PATH" ]; then + echo "$composite" > "$WEIGHTS_DIR/best_score.txt" cp "$SACLSTM_WEIGHTS_PATH" "$WEIGHTS_DIR/sac_best.zip" - echo ">>> [eval] new best ($wr%) -> sac_best.zip" + echo ">>> [eval] new best composite ($composite) -> sac_best.zip" fi } @@ -134,5 +169,5 @@ echo ">>> training complete: $NUM_CHUNKS chunks. Logs:" echo " training: $LOG_FILE" echo " eval: $EVAL_LOG_FILE" [ -f "$WEIGHTS_DIR/sac_best.zip" ] && \ - echo " best: $WEIGHTS_DIR/sac_best.zip ($(cat "$WEIGHTS_DIR/best_score.txt")%)" + echo " best: $WEIGHTS_DIR/sac_best.zip (composite $(cat "$WEIGHTS_DIR/best_score.txt"))" exit 0 diff --git a/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim b/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim index aa9102c..62a620e 100644 --- a/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim +++ b/SAC_LSTM_Bot/src/SAC_LSTM_Bot/integration.nim @@ -12,7 +12,7 @@ ## Decisions Q1–Q14: Gitea #48. import arraymancer except Linear -import std/[locks, os, math, random, strutils] +import std/[locks, os, math, random, strutils, times] import tankroyale_botapi # getBotName (#49 name-based opponent identity) import SAC_LSTM_Bot/network import SAC_LSTM_Bot/state # STATE_DIM @@ -216,8 +216,18 @@ proc pullWeights*(myVersion: var int; hidden: var int; hidden = gSharedSnap.hiddenDim true +proc evalModeActive*(): bool {.inline.} = + ## Lever 4 (#59): the harness's deterministic eval battles already run the bot + ## with SACLSTM_EVAL_MODE=1 (sac_train.sh eval_checkpoint, mechanism from #49). + ## While set, eval ticks must NOT feed the trainer — transitions would pollute + ## the replay buffer with eval-only data and trigger gradient updates. + getEnv("SACLSTM_EVAL_MODE") == "1" + proc sendTrainingMsg*(msg: TrainingMsg): bool {.inline.} = ## Bot-side enqueue (cap-256, drops on overflow per Q10). Thread-safe. + ## Lever 4 (#59): fully suppressed in eval mode — NewBattle drops too, so an + ## eval battle can neither add transitions nor clear/retarget the buffer. + if evalModeActive(): return false gTrainChan.trySend(msg) # ── Training state (testable without threads) ───────────────────────────────── @@ -256,6 +266,41 @@ proc initTrainState*(initial: FullSnap): TrainState = result.lastEnemyKey = "" result.nextSave = getSaveInterval() +# ── Training-loss metrics (campaign v2 lever 3, #59) ────────────────────────── + +proc metricsFilePath*(): string = + ## Sits next to the weights dir's parent: SAC_LSTM_Bot/training_metrics.jsonl + ## under the #49 harness (weights live in SAC_LSTM_Bot/weights/). + getWeightsPath().parentDir.parentDir / "training_metrics.jsonl" + +proc metricsLine*(epoch: float64; stepCount, bufferLen, drained, gradSteps: int; + m: SACMetrics): string = + ## One JSONL line with exactly the scalars SACTrainer.sacUpdate exposes + ## (#59 lever 3 — SACMetrics was already returned, no trainer change needed): + ## losses/alpha averaged over this pass's gradient steps, buffer size from + ## replay_buffer.len, cumulative step count and drained transition count. + "{\"epoch\":" & $epoch & + ",\"steps\":" & $stepCount & + ",\"buffer_size\":" & $bufferLen & + ",\"drained\":" & $drained & + ",\"grad_steps\":" & $gradSteps & + ",\"critic_loss\":" & $m.criticLoss & + ",\"actor_loss\":" & $m.actorLoss & + ",\"alpha_loss\":" & $m.alphaLoss & + ",\"alpha\":" & $m.alpha & "}" + +proc appendMetricsLine(st: TrainState; drained, gradSteps: int; m: SACMetrics) = + ## Lever 3 (#59): one append per trainPass (never per gradient step). Open, + ## write, close — cheap and crash-tolerant; a metrics failure never kills + ## training. + try: + let f = open(metricsFilePath(), fmAppend) + f.writeLine(metricsLine(epochTime(), st.stepCount, st.buf.len, + drained, gradSteps, m)) + f.close() + except CatchableError: + discard + proc handleTrainingMsg*(st: var TrainState; msg: TrainingMsg): bool = ## Process one message. Returns false for Shutdown (caller stops). ## Tensors are born HERE from the message's plain arrays — training thread only. @@ -284,11 +329,16 @@ proc trainPass*(st: var TrainState; drained: int) = if drained <= 0 or not st.buf.canSample: return let steps = drained * getUtdRatio() # Q2 + var gradSteps = 0 + var sumCritic, sumActor, sumAlphaLoss, sumAlpha = 0.0'f32 for i in 1 .. steps: let seqs = st.buf.sampleSequences(getBatchSize()) if seqs.len == 0: break - discard sacUpdate(st.trainer, seqs) + let m = sacUpdate(st.trainer, seqs) + sumCritic += m.criticLoss; sumActor += m.actorLoss + sumAlphaLoss += m.alphaLoss; sumAlpha += m.alpha + inc gradSteps inc st.stepCount # Save check INSIDE the step loop (#56 launch finding): at production sizes # (hidden 256 ⇒ ~1 s/step) a drain burst queues minutes of steps; checking @@ -299,6 +349,13 @@ proc trainPass*(st: var TrainState; drained: int) = st.nextSave += getSaveInterval() var full = packFull(st.trainer) discard gSaveChan.trySend(move(full)) # cap-1: drop if I/O thread is busy (Q5) + if gradSteps > 0: + # Lever 3 (#59): one metrics line per pass, losses averaged over its steps. + appendMetricsLine(st, drained, gradSteps, SACMetrics( + criticLoss: sumCritic / gradSteps.float32, + actorLoss: sumActor / gradSteps.float32, + alphaLoss: sumAlphaLoss / gradSteps.float32, + alpha: sumAlpha / gradSteps.float32)) # Publish latest actor (Q7): in-place write under the lock, bump version. withLock(gWeightLock): assert gSharedSnap.hiddenDim == st.trainer.actor.hiddenDim, @@ -382,6 +439,10 @@ proc initIntegration*() = initLock(gWeightLock) gTrainChan.open(256) # Q10 cap-256 gSaveChan.open(1) # Q5 cap-1 + # Lever 4 (#59): one-time visibility for the suppression gate (see + # sendTrainingMsg) — the eval bot trains nothing by design. + if evalModeActive(): + stderr.writeLine "[sac] SACLSTM_EVAL_MODE=1 — training input suppressed (lever 4, #59)" createThread(gTrainingThread, trainingThreadEntry) createThread(gIoThread, ioThreadEntry) diff --git a/SAC_LSTM_Bot/tests/test_integration.nim b/SAC_LSTM_Bot/tests/test_integration.nim index 5d9c11e..f30aa13 100644 --- a/SAC_LSTM_Bot/tests/test_integration.nim +++ b/SAC_LSTM_Bot/tests/test_integration.nim @@ -122,3 +122,29 @@ block: assert alpha == t0.alpha() discard tc1 echo "PASS flat snapshot pack/unpack round-trip" + +# ── 5. Lever 3 (#59): metricsLine emits exactly the exposed trainer scalars ─── +block: + let line = metricsLine(1787394115.123, 42, 500, 20, 20, + SACMetrics(criticLoss: 0.5'f32, actorLoss: -1.5'f32, + alphaLoss: 0.25'f32, alpha: 2.0'f32)) + let j = parseJson(line) # throws on malformed JSONL + assert j["steps"].getInt() == 42 and j["buffer_size"].getInt() == 500 + assert j["drained"].getInt() == 20 and j["grad_steps"].getInt() == 20 + assert abs(j["critic_loss"].getFloat() - 0.5) < 1e-3 + assert abs(j["actor_loss"].getFloat() + 1.5) < 1e-3 + assert abs(j["alpha_loss"].getFloat() - 0.25) < 1e-3 + assert abs(j["alpha"].getFloat() - 2.0) < 1e-3 + assert j["epoch"].getFloat() > 1e9 + echo "PASS metricsLine JSONL scalars" + +# ── 6. Lever 4 (#59): SACLSTM_EVAL_MODE=1 suppresses training input ─────────── +block: + putEnv("SACLSTM_EVAL_MODE", "1") + # Gate fires before any channel traffic: false = dropped, nothing enqueued. + assert not sendTrainingMsg(TrainingMsg(kind: tmkTransition)), + "eval mode must drop transitions" + assert not sendTrainingMsg(TrainingMsg(kind: tmkNewBattle, enemyId: 7)), + "eval mode must drop NewBattle (no buffer clears from eval)" + delEnv("SACLSTM_EVAL_MODE") + echo "PASS eval-mode training-input suppression"