feat(SAC_LSTM_Bot): campaign v2 levers — loss metrics, eval-mode gate, eval rotation + MA gating (part 1)
Levers 3, 4, 1 of the #57 sign-off (execution order 3->4->1), tracked in #59. - Lever 3 (#59): one JSONL line per trainPass in training_metrics.jsonl with exactly the scalars sacUpdate already exposes (SACMetrics: critic/actor/alpha losses + alpha, averaged per pass) plus epoch, buffer size (replay_buffer.len), cumulative steps and drained count. No trainer change needed. - Lever 4 (#59): sendTrainingMsg drops all training input while SACLSTM_EVAL_MODE=1 (existing #49 harness mechanism) — eval battles can neither pollute the replay buffer nor trigger gradient updates; one-time stderr notice at bot init. - Lever 1 (#59): sac_train.sh evaluates every SAC_EVAL_OPPONENTS entry per cycle (results carry opponent name in eval_log.jsonl); best-gating now uses a composite = mean over opponents of the last-5-evals moving average per opponent. best_score.txt format change: float composite replaces the single-opponent integer win rate semantics (retired). - Tests: metricsLine JSONL scalars + eval-mode suppression asserts. Refs: #59, #57
This commit is contained in:
+57
-22
@@ -5,7 +5,8 @@
|
||||
# owns server lifecycle, opponent connection and dead-bot liveness detection
|
||||
# through weights/round_counter.txt), samples opponents by weight per chunk,
|
||||
# runs deterministic evaluation (SACLSTM_EVAL_MODE=1) every N chunks, and keeps
|
||||
# the best checkpoint (weights/sac_best.zip) by eval win rate.
|
||||
# the best checkpoint (weights/sac_best.zip) by a moving-average composite over
|
||||
# the eval opponent set (campaign v2 lever 1, #59).
|
||||
#
|
||||
# Config (env vars):
|
||||
# SAC_OPPONENTS "Name:weight,Name:weight,..." (default below)
|
||||
@@ -13,7 +14,9 @@
|
||||
# SAC_CHUNK_SIZE rounds per RunTraining battle (default 10)
|
||||
# SAC_EVAL_INTERVAL eval every N chunks (default 2)
|
||||
# SAC_EVAL_ROUNDS rounds per evaluation battle (default 10)
|
||||
# SAC_EVAL_OPPONENT fixed eval opponent (default first opponent)
|
||||
# SAC_EVAL_OPPONENTS comma-separated eval set (default Corners,Crazy,Target)
|
||||
# — each cycle evaluates EVERY one; results all land in
|
||||
# eval_log.jsonl (lines carry "opponent":"Name")
|
||||
# SAC_MAX_CRASHES consecutive crashes before abort (default 5)
|
||||
# SAC_LOG_FILE / SAC_EVAL_LOG_FILE (JSON-lines logs)
|
||||
# SACLSTM_* passed through to the bot (UTD_RATIO, BATCH_SIZE, ...)
|
||||
@@ -38,7 +41,8 @@ TOTAL_ROUNDS="${SAC_TOTAL_ROUNDS:-100}"
|
||||
CHUNK_SIZE="${SAC_CHUNK_SIZE:-10}"
|
||||
EVAL_INTERVAL="${SAC_EVAL_INTERVAL:-2}"
|
||||
EVAL_ROUNDS="${SAC_EVAL_ROUNDS:-10}"
|
||||
EVAL_OPPONENT="${SAC_EVAL_OPPONENT:-${OPPONENTS%%:*}}"
|
||||
EVAL_OPPONENTS="${SAC_EVAL_OPPONENTS:-Corners,Crazy,Target}"
|
||||
MA_WINDOW=5 # lever 1 (#59): per-opponent moving average over last N evals
|
||||
MAX_CRASHES="${SAC_MAX_CRASHES:-5}"
|
||||
LOG_FILE="${SAC_LOG_FILE:-$SCRIPT_DIR/training_log.jsonl}"
|
||||
EVAL_LOG_FILE="${SAC_EVAL_LOG_FILE:-$SCRIPT_DIR/eval_log.jsonl}"
|
||||
@@ -46,7 +50,7 @@ CLASSES_DIR="/tmp/opencode/sac_train_classes"
|
||||
|
||||
echo "=== SAC_LSTM_Bot training harness ==="
|
||||
echo "Opponents: $OPPONENTS | budget: $TOTAL_ROUNDS rounds in chunks of $CHUNK_SIZE"
|
||||
echo "Eval: every $EVAL_INTERVAL chunks, $EVAL_ROUNDS rounds vs $EVAL_OPPONENT"
|
||||
echo "Eval: every $EVAL_INTERVAL chunks, $EVAL_ROUNDS rounds vs [$EVAL_OPPONENTS], MA-$MA_WINDOW composite best-gating"
|
||||
echo "Weights: $SACLSTM_WEIGHTS_PATH"
|
||||
|
||||
# ── compile bot + java runner ─────────────────────────────────────────────────
|
||||
@@ -76,29 +80,60 @@ run_battle() { # $1=opponent $2=rounds $3=log file
|
||||
PPOB_LOG_FILE="$3" java -cp "$CLASSES_DIR:$JAR" RunTraining "$1" "$2"
|
||||
}
|
||||
|
||||
# ── Lever 1 (#59): eval rotation + MA best-gating ─────────────────────────────
|
||||
# best_score.txt FORMAT CHANGE: it used to store the single-opponent integer
|
||||
# win rate (%); that semantics is retired. It now stores the COMPOSITE score —
|
||||
# the mean over SAC_EVAL_OPPONENTS of each opponent's moving average (last
|
||||
# MA_WINDOW eval win rates, %). sac_best.zip is rewritten only when the
|
||||
# composite strictly improves.
|
||||
ma_hist_file() { echo "$WEIGHTS_DIR/ma_history_$1.txt"; }
|
||||
|
||||
composite_of() { # reads one "w w w ..." history line per opponent on stdin
|
||||
awk -v W="$MA_WINDOW" '
|
||||
NF > 0 { n=NF; k=(n>W)?W:n; s=0; for(j=n-k+1;j<=n;j++) s+=$j; tot+=s/k; c++ }
|
||||
END { if (c>0) printf "%.4f", tot/c; else print "-1" }'
|
||||
}
|
||||
|
||||
eval_checkpoint() {
|
||||
local tmp="$EVAL_LOG_FILE.tmp" wins rounds wr best
|
||||
# ponytail: opponent names are split by whitespace — fine for Tank Royale bot
|
||||
# names (no spaces); switch to a mapfile IFS=',\n' read if that ever changes.
|
||||
local opps=(${EVAL_OPPONENTS//,/ })
|
||||
local tmp="$EVAL_LOG_FILE.tmp" otmp opp wins rounds wr composite best
|
||||
: > "$tmp"
|
||||
echo ">>> [eval] $EVAL_ROUNDS deterministic rounds vs $EVAL_OPPONENT"
|
||||
if ! SACLSTM_EVAL_MODE=1 run_battle "$EVAL_OPPONENT" "$EVAL_ROUNDS" "$tmp"; then
|
||||
rm -f "$tmp"
|
||||
echo ">>> [eval] crashed — keeping previous best"
|
||||
return 0
|
||||
fi
|
||||
for opp in "${opps[@]}"; do
|
||||
otmp="$EVAL_LOG_FILE.$opp.tmp"
|
||||
: > "$otmp"
|
||||
echo ">>> [eval] $EVAL_ROUNDS deterministic rounds vs $opp"
|
||||
if ! SACLSTM_EVAL_MODE=1 run_battle "$opp" "$EVAL_ROUNDS" "$otmp"; then
|
||||
rm -f "$otmp" "$tmp"
|
||||
echo ">>> [eval] crashed vs $opp — keeping previous best"
|
||||
return 0
|
||||
fi
|
||||
wins=$(grep -c '"win":true' "$otmp" || true)
|
||||
rounds=$(grep -c '"type":"game"' "$otmp" || true)
|
||||
if (( rounds == 0 )); then
|
||||
rm -f "$otmp" "$tmp"
|
||||
echo ">>> [eval] no results vs $opp — keeping previous best"
|
||||
return 0
|
||||
fi
|
||||
wr=$(( 100 * wins / rounds ))
|
||||
echo ">>> [eval] win rate: $wins/$rounds ($wr%) vs $opp"
|
||||
cat "$otmp" >> "$tmp"; rm -f "$otmp"
|
||||
# Per-opponent history: append this cycle's win rate, keep last MA_WINDOW.
|
||||
printf '%s\n' "$(cat "$(ma_hist_file "$opp")" 2>/dev/null)" "$wr" \
|
||||
| tail -n "$MA_WINDOW" | tr '\n' ' ' > "$(ma_hist_file "$opp")"
|
||||
done
|
||||
mv "$tmp" "$EVAL_LOG_FILE"
|
||||
wins=$(grep -c '"win":true' "$EVAL_LOG_FILE" || true)
|
||||
rounds=$(grep -c '"type":"game"' "$EVAL_LOG_FILE" || true)
|
||||
(( rounds == 0 )) && { echo ">>> [eval] no results"; return 0; }
|
||||
wr=$(( 100 * wins / rounds ))
|
||||
echo ">>> [eval] win rate: $wins/$rounds ($wr%) vs $EVAL_OPPONENT"
|
||||
composite=$(for opp in "${opps[@]}"; do cat "$(ma_hist_file "$opp")"; echo; done | composite_of)
|
||||
# ponytail: best-score state is a plain file next to the checkpoint; survives
|
||||
# harness restarts, no lock needed (single harness instance assumed).
|
||||
best=-1
|
||||
[ -f "$WEIGHTS_DIR/best_score.txt" ] && best=$(cat "$WEIGHTS_DIR/best_score.txt")
|
||||
if (( wr > best )) && [ -f "$SACLSTM_WEIGHTS_PATH" ]; then
|
||||
echo "$wr" > "$WEIGHTS_DIR/best_score.txt"
|
||||
best=$(cat "$WEIGHTS_DIR/best_score.txt" 2>/dev/null)
|
||||
[ -z "$best" ] && best=-1
|
||||
if awk -v a="$composite" -v b="$best" 'BEGIN{exit !(a+0 > b+0)}' \
|
||||
&& [ -f "$SACLSTM_WEIGHTS_PATH" ]; then
|
||||
echo "$composite" > "$WEIGHTS_DIR/best_score.txt"
|
||||
cp "$SACLSTM_WEIGHTS_PATH" "$WEIGHTS_DIR/sac_best.zip"
|
||||
echo ">>> [eval] new best ($wr%) -> sac_best.zip"
|
||||
echo ">>> [eval] new best composite ($composite) -> sac_best.zip"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -134,5 +169,5 @@ echo ">>> training complete: $NUM_CHUNKS chunks. Logs:"
|
||||
echo " training: $LOG_FILE"
|
||||
echo " eval: $EVAL_LOG_FILE"
|
||||
[ -f "$WEIGHTS_DIR/sac_best.zip" ] && \
|
||||
echo " best: $WEIGHTS_DIR/sac_best.zip ($(cat "$WEIGHTS_DIR/best_score.txt")%)"
|
||||
echo " best: $WEIGHTS_DIR/sac_best.zip (composite $(cat "$WEIGHTS_DIR/best_score.txt"))"
|
||||
exit 0
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
## Decisions Q1–Q14: Gitea #48.
|
||||
|
||||
import arraymancer except Linear
|
||||
import std/[locks, os, math, random, strutils]
|
||||
import std/[locks, os, math, random, strutils, times]
|
||||
import tankroyale_botapi # getBotName (#49 name-based opponent identity)
|
||||
import SAC_LSTM_Bot/network
|
||||
import SAC_LSTM_Bot/state # STATE_DIM
|
||||
@@ -216,8 +216,18 @@ proc pullWeights*(myVersion: var int; hidden: var int;
|
||||
hidden = gSharedSnap.hiddenDim
|
||||
true
|
||||
|
||||
proc evalModeActive*(): bool {.inline.} =
|
||||
## Lever 4 (#59): the harness's deterministic eval battles already run the bot
|
||||
## with SACLSTM_EVAL_MODE=1 (sac_train.sh eval_checkpoint, mechanism from #49).
|
||||
## While set, eval ticks must NOT feed the trainer — transitions would pollute
|
||||
## the replay buffer with eval-only data and trigger gradient updates.
|
||||
getEnv("SACLSTM_EVAL_MODE") == "1"
|
||||
|
||||
proc sendTrainingMsg*(msg: TrainingMsg): bool {.inline.} =
|
||||
## Bot-side enqueue (cap-256, drops on overflow per Q10). Thread-safe.
|
||||
## Lever 4 (#59): fully suppressed in eval mode — NewBattle drops too, so an
|
||||
## eval battle can neither add transitions nor clear/retarget the buffer.
|
||||
if evalModeActive(): return false
|
||||
gTrainChan.trySend(msg)
|
||||
|
||||
# ── Training state (testable without threads) ─────────────────────────────────
|
||||
@@ -256,6 +266,41 @@ proc initTrainState*(initial: FullSnap): TrainState =
|
||||
result.lastEnemyKey = ""
|
||||
result.nextSave = getSaveInterval()
|
||||
|
||||
# ── Training-loss metrics (campaign v2 lever 3, #59) ──────────────────────────
|
||||
|
||||
proc metricsFilePath*(): string =
|
||||
## Sits next to the weights dir's parent: SAC_LSTM_Bot/training_metrics.jsonl
|
||||
## under the #49 harness (weights live in SAC_LSTM_Bot/weights/).
|
||||
getWeightsPath().parentDir.parentDir / "training_metrics.jsonl"
|
||||
|
||||
proc metricsLine*(epoch: float64; stepCount, bufferLen, drained, gradSteps: int;
|
||||
m: SACMetrics): string =
|
||||
## One JSONL line with exactly the scalars SACTrainer.sacUpdate exposes
|
||||
## (#59 lever 3 — SACMetrics was already returned, no trainer change needed):
|
||||
## losses/alpha averaged over this pass's gradient steps, buffer size from
|
||||
## replay_buffer.len, cumulative step count and drained transition count.
|
||||
"{\"epoch\":" & $epoch &
|
||||
",\"steps\":" & $stepCount &
|
||||
",\"buffer_size\":" & $bufferLen &
|
||||
",\"drained\":" & $drained &
|
||||
",\"grad_steps\":" & $gradSteps &
|
||||
",\"critic_loss\":" & $m.criticLoss &
|
||||
",\"actor_loss\":" & $m.actorLoss &
|
||||
",\"alpha_loss\":" & $m.alphaLoss &
|
||||
",\"alpha\":" & $m.alpha & "}"
|
||||
|
||||
proc appendMetricsLine(st: TrainState; drained, gradSteps: int; m: SACMetrics) =
|
||||
## Lever 3 (#59): one append per trainPass (never per gradient step). Open,
|
||||
## write, close — cheap and crash-tolerant; a metrics failure never kills
|
||||
## training.
|
||||
try:
|
||||
let f = open(metricsFilePath(), fmAppend)
|
||||
f.writeLine(metricsLine(epochTime(), st.stepCount, st.buf.len,
|
||||
drained, gradSteps, m))
|
||||
f.close()
|
||||
except CatchableError:
|
||||
discard
|
||||
|
||||
proc handleTrainingMsg*(st: var TrainState; msg: TrainingMsg): bool =
|
||||
## Process one message. Returns false for Shutdown (caller stops).
|
||||
## Tensors are born HERE from the message's plain arrays — training thread only.
|
||||
@@ -284,11 +329,16 @@ proc trainPass*(st: var TrainState; drained: int) =
|
||||
if drained <= 0 or not st.buf.canSample:
|
||||
return
|
||||
let steps = drained * getUtdRatio() # Q2
|
||||
var gradSteps = 0
|
||||
var sumCritic, sumActor, sumAlphaLoss, sumAlpha = 0.0'f32
|
||||
for i in 1 .. steps:
|
||||
let seqs = st.buf.sampleSequences(getBatchSize())
|
||||
if seqs.len == 0:
|
||||
break
|
||||
discard sacUpdate(st.trainer, seqs)
|
||||
let m = sacUpdate(st.trainer, seqs)
|
||||
sumCritic += m.criticLoss; sumActor += m.actorLoss
|
||||
sumAlphaLoss += m.alphaLoss; sumAlpha += m.alpha
|
||||
inc gradSteps
|
||||
inc st.stepCount
|
||||
# Save check INSIDE the step loop (#56 launch finding): at production sizes
|
||||
# (hidden 256 ⇒ ~1 s/step) a drain burst queues minutes of steps; checking
|
||||
@@ -299,6 +349,13 @@ proc trainPass*(st: var TrainState; drained: int) =
|
||||
st.nextSave += getSaveInterval()
|
||||
var full = packFull(st.trainer)
|
||||
discard gSaveChan.trySend(move(full)) # cap-1: drop if I/O thread is busy (Q5)
|
||||
if gradSteps > 0:
|
||||
# Lever 3 (#59): one metrics line per pass, losses averaged over its steps.
|
||||
appendMetricsLine(st, drained, gradSteps, SACMetrics(
|
||||
criticLoss: sumCritic / gradSteps.float32,
|
||||
actorLoss: sumActor / gradSteps.float32,
|
||||
alphaLoss: sumAlphaLoss / gradSteps.float32,
|
||||
alpha: sumAlpha / gradSteps.float32))
|
||||
# Publish latest actor (Q7): in-place write under the lock, bump version.
|
||||
withLock(gWeightLock):
|
||||
assert gSharedSnap.hiddenDim == st.trainer.actor.hiddenDim,
|
||||
@@ -382,6 +439,10 @@ proc initIntegration*() =
|
||||
initLock(gWeightLock)
|
||||
gTrainChan.open(256) # Q10 cap-256
|
||||
gSaveChan.open(1) # Q5 cap-1
|
||||
# Lever 4 (#59): one-time visibility for the suppression gate (see
|
||||
# sendTrainingMsg) — the eval bot trains nothing by design.
|
||||
if evalModeActive():
|
||||
stderr.writeLine "[sac] SACLSTM_EVAL_MODE=1 — training input suppressed (lever 4, #59)"
|
||||
createThread(gTrainingThread, trainingThreadEntry)
|
||||
createThread(gIoThread, ioThreadEntry)
|
||||
|
||||
|
||||
@@ -122,3 +122,29 @@ block:
|
||||
assert alpha == t0.alpha()
|
||||
discard tc1
|
||||
echo "PASS flat snapshot pack/unpack round-trip"
|
||||
|
||||
# ── 5. Lever 3 (#59): metricsLine emits exactly the exposed trainer scalars ───
|
||||
block:
|
||||
let line = metricsLine(1787394115.123, 42, 500, 20, 20,
|
||||
SACMetrics(criticLoss: 0.5'f32, actorLoss: -1.5'f32,
|
||||
alphaLoss: 0.25'f32, alpha: 2.0'f32))
|
||||
let j = parseJson(line) # throws on malformed JSONL
|
||||
assert j["steps"].getInt() == 42 and j["buffer_size"].getInt() == 500
|
||||
assert j["drained"].getInt() == 20 and j["grad_steps"].getInt() == 20
|
||||
assert abs(j["critic_loss"].getFloat() - 0.5) < 1e-3
|
||||
assert abs(j["actor_loss"].getFloat() + 1.5) < 1e-3
|
||||
assert abs(j["alpha_loss"].getFloat() - 0.25) < 1e-3
|
||||
assert abs(j["alpha"].getFloat() - 2.0) < 1e-3
|
||||
assert j["epoch"].getFloat() > 1e9
|
||||
echo "PASS metricsLine JSONL scalars"
|
||||
|
||||
# ── 6. Lever 4 (#59): SACLSTM_EVAL_MODE=1 suppresses training input ───────────
|
||||
block:
|
||||
putEnv("SACLSTM_EVAL_MODE", "1")
|
||||
# Gate fires before any channel traffic: false = dropped, nothing enqueued.
|
||||
assert not sendTrainingMsg(TrainingMsg(kind: tmkTransition)),
|
||||
"eval mode must drop transitions"
|
||||
assert not sendTrainingMsg(TrainingMsg(kind: tmkNewBattle, enemyId: 7)),
|
||||
"eval mode must drop NewBattle (no buffer clears from eval)"
|
||||
delEnv("SACLSTM_EVAL_MODE")
|
||||
echo "PASS eval-mode training-input suppression"
|
||||
|
||||
Reference in New Issue
Block a user