From fedab54bc0876c97ef1c1f2a7bb2e432d1e56509 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Thu, 20 Aug 2026 15:06:00 +0200 Subject: [PATCH] feat(PPO_Bot): multi-round transition accumulation (UPDATE_INTERVAL=10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Accumulate transitions across 10 rounds (~3000) before PPO update (was per-round ~300 — gradient estimates were far too noisy) - training.nim: MAX_TRANSITIONS 4096→8192, done flag on transitions, GAE handles episode boundaries correctly - PPO_Bot.nim: buffer persists across rounds, update every N rounds - training.env: lr 5e-5→1e-4, entropy 0.001, UPDATE_INTERVAL=10 --- PPO_Bot/PPO_Bot.nim | 82 +++++++++++++++++------------- PPO_Bot/training.nim | 31 ++++++----- tools/training_runner/training.env | 8 +-- 3 files changed, 71 insertions(+), 50 deletions(-) diff --git a/PPO_Bot/PPO_Bot.nim b/PPO_Bot/PPO_Bot.nim index 235e38d..0324b50 100644 --- a/PPO_Bot/PPO_Bot.nim +++ b/PPO_Bot/PPO_Bot.nim @@ -32,6 +32,7 @@ var hpLam: float32 = getEnvFloat("PPOB_LAM", 0.95'f32) hpEpochs: int = getEnvInt("PPOB_EPOCHS", 4) hpMiniBatchSize: int = getEnvInt("PPOB_MINI_BATCH_SIZE", 64) + hpUpdateInterval: int = getEnvInt("PPOB_UPDATE_INTERVAL", 10) # Wire logStd tunable params into network module vars (read before initActorCritic) logStdFloor = getEnvFloat("PPOB_LOG_STD_FLOOR", -3.0'f32) @@ -94,6 +95,7 @@ type PPOBot = ref object of Bot var ac = initActorCritic() var gAdamStates: ACAdamStates # persists across rounds var roundCounter = 0 +var roundsSinceUpdate = 0 # ── Bot methods ─────────────────────────────────────────────────────────────── @@ -105,7 +107,8 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) = setAdjustRadarForBodyTurn(true) setAdjustRadarForGunTurn(true) bot.tracker = initEnemyTracker() - bot.buffer = initTrajectoryBuffer() + # Do NOT clear bot.buffer here — transitions accumulate across rounds + # until hpUpdateInterval rounds have passed (cleared in onRoundEnded). bot.prevEnergy = 0.0'f32 bot.prevEnemyE = 0.0'f32 bot.hasLastTrans = false @@ -115,12 +118,15 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) = method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = inc roundCounter + inc roundsSinceUpdate debugLog("[PO-ENTER] round=" & $roundCounter & " tid=" & $getThreadId()) - # Add round-end score bonus to last transition (if any) + # Add round-end score bonus to last transition and mark it as episode boundary let roundReward = computeRoundReward(e.results.totalScore.float32) if bot.hasLastTrans and bot.buffer.len > 0: - bot.buffer.transitions[bot.buffer.len - 1].reward += roundReward + let lastIdx = bot.buffer.len - 1 + bot.buffer.transitions[lastIdx].reward += roundReward + bot.buffer.transitions[lastIdx].done = true # Training progress display — one line per round in the UI console let ticks = bot.buffer.len @@ -130,8 +136,8 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = for i in 0 ..< bot.buffer.len: rewardSum += bot.buffer.transitions[i].reward avgR = rewardSum / ticks.float32 let avgRStr = formatFloat(avgR.float, ffDecimal, 3) - printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n") - echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}" + printToStdOut(&"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}\n") + echo &"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}" if bot.buffer.len == 0: bot.hasLastTrans = false @@ -139,7 +145,7 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = # Emit per-round game-stats JSON line let ts = int(epochTime()) - let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}""" + let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"bufLen":{bot.buffer.len},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}""" appendJsonLine(logFile, jline) # PPOB_EVAL_ONLY=1 → freeze training (pure evaluation): skip ppoUpdate and @@ -149,39 +155,44 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = writeFile(weightsRoot / "round_counter.txt", $roundCounter) bot.buffer.clear() bot.hasLastTrans = false + roundsSinceUpdate = 0 return - # ponytail: synchronous update. Arraymancer tensors can't cross threads under - # ORC — training-thread ppoUpdate frees/rebinds tensors owned by the bot - # thread's heap (SIGSEGV, reproduced with a lone trainer thread on a fixed - # buffer; save/channel/forward exonerated). The bot API runs events on one - # bot thread, so inline is single-threaded; ~0.5s per round, and every round - # trains (the old drop-loop trained ~1 in 60). Revert to a background thread - # only if tensors are rebuilt from plain data on that thread. - printToStdOut(&" train→ R:{roundCounter} ticks:{ticks}\n") - echo &" train→ R:{roundCounter} ticks:{ticks}" - let m = ppoUpdate(ac, bot.buffer, - lastValue = 0.0'f32, - adamStates = gAdamStates, - epochs = hpEpochs, - miniBatchSize = hpMiniBatchSize, - clipEpsilon = hpClipEpsilon, - entropyCoeff = hpEntropyCoeff, - valueLossCoeff = hpValueLossCoeff, - lr = hpLr, - maxGradNorm = hpMaxGradNorm, - gamma = hpGamma, - lam = hpLam) + # Always save weights every round so round_counter.txt stays current. saveCheckpoint(ac, gAdamStates, weightsRoot, roundCounter) - printToStdOut(&" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n") - echo &" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}" - # Emit training-health JSON line - let hp = hyperparmSnapshot() - let ts2 = int(epochTime()) - let jline2 = &"""{{"type":"train","round":{roundCounter},"actorLoss":{jsonFloat(m.actorLoss)},"valueLoss":{jsonFloat(m.valueLoss)},"gradNorm":{jsonFloat(m.gradNorm)},"ts":{ts2},{hp}}}""" - appendJsonLine(logFile, jline2) - bot.buffer.clear() + # Only update policy every hpUpdateInterval rounds (~3000 transitions). + if roundsSinceUpdate >= hpUpdateInterval: + # ponytail: synchronous update. Arraymancer tensors can't cross threads under + # ORC — training-thread ppoUpdate frees/rebinds tensors owned by the bot + # thread's heap (SIGSEGV, reproduced with a lone trainer thread on a fixed + # buffer; save/channel/forward exonerated). The bot API runs events on one + # bot thread, so inline is single-threaded. Revert to a background thread + # only if tensors are rebuilt from plain data on that thread. + printToStdOut(&" train→ R:{roundCounter} bufLen:{bot.buffer.len}\n") + echo &" train→ R:{roundCounter} bufLen:{bot.buffer.len}" + let m = ppoUpdate(ac, bot.buffer, + lastValue = 0.0'f32, + adamStates = gAdamStates, + epochs = hpEpochs, + miniBatchSize = hpMiniBatchSize, + clipEpsilon = hpClipEpsilon, + entropyCoeff = hpEntropyCoeff, + valueLossCoeff = hpValueLossCoeff, + lr = hpLr, + maxGradNorm = hpMaxGradNorm, + gamma = hpGamma, + lam = hpLam) + printToStdOut(&" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n") + echo &" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}" + # Emit training-health JSON line + let hp = hyperparmSnapshot() + let ts2 = int(epochTime()) + let jline2 = &"""{{"type":"train","round":{roundCounter},"actorLoss":{jsonFloat(m.actorLoss)},"valueLoss":{jsonFloat(m.valueLoss)},"gradNorm":{jsonFloat(m.gradNorm)},"ts":{ts2},{hp}}}""" + appendJsonLine(logFile, jline2) + bot.buffer.clear() + roundsSinceUpdate = 0 + bot.hasLastTrans = false debugLog("[PO-EXIT] round=" & $roundCounter & " tid=" & $getThreadId()) @@ -314,6 +325,7 @@ method run(bot: PPOBot) = logProb: bot.lastLogP, reward: tickReward, value: bot.lastValue, + done: false, # episode boundary set in onRoundEnded ) bot.buffer.add(tr) diff --git a/PPO_Bot/training.nim b/PPO_Bot/training.nim index 095ef87..645bfb3 100644 --- a/PPO_Bot/training.nim +++ b/PPO_Bot/training.nim @@ -14,7 +14,7 @@ import ./network # threads. Tensors are rebuilt from the arrays on the consuming (training) thread. const - MAX_TRANSITIONS* = 4096 # server rounds are 2000 ticks; headroom for config drift + MAX_TRANSITIONS* = 8192 # 10 rounds × ~300 ticks + headroom type Transition* = object @@ -23,6 +23,7 @@ type logProb*: float32 reward*: float32 value*: float32 # critic estimate at collection time + done*: bool # true at episode (round) boundary TrajectoryBuffer* = object transitions*: array[MAX_TRANSITIONS, Transition] @@ -34,9 +35,8 @@ proc initTrajectoryBuffer*(): TrajectoryBuffer = result = TrajectoryBuffer() proc add*(buf: var TrajectoryBuffer, t: Transition) = - ## ponytail: fixed 4096 cap — server rounds run 2000 ticks; if a round ever - ## exceeds the cap new transitions are dropped (oldest kept). Raise the cap - ## if arena rounds get longer. + ## ponytail: fixed 8192 cap — 10 rounds × ~300 ticks with headroom. Drops + ## new transitions when full. Raise cap if accumulation window grows. if buf.len < MAX_TRANSITIONS: buf.transitions[buf.len] = t inc buf.len @@ -76,21 +76,27 @@ proc computeRoundReward*(roundScore: float32): float32 = # ── GAE ─────────────────────────────────────────────────────────────────────── proc computeGAE*(rewards, values: seq[float32]; + dones: seq[bool]; lastValue: float32; gamma: float32 = 0.99'f32; lam: float32 = 0.95'f32): tuple[advantages: seq[float32], returns: seq[float32]] = - ## Generalised Advantage Estimation — reverse sweep. - ## lastValue = 0 for natural episode end (death/win). + ## Generalised Advantage Estimation — reverse sweep with episode boundaries. + ## When done=true on transition t, bootstrap value and accumulated GAE are + ## reset to 0 at that boundary (terminal state has no future value). let n = rewards.len var advantages = newSeq[float32](n) - var gaeAcc = 0.0'f32 + var lastGae = 0.0'f32 for t in countdown(n - 1, 0): - let nextVal = if t == n - 1: lastValue else: values[t + 1] - let delta = rewards[t] + gamma * nextVal - values[t] - gaeAcc = delta + gamma * lam * gaeAcc - advantages[t] = gaeAcc + let nextVal: float32 = + if t == n - 1 or dones[t]: 0.0'f32 + else: values[t + 1] + if t == n - 1 or dones[t]: + lastGae = 0.0'f32 + let delta = rewards[t] + gamma * nextVal - values[t] + lastGae = delta + gamma * lam * lastGae + advantages[t] = lastGae var returns = newSeq[float32](n) for t in 0..