feat(PPO_Bot): multi-round transition accumulation (UPDATE_INTERVAL=10)

- Accumulate transitions across 10 rounds (~3000) before PPO update
  (was per-round ~300 — gradient estimates were far too noisy)
- training.nim: MAX_TRANSITIONS 4096→8192, done flag on transitions,
  GAE handles episode boundaries correctly
- PPO_Bot.nim: buffer persists across rounds, update every N rounds
- training.env: lr 5e-5→1e-4, entropy 0.001, UPDATE_INTERVAL=10

PPO_Bot/PPO_Bot.nim

@@ -32,6 +32,7 @@ var
   hpLam:            float32 = getEnvFloat("PPOB_LAM", 0.95'f32)
   hpEpochs:         int     = getEnvInt("PPOB_EPOCHS", 4)
   hpMiniBatchSize:  int     = getEnvInt("PPOB_MINI_BATCH_SIZE", 64)
+  hpUpdateInterval: int     = getEnvInt("PPOB_UPDATE_INTERVAL", 10)
 
 # Wire logStd tunable params into network module vars (read before initActorCritic)
 logStdFloor   = getEnvFloat("PPOB_LOG_STD_FLOOR", -3.0'f32)
@@ -94,6 +95,7 @@ type PPOBot = ref object of Bot
 var ac = initActorCritic()
 var gAdamStates: ACAdamStates  # persists across rounds
 var roundCounter = 0
+var roundsSinceUpdate = 0
 
 # ── Bot methods ───────────────────────────────────────────────────────────────
 
@@ -105,7 +107,8 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) =
   setAdjustRadarForBodyTurn(true)
   setAdjustRadarForGunTurn(true)
   bot.tracker        = initEnemyTracker()
-  bot.buffer         = initTrajectoryBuffer()
+  # Do NOT clear bot.buffer here — transitions accumulate across rounds
+  # until hpUpdateInterval rounds have passed (cleared in onRoundEnded).
   bot.prevEnergy     = 0.0'f32
   bot.prevEnemyE     = 0.0'f32
   bot.hasLastTrans   = false
@@ -115,12 +118,15 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) =
 
 method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
   inc roundCounter
+  inc roundsSinceUpdate
   debugLog("[PO-ENTER] round=" & $roundCounter & " tid=" & $getThreadId())
 
-  # Add round-end score bonus to last transition (if any)
+  # Add round-end score bonus to last transition and mark it as episode boundary
   let roundReward = computeRoundReward(e.results.totalScore.float32)
   if bot.hasLastTrans and bot.buffer.len > 0:
-    bot.buffer.transitions[bot.buffer.len - 1].reward += roundReward
+    let lastIdx = bot.buffer.len - 1
+    bot.buffer.transitions[lastIdx].reward += roundReward
+    bot.buffer.transitions[lastIdx].done = true
 
   # Training progress display — one line per round in the UI console
   let ticks = bot.buffer.len
@@ -130,8 +136,8 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
     for i in 0 ..< bot.buffer.len: rewardSum += bot.buffer.transitions[i].reward
     avgR = rewardSum / ticks.float32
     let avgRStr = formatFloat(avgR.float, ffDecimal, 3)
-    printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n")
+    printToStdOut(&"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}\n")
-    echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}"
+    echo &"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}"
 
   if bot.buffer.len == 0:
     bot.hasLastTrans = false
@@ -139,7 +145,7 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
 
   # Emit per-round game-stats JSON line
   let ts = int(epochTime())
-  let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}"""
+  let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"bufLen":{bot.buffer.len},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}"""
   appendJsonLine(logFile, jline)
 
   # PPOB_EVAL_ONLY=1 → freeze training (pure evaluation): skip ppoUpdate and
@@ -149,17 +155,22 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
     writeFile(weightsRoot / "round_counter.txt", $roundCounter)
     bot.buffer.clear()
     bot.hasLastTrans = false
+    roundsSinceUpdate = 0
     return
 
+  # Always save weights every round so round_counter.txt stays current.
+  saveCheckpoint(ac, gAdamStates, weightsRoot, roundCounter)
+
+  # Only update policy every hpUpdateInterval rounds (~3000 transitions).
+  if roundsSinceUpdate >= hpUpdateInterval:
     # ponytail: synchronous update. Arraymancer tensors can't cross threads under
     # ORC — training-thread ppoUpdate frees/rebinds tensors owned by the bot
     # thread's heap (SIGSEGV, reproduced with a lone trainer thread on a fixed
     # buffer; save/channel/forward exonerated). The bot API runs events on one
-  # bot thread, so inline is single-threaded; ~0.5s per round, and every round
+    # bot thread, so inline is single-threaded. Revert to a background thread
-  # trains (the old drop-loop trained ~1 in 60). Revert to a background thread
     # only if tensors are rebuilt from plain data on that thread.
-  printToStdOut(&"  train→ R:{roundCounter} ticks:{ticks}\n")
+    printToStdOut(&"  train→ R:{roundCounter} bufLen:{bot.buffer.len}\n")
-  echo &"  train→ R:{roundCounter} ticks:{ticks}"
+    echo &"  train→ R:{roundCounter} bufLen:{bot.buffer.len}"
     let m = ppoUpdate(ac, bot.buffer,
                       lastValue      = 0.0'f32,
                       adamStates     = gAdamStates,
@@ -172,7 +183,6 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
                       maxGradNorm    = hpMaxGradNorm,
                       gamma          = hpGamma,
                       lam            = hpLam)
-  saveCheckpoint(ac, gAdamStates, weightsRoot, roundCounter)
     printToStdOut(&"  trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n")
     echo &"  trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}"
     # Emit training-health JSON line
@@ -180,8 +190,9 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
     let ts2 = int(epochTime())
     let jline2 = &"""{{"type":"train","round":{roundCounter},"actorLoss":{jsonFloat(m.actorLoss)},"valueLoss":{jsonFloat(m.valueLoss)},"gradNorm":{jsonFloat(m.gradNorm)},"ts":{ts2},{hp}}}"""
     appendJsonLine(logFile, jline2)
-
     bot.buffer.clear()
+    roundsSinceUpdate = 0
+
   bot.hasLastTrans = false
   debugLog("[PO-EXIT] round=" & $roundCounter & " tid=" & $getThreadId())
 
@@ -314,6 +325,7 @@ method run(bot: PPOBot) =
         logProb: bot.lastLogP,
         reward:  tickReward,
         value:   bot.lastValue,
+        done:    false,  # episode boundary set in onRoundEnded
       )
       bot.buffer.add(tr)
 

PPO_Bot/training.nim

@@ -14,7 +14,7 @@ import ./network
 # threads. Tensors are rebuilt from the arrays on the consuming (training) thread.
 
 const
-  MAX_TRANSITIONS* = 4096  # server rounds are 2000 ticks; headroom for config drift
+  MAX_TRANSITIONS* = 8192  # 10 rounds × ~300 ticks + headroom
 
 type
   Transition* = object
@@ -23,6 +23,7 @@ type
     logProb*: float32
     reward*:  float32
     value*:   float32          # critic estimate at collection time
+    done*:    bool             # true at episode (round) boundary
 
   TrajectoryBuffer* = object
     transitions*: array[MAX_TRANSITIONS, Transition]
@@ -34,9 +35,8 @@ proc initTrajectoryBuffer*(): TrajectoryBuffer =
   result = TrajectoryBuffer()
 
 proc add*(buf: var TrajectoryBuffer, t: Transition) =
-  ## ponytail: fixed 4096 cap — server rounds run 2000 ticks; if a round ever
+  ## ponytail: fixed 8192 cap — 10 rounds × ~300 ticks with headroom. Drops
-  ## exceeds the cap new transitions are dropped (oldest kept). Raise the cap
+  ## new transitions when full. Raise cap if accumulation window grows.
-  ## if arena rounds get longer.
   if buf.len < MAX_TRANSITIONS:
     buf.transitions[buf.len] = t
     inc buf.len
@@ -76,21 +76,27 @@ proc computeRoundReward*(roundScore: float32): float32 =
 # ── GAE ───────────────────────────────────────────────────────────────────────
 
 proc computeGAE*(rewards, values: seq[float32];
+                 dones: seq[bool];
                  lastValue: float32;
                  gamma: float32 = 0.99'f32;
                  lam:   float32 = 0.95'f32):
     tuple[advantages: seq[float32], returns: seq[float32]] =
-  ## Generalised Advantage Estimation — reverse sweep.
+  ## Generalised Advantage Estimation — reverse sweep with episode boundaries.
-  ## lastValue = 0 for natural episode end (death/win).
+  ## When done=true on transition t, bootstrap value and accumulated GAE are
+  ## reset to 0 at that boundary (terminal state has no future value).
   let n = rewards.len
   var advantages = newSeq[float32](n)
-  var gaeAcc = 0.0'f32
+  var lastGae = 0.0'f32
 
   for t in countdown(n - 1, 0):
-    let nextVal = if t == n - 1: lastValue else: values[t + 1]
+    let nextVal: float32 =
+      if t == n - 1 or dones[t]: 0.0'f32
+      else: values[t + 1]
+    if t == n - 1 or dones[t]:
+      lastGae = 0.0'f32
     let delta = rewards[t] + gamma * nextVal - values[t]
-    gaeAcc      = delta + gamma * lam * gaeAcc
+    lastGae   = delta + gamma * lam * lastGae
-    advantages[t] = gaeAcc
+    advantages[t] = lastGae
 
   var returns = newSeq[float32](n)
   for t in 0..<n:
@@ -229,7 +235,8 @@ proc ppoUpdate*(ac: var ActorCritic;
   # 1. GAE
   let rewards = buffer.transitions[0 ..< buffer.len].mapIt(it.reward)
   let values  = buffer.transitions[0 ..< buffer.len].mapIt(it.value)
-  let (advantages, returns) = computeGAE(rewards, values, lastValue, gamma = gamma, lam = lam)
+  let dones   = buffer.transitions[0 ..< buffer.len].mapIt(it.done)
+  let (advantages, returns) = computeGAE(rewards, values, dones, lastValue, gamma = gamma, lam = lam)
 
   # 2. Normalise advantages
   let n = advantages.len.float32

tools/training_runner/training.env

@@ -1,9 +1,10 @@
 # TRAINING_OPPONENT=Fire
 TRAINING_ROUNDS=30000
 PPOB_LOG_FILE=/home/davide/Projects/SirRoboGarage/tools/training_runner/logs/fire_training.jsonl
-PPOB_LR=0.0003
+PPOB_LR=1e-4
+PPOB_UPDATE_INTERVAL=10
 PPOB_CLIP_EPSILON=0.2
-PPOB_ENTROPY_COEFF=0.01
+PPOB_ENTROPY_COEFF=0.001
 PPOB_VALUE_LOSS_COEFF=0.5
 PPOB_MAX_GRAD_NORM=0.5
 PPOB_GAMMA=0.99
@@ -11,6 +12,7 @@ PPOB_LAM=0.95
 PPOB_EPOCHS=4
 PPOB_MINI_BATCH_SIZE=64
 PPOB_LOG_STD_FLOOR=-3.0
-PPOB_INITIAL_LOG_STD=0.0
+PPOB_LOG_STD_CEILING=0.0
+PPOB_INITIAL_LOG_STD=-0.5
 
 # PPOB_EVAL_ONLY=1 → freeze training (pure evaluation)