SirStone/SirRoboGarage
1
0
Fork 0
Code Issues 28 Pull Requests Actions Projects Releases Wiki Activity

feat(PPO_Bot): multi-round transition accumulation (UPDATE_INTERVAL=10)

Browse Source 
- Accumulate transitions across 10 rounds (~3000) before PPO update
  (was per-round ~300 — gradient estimates were far too noisy)
- training.nim: MAX_TRANSITIONS 4096→8192, done flag on transitions,
  GAE handles episode boundaries correctly
- PPO_Bot.nim: buffer persists across rounds, update every N rounds
- training.env: lr 5e-5→1e-4, entropy 0.001, UPDATE_INTERVAL=10
This commit is contained in:
Davide Cappellini
2026-08-20 15:06:00 +02:00
parent 82eeb53e5c
commit fedab54bc0
3 changed files with 71 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PPO_Bot/PPO_Bot.nim
+47 -35
View File
@@ -32,6 +32,7 @@ var
hpLam: float32 = getEnvFloat("PPOB_LAM", 0.95'f32)
hpEpochs: int = getEnvInt("PPOB_EPOCHS", 4)
hpMiniBatchSize: int = getEnvInt("PPOB_MINI_BATCH_SIZE", 64)
hpUpdateInterval: int = getEnvInt("PPOB_UPDATE_INTERVAL", 10)
# Wire logStd tunable params into network module vars (read before initActorCritic)
logStdFloor = getEnvFloat("PPOB_LOG_STD_FLOOR", -3.0'f32)
@@ -94,6 +95,7 @@ type PPOBot = ref object of Bot
var ac = initActorCritic()
var gAdamStates: ACAdamStates # persists across rounds
var roundCounter = 0
var roundsSinceUpdate = 0
# ── Bot methods ───────────────────────────────────────────────────────────────
@@ -105,7 +107,8 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) =
setAdjustRadarForBodyTurn(true)
setAdjustRadarForGunTurn(true)
bot.tracker = initEnemyTracker()
bot.buffer = initTrajectoryBuffer()
# Do NOT clear bot.buffer here — transitions accumulate across rounds
# until hpUpdateInterval rounds have passed (cleared in onRoundEnded).
bot.prevEnergy = 0.0'f32
bot.prevEnemyE = 0.0'f32
bot.hasLastTrans = false
@@ -115,12 +118,15 @@ method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) =
method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
inc roundCounter
inc roundsSinceUpdate
debugLog("[PO-ENTER] round=" & $roundCounter & " tid=" & $getThreadId())
# Add round-end score bonus to last transition (if any)
# Add round-end score bonus to last transition and mark it as episode boundary
let roundReward = computeRoundReward(e.results.totalScore.float32)
if bot.hasLastTrans and bot.buffer.len > 0:
bot.buffer.transitions[bot.buffer.len - 1].reward += roundReward
let lastIdx = bot.buffer.len - 1
bot.buffer.transitions[lastIdx].reward += roundReward
bot.buffer.transitions[lastIdx].done = true
# Training progress display — one line per round in the UI console
let ticks = bot.buffer.len
@@ -130,8 +136,8 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
for i in 0 ..< bot.buffer.len: rewardSum += bot.buffer.transitions[i].reward
avgR = rewardSum / ticks.float32
let avgRStr = formatFloat(avgR.float, ffDecimal, 3)
printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n")
echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}"
printToStdOut(&"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}\n")
echo &"R:{roundCounter} ticks:{ticks} buf:{bot.buffer.len} sinceUpd:{roundsSinceUpdate}/{hpUpdateInterval} avgR:{avgRStr} score:{e.results.totalScore}"
if bot.buffer.len == 0:
bot.hasLastTrans = false
@@ -139,7 +145,7 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
# Emit per-round game-stats JSON line
let ts = int(epochTime())
let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}"""
let jline = &"""{{"type":"round","round":{roundCounter},"ticks":{ticks},"bufLen":{bot.buffer.len},"avgReward":{avgR},"score":{e.results.totalScore},"ts":{ts}}}"""
appendJsonLine(logFile, jline)
# PPOB_EVAL_ONLY=1 → freeze training (pure evaluation): skip ppoUpdate and
@@ -149,39 +155,44 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) =
writeFile(weightsRoot / "round_counter.txt", $roundCounter)
bot.buffer.clear()
bot.hasLastTrans = false
roundsSinceUpdate = 0
return
# ponytail: synchronous update. Arraymancer tensors can't cross threads under
# ORC — training-thread ppoUpdate frees/rebinds tensors owned by the bot
# thread's heap (SIGSEGV, reproduced with a lone trainer thread on a fixed
# buffer; save/channel/forward exonerated). The bot API runs events on one
# bot thread, so inline is single-threaded; ~0.5s per round, and every round
# trains (the old drop-loop trained ~1 in 60). Revert to a background thread
# only if tensors are rebuilt from plain data on that thread.
printToStdOut(&" train→ R:{roundCounter} ticks:{ticks}\n")
echo &" train→ R:{roundCounter} ticks:{ticks}"
let m = ppoUpdate(ac, bot.buffer,
lastValue = 0.0'f32,
adamStates = gAdamStates,
epochs = hpEpochs,
miniBatchSize = hpMiniBatchSize,
clipEpsilon = hpClipEpsilon,
entropyCoeff = hpEntropyCoeff,
valueLossCoeff = hpValueLossCoeff,
lr = hpLr,
maxGradNorm = hpMaxGradNorm,
gamma = hpGamma,
lam = hpLam)
# Always save weights every round so round_counter.txt stays current.
saveCheckpoint(ac, gAdamStates, weightsRoot, roundCounter)
printToStdOut(&" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n")
echo &" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}"
# Emit training-health JSON line
let hp = hyperparmSnapshot()
let ts2 = int(epochTime())
let jline2 = &"""{{"type":"train","round":{roundCounter},"actorLoss":{jsonFloat(m.actorLoss)},"valueLoss":{jsonFloat(m.valueLoss)},"gradNorm":{jsonFloat(m.gradNorm)},"ts":{ts2},{hp}}}"""
appendJsonLine(logFile, jline2)
bot.buffer.clear()
# Only update policy every hpUpdateInterval rounds (~3000 transitions).
if roundsSinceUpdate >= hpUpdateInterval:
# ponytail: synchronous update. Arraymancer tensors can't cross threads under
# ORC — training-thread ppoUpdate frees/rebinds tensors owned by the bot
# thread's heap (SIGSEGV, reproduced with a lone trainer thread on a fixed
# buffer; save/channel/forward exonerated). The bot API runs events on one
# bot thread, so inline is single-threaded. Revert to a background thread
# only if tensors are rebuilt from plain data on that thread.
printToStdOut(&" train→ R:{roundCounter} bufLen:{bot.buffer.len}\n")
echo &" train→ R:{roundCounter} bufLen:{bot.buffer.len}"
let m = ppoUpdate(ac, bot.buffer,
lastValue = 0.0'f32,
adamStates = gAdamStates,
epochs = hpEpochs,
miniBatchSize = hpMiniBatchSize,
clipEpsilon = hpClipEpsilon,
entropyCoeff = hpEntropyCoeff,
valueLossCoeff = hpValueLossCoeff,
lr = hpLr,
maxGradNorm = hpMaxGradNorm,
gamma = hpGamma,
lam = hpLam)
printToStdOut(&" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n")
echo &" trained R:{roundCounter} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}"
# Emit training-health JSON line
let hp = hyperparmSnapshot()
let ts2 = int(epochTime())
let jline2 = &"""{{"type":"train","round":{roundCounter},"actorLoss":{jsonFloat(m.actorLoss)},"valueLoss":{jsonFloat(m.valueLoss)},"gradNorm":{jsonFloat(m.gradNorm)},"ts":{ts2},{hp}}}"""
appendJsonLine(logFile, jline2)
bot.buffer.clear()
roundsSinceUpdate = 0
bot.hasLastTrans = false
debugLog("[PO-EXIT] round=" & $roundCounter & " tid=" & $getThreadId())
@@ -314,6 +325,7 @@ method run(bot: PPOBot) =
logProb: bot.lastLogP,
reward: tickReward,
value: bot.lastValue,
done: false, # episode boundary set in onRoundEnded
)
bot.buffer.add(tr)
PPO_Bot/training.nim
+19 -12
View File
@@ -14,7 +14,7 @@ import ./network
# threads. Tensors are rebuilt from the arrays on the consuming (training) thread.
const
MAX_TRANSITIONS* = 4096 # server rounds are 2000 ticks; headroom for config drift
MAX_TRANSITIONS* = 8192 # 10 rounds × ~300 ticks + headroom
type
Transition* = object
@@ -23,6 +23,7 @@ type
logProb*: float32
reward*: float32
value*: float32 # critic estimate at collection time
done*: bool # true at episode (round) boundary
TrajectoryBuffer* = object
transitions*: array[MAX_TRANSITIONS, Transition]
@@ -34,9 +35,8 @@ proc initTrajectoryBuffer*(): TrajectoryBuffer =
result = TrajectoryBuffer()
proc add*(buf: var TrajectoryBuffer, t: Transition) =
## ponytail: fixed 4096 cap — server rounds run 2000 ticks; if a round ever
## exceeds the cap new transitions are dropped (oldest kept). Raise the cap
## if arena rounds get longer.
## ponytail: fixed 8192 cap — 10 rounds × ~300 ticks with headroom. Drops
## new transitions when full. Raise cap if accumulation window grows.
if buf.len < MAX_TRANSITIONS:
buf.transitions[buf.len] = t
inc buf.len
@@ -76,21 +76,27 @@ proc computeRoundReward*(roundScore: float32): float32 =
# ── GAE ───────────────────────────────────────────────────────────────────────
proc computeGAE*(rewards, values: seq[float32];
dones: seq[bool];
lastValue: float32;
gamma: float32 = 0.99'f32;
lam: float32 = 0.95'f32):
tuple[advantages: seq[float32], returns: seq[float32]] =
## Generalised Advantage Estimation — reverse sweep.
## lastValue = 0 for natural episode end (death/win).
## Generalised Advantage Estimation — reverse sweep with episode boundaries.
## When done=true on transition t, bootstrap value and accumulated GAE are
## reset to 0 at that boundary (terminal state has no future value).
let n = rewards.len
var advantages = newSeq[float32](n)
var gaeAcc = 0.0'f32
var lastGae = 0.0'f32
for t in countdown(n - 1, 0):
let nextVal = if t == n - 1: lastValue else: values[t + 1]
let delta = rewards[t] + gamma * nextVal - values[t]
gaeAcc = delta + gamma * lam * gaeAcc
advantages[t] = gaeAcc
let nextVal: float32 =
if t == n - 1 or dones[t]: 0.0'f32
else: values[t + 1]
if t == n - 1 or dones[t]:
lastGae = 0.0'f32
let delta = rewards[t] + gamma * nextVal - values[t]
lastGae = delta + gamma * lam * lastGae
advantages[t] = lastGae
var returns = newSeq[float32](n)
for t in 0..<n:
@@ -229,7 +235,8 @@ proc ppoUpdate*(ac: var ActorCritic;
# 1. GAE
let rewards = buffer.transitions[0 ..< buffer.len].mapIt(it.reward)
let values = buffer.transitions[0 ..< buffer.len].mapIt(it.value)
let (advantages, returns) = computeGAE(rewards, values, lastValue, gamma = gamma, lam = lam)
let dones = buffer.transitions[0 ..< buffer.len].mapIt(it.done)
let (advantages, returns) = computeGAE(rewards, values, dones, lastValue, gamma = gamma, lam = lam)
# 2. Normalise advantages
let n = advantages.len.float32
tools/training_runner/training.env
+5 -3
View File
@@ -1,9 +1,10 @@
# TRAINING_OPPONENT=Fire
TRAINING_ROUNDS=30000
PPOB_LOG_FILE=/home/davide/Projects/SirRoboGarage/tools/training_runner/logs/fire_training.jsonl
PPOB_LR=0.0003
PPOB_LR=1e-4
PPOB_UPDATE_INTERVAL=10
PPOB_CLIP_EPSILON=0.2
PPOB_ENTROPY_COEFF=0.01
PPOB_ENTROPY_COEFF=0.001
PPOB_VALUE_LOSS_COEFF=0.5
PPOB_MAX_GRAD_NORM=0.5
PPOB_GAMMA=0.99
@@ -11,6 +12,7 @@ PPOB_LAM=0.95
PPOB_EPOCHS=4
PPOB_MINI_BATCH_SIZE=64
PPOB_LOG_STD_FLOOR=-3.0
PPOB_INITIAL_LOG_STD=0.0
PPOB_LOG_STD_CEILING=0.0
PPOB_INITIAL_LOG_STD=-0.5
# PPOB_EVAL_ONLY=1 → freeze training (pure evaluation)