fix(PPO_Bot): SIGSEGV crash fixes + static buffers for thread safety

- bullets: seq[InFlightBullet] → array[4, InFlightBullet] + bulletCount
  (eliminates cross-thread heap realloc under ORC)
- hasFired: edge-triggered (cleared after state build, not level-triggered)
- round_counter parseInt: wrapped for empty/torn file → 0
- Static SVG + intent buffers to kill cross-thread heap realloc
- Tick-local alive/bulletData also fixed arrays

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-08-20 14:23:08 +02:00
parent 75e32e3315
commit 6ad51148f4
11 changed files with 239 additions and 52 deletions
+33 -10
View File
@@ -218,8 +218,12 @@ proc ppoUpdate*(ac: var ActorCritic;
var totalGradNorm = 0.0'f32
var totalMiniBatches = 0
# Initialise Adam states once; caller persists them across rounds
if not adamStates.initialized:
# Initialise Adam states once; caller persists them across rounds.
# Also reinit if aw1.m has wrong shape (e.g. loaded from old checkpoint with
# different STATE_DIM, leaving a (0,) placeholder after shape-mismatch skip).
if not adamStates.initialized or
adamStates.aw1.m.shape.len == 0 or
adamStates.aw1.m.shape != ac.actor.w1.shape:
adamStates = initACAdamStates(ac)
# 1. GAE
@@ -253,7 +257,7 @@ proc ppoUpdate*(ac: var ActorCritic;
# Treat as full-batch; breaks the loop unconditionally.
let mbSizeCap = if miniBatchSize > 0: miniBatchSize else: bufLen
for _ in 1..epochs:
for epochNum in 1..epochs:
# Shuffle indices
var indices = toSeq(0..<bufLen)
shuffle(indices)
@@ -263,7 +267,6 @@ proc ppoUpdate*(ac: var ActorCritic;
let mbEnd = min(mbStart + mbSizeCap, bufLen)
if mbEnd <= mbStart: break
let mbSize = mbEnd - mbStart
# Accumulators for gradients (zero-init)
var dActorW1 = zeros[float32](ac.actor.w1.shape)
var dActorB1 = zeros[float32](ac.actor.b1.shape)
@@ -292,9 +295,13 @@ proc ppoUpdate*(ac: var ActorCritic;
# ── Actor forward ──
let actorFwd = mlpForwardCached(ac.actor, x)
# Critic forward
let newMean = actorFwd.y # [ACTION_DIM]
let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, logStdFloor))
# Same clamp as collection (network.nim actorForward): train-time std must
# exactly match the std the acting policy used, or ratios are distorted.
let logStdClamped = ac.logStd.map(proc(v: float32): float32 = clamp(v, logStdFloor, logStdCeiling))
let std = logStdClamped.map(proc(v: float32): float32 = exp(v))
# New log prob
@@ -337,13 +344,18 @@ proc ppoUpdate*(ac: var ActorCritic;
# Also: d(newLogP)/d(logStd_i) when logStd_i > -3:
# = (action_i - mean_i)^2/std_i^2 - 1
for i in 0..<ACTION_DIM:
let isClamped = (ac.logStd[i] <= logStdFloor)
if not isClamped:
let isFloorClamped = (ac.logStd[i] <= logStdFloor)
let isCeilingClamped = (ac.logStd[i] >= logStdCeiling)
if not isFloorClamped:
let s = std[i]
let diff = (tr.action[i] - newMean[i]) / s
let dLogP_dLogStdI = diff * diff - 1.0'f32
dLogStd[i] += dLoss_dNewLogP * dLogP_dLogStdI -
entropyCoeff / mbSize.float32 # entropy: d(-entropyCoeff*H)/d(logStd_i) = -entropyCoeff
let ppoGrad = dLoss_dNewLogP * dLogP_dLogStdI
# Entropy term pushes logStd up (update = param - lr*grad, grad is -entropyCoeff < 0).
# Gate it off at the ceiling to prevent runaway logStd.
let entropyGrad = if isCeilingClamped: 0.0'f32
else: -entropyCoeff / mbSize.float32
dLogStd[i] += ppoGrad + entropyGrad
# Backprop actor gradients
let gradActorOut = dLoss_dNewLogP *. dLogP_dMean # [5]
@@ -380,6 +392,13 @@ proc ppoUpdate*(ac: var ActorCritic;
dCriticW1, dCriticB1, dCriticW2, dCriticB2, dCriticW3, dCriticB3
]
let norm = globalNorm(allGrads)
# ponytail: NaN/Inf grad norm means something exploded this minibatch
# (extreme logprob ratios, poisoned initial weights, etc.). Skip the
# Adam update entirely — no-op is safer than writing NaN into weights,
# which corrupts all future inference and hangs the bot.
if norm != norm or norm > 1e15'f32:
mbStart = mbEnd
continue
totalGradNorm += norm
inc totalMiniBatches
if norm > maxGradNorm:
@@ -403,13 +422,17 @@ proc ppoUpdate*(ac: var ActorCritic;
adamStep(ac.actor.w3, dActorW3, adamStates.aw3, lr)
adamStep(ac.actor.b3, dActorB3, adamStates.ab3, lr)
adamStep(ac.logStd, dLogStd, adamStates.logStd, lr)
# Collection clamps logStd to [floor, ceiling] at inference; clamp the raw
# param after the step so it can't drift above the ceiling (the old code
# only clamped at collection → train-time recompute used a bigger std than
# the policy that actually acted → distorted importance ratios).
ac.logStd = ac.logStd.map(proc(v: float32): float32 = clamp(v, logStdFloor, logStdCeiling))
adamStep(ac.critic.w1, dCriticW1, adamStates.cw1, lr)
adamStep(ac.critic.b1, dCriticB1, adamStates.cb1, lr)
adamStep(ac.critic.w2, dCriticW2, adamStates.cw2, lr)
adamStep(ac.critic.b2, dCriticB2, adamStates.cb2, lr)
adamStep(ac.critic.w3, dCriticW3, adamStates.cw3, lr)
adamStep(ac.critic.b3, dCriticB3, adamStates.cb3, lr)
mbStart = mbEnd
let totalSamples = (epochs * bufLen).float32