feat(PPO_Bot): reward + trajectory + GAE + PPO training (#16)

Manual-backprop PPO with Adam: TrajectoryBuffer, computeGAE, ppoUpdate
(4 epochs, minibatch 64, clip 0.2, grad norm 0.5). Reward helpers
computeTickReward/computeRoundReward. Bot wired: tick transitions
collected in run loop, ppoUpdate called on onRoundEnded. Fix: add
arraymancer import to PPO_Bot.nim so Tensor resolves at top level.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-08-16 15:27:15 +02:00
parent 588c9ebc2f
commit eadd177d3b
10 changed files with 572 additions and 5 deletions
+13
View File
@@ -58,3 +58,16 @@ proc criticForward*(ac: ActorCritic, state: Tensor[float32]): float32 =
## state: [42]. Returns scalar value estimate.
let val = ac.critic.forward(state)
result = val[0]
proc computeLogProb*(ac: ActorCritic, state, action: Tensor[float32]): float32 =
## Log-probability of action under current policy (no sampling).
let mean = ac.actor.forward(state)
let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32))
let std = logStdClamped.map(proc(v: float32): float32 = exp(v))
var logP = 0.0'f32
for i in 0..<5:
let mu = mean[i]
let s = std[i]
let diff = (action[i] - mu) / s
logP += -0.5'f32 * diff * diff - ln(s) - 0.5'f32 * ln(2.0'f32 * PI.float32)
result = logP