feat(PPO_Bot): reward + trajectory + GAE + PPO training (#16)

Manual-backprop PPO with Adam: TrajectoryBuffer, computeGAE, ppoUpdate
(4 epochs, minibatch 64, clip 0.2, grad norm 0.5). Reward helpers
computeTickReward/computeRoundReward. Bot wired: tick transitions
collected in run loop, ppoUpdate called on onRoundEnded. Fix: add
arraymancer import to PPO_Bot.nim so Tensor resolves at top level.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

PPO_Bot/tests/test_training.nim

@@ -0,0 +1,102 @@
+## test_training.nim — assert-based tests for training.nim
+## Run: nim c tests/test_training.nim && ./tests/test_training
+
+import std/[math, random]
+import arraymancer
+import "../network"
+import "../training"
+
+template check(cond: bool, msg: string) =
+  if not cond:
+    quit("FAIL: " & msg, 1)
+
+# ── computeTickReward ─────────────────────────────────────────────────────────
+
+block testTickReward:
+  # I lost 2, enemy lost 10 → reward = -2 - (-10) = 8
+  let r = computeTickReward(-2.0'f32, -10.0'f32)
+  check abs(r - 8.0'f32) < 1e-6'f32, "computeTickReward(-2, -10) == 8, got " & $r
+
+# ── computeRoundReward ────────────────────────────────────────────────────────
+
+block testRoundReward:
+  let r = computeRoundReward(350.0'f32)
+  check abs(r - 3.5'f32) < 1e-6'f32, "computeRoundReward(350) == 3.5, got " & $r
+
+# ── TrajectoryBuffer ──────────────────────────────────────────────────────────
+
+block testBuffer:
+  var buf = initTrajectoryBuffer()
+  check buf.len == 0, "empty buffer len == 0"
+
+  let t1 = Transition(state: zeros[float32](42), action: zeros[float32](5),
+                       logProb: -1.0'f32, reward: 0.5'f32, value: 0.3'f32)
+  buf.add(t1)
+  buf.add(t1)
+  buf.add(t1)
+  check buf.len == 3, "buffer len == 3 after 3 adds"
+
+  buf.clear()
+  check buf.len == 0, "buffer len == 0 after clear"
+
+# ── computeGAE — hand-calculated 3-step ──────────────────────────────────────
+
+block testGAE:
+  # rewards = [1.0, 0.0, 1.0], values = [0.5, 0.5, 0.5], lastValue = 0.0
+  # gamma = 0.99, lam = 0.95
+  # delta_2 = 1.0 + 0.99*0.0 - 0.5 = 0.5
+  # adv_2   = 0.5
+  # delta_1 = 0.0 + 0.99*0.5 - 0.5 = -0.005
+  # adv_1   = -0.005 + 0.99*0.95*0.5 ≈ -0.005 + 0.47025 = 0.46525
+  # delta_0 = 1.0 + 0.99*0.5 - 0.5 = 0.995
+  # adv_0   = 0.995 + 0.99*0.95*0.46525 ≈ 0.995 + 0.43744 = 1.43244
+  let (adv, ret) = computeGAE(
+    rewards   = @[1.0'f32, 0.0'f32, 1.0'f32],
+    values    = @[0.5'f32, 0.5'f32, 0.5'f32],
+    lastValue = 0.0'f32,
+    gamma     = 0.99'f32,
+    lam       = 0.95'f32
+  )
+
+  check abs(adv[2] - 0.5'f32) < 1e-4'f32,
+    "adv[2] should be ~0.5, got " & $adv[2]
+  check abs(adv[1] - 0.46525'f32) < 1e-3'f32,
+    "adv[1] should be ~0.46525, got " & $adv[1]
+  check abs(adv[0] - 1.43244'f32) < 1e-2'f32,
+    "adv[0] should be ~1.43244, got " & $adv[0]
+
+  # returns = adv + values
+  check abs(ret[2] - (0.5'f32 + 0.5'f32)) < 1e-4'f32, "ret[2] = adv[2] + 0.5"
+  check abs(ret[0] - (adv[0] + 0.5'f32)) < 1e-4'f32,  "ret[0] = adv[0] + 0.5"
+
+# ── ppoUpdate runs without crash; weights change ──────────────────────────────
+
+block testPpoUpdate:
+  randomize(42)
+  var ac = initActorCritic()
+
+  # Save a copy of w1 before update
+  let w1Before = ac.actor.w1.clone()
+
+  var buf = initTrajectoryBuffer()
+  for _ in 0..<10:
+    let s = randomNormalTensor[float32](42)
+    let a = randomNormalTensor[float32](5)
+    let lp = ac.computeLogProb(s, a)
+    let v  = ac.criticForward(s)
+    buf.add(Transition(state: s, action: a, logProb: lp, reward: 0.1'f32, value: v))
+
+  ppoUpdate(ac, buf, lastValue = 0.0'f32, epochs = 2, miniBatchSize = 5)
+
+  # Weights should have changed — compare flattened
+  let n = ac.actor.w1.shape[0] * ac.actor.w1.shape[1]
+  let w1After = ac.actor.w1.reshape(n)
+  let w1Flat  = w1Before.reshape(n)
+  var changed = false
+  for i in 0..<n:
+    if abs(w1After[i] - w1Flat[i]) > 1e-9'f32:
+      changed = true
+      break
+  check changed, "actor w1 should change after ppoUpdate"
+
+echo "All tests passed"