diff --git a/PPO_Bot/.gitignore b/PPO_Bot/.gitignore new file mode 100644 index 0000000..f3685e2 --- /dev/null +++ b/PPO_Bot/.gitignore @@ -0,0 +1,3 @@ +nimble.develop +nimble.paths +nimbledeps diff --git a/PPO_Bot/PPO_Bot.json b/PPO_Bot/PPO_Bot.json new file mode 100644 index 0000000..aaaeac5 --- /dev/null +++ b/PPO_Bot/PPO_Bot.json @@ -0,0 +1,11 @@ +{ + "name": "PPO_Bot", + "version": "0.1.0", + "authors": ["Davide Cappellini"], + "description": "PPO-trained RL bot", + "homepage": "", + "countryCodes": ["IT"], + "gameTypes": ["classic", "melee", "1v1"], + "platform": "Nim", + "programmingLang": "Nim" +} diff --git a/PPO_Bot/PPO_Bot.nim b/PPO_Bot/PPO_Bot.nim index e664115..081fc49 100644 --- a/PPO_Bot/PPO_Bot.nim +++ b/PPO_Bot/PPO_Bot.nim @@ -1,17 +1,27 @@ ## PPO_Bot — enemy tracker + state vector wired into the game loop. -## Forward pass uses a random ActorCritic policy (weights not yet trained). +## Training: trajectory collected per tick, PPO update on round end. import std/os +import arraymancer import tankroyale_botapi import network import actions +import training import ./enemy_tracker import ./state_vector const botJsonPath = currentSourcePath().parentDir / "PPO_Bot.json" type PPOBot = ref object of Bot - tracker: EnemyTracker + tracker: EnemyTracker + buffer: TrajectoryBuffer + prevEnergy: float32 # own energy last tick + prevEnemyE: float32 # enemy energy last tick (from tracker) + lastState: Tensor[float32] + lastAction: Tensor[float32] + lastLogP: float32 + lastValue: float32 + hasLastTrans: bool var ac = initActorCritic() @@ -19,9 +29,29 @@ method onScannedBot*(bot: PPOBot, e: ScannedBotEvent) = bot.tracker.update(e.x, e.y, e.direction, e.speed, e.energy) method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) = - bot.tracker = initEnemyTracker() + bot.tracker = initEnemyTracker() + bot.buffer = initTrajectoryBuffer() + bot.prevEnergy = 0.0'f32 + bot.prevEnemyE = 0.0'f32 + bot.hasLastTrans = false + +method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = + # Add round-end score bonus to last transition (if any) + let roundReward = computeRoundReward(e.results.totalScore.float32) + if bot.hasLastTrans and bot.buffer.len > 0: + bot.buffer.transitions[^1].reward += roundReward + + # PPO update synchronous; background thread is issue #17 + # ponytail: blocking update per round; move to thread pool when #17 lands + ppoUpdate(ac, bot.buffer, lastValue = 0.0'f32) + bot.buffer.clear() + bot.hasLastTrans = false method run(bot: PPOBot) = + # Seed energy on first tick + bot.prevEnergy = getEnergy().float32 + bot.prevEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: 0.0'f32 + while isRunning(): bot.tracker.deadReckon() @@ -41,9 +71,37 @@ method run(bot: PPOBot) = ) let state = buildStateVector(botData, bot.tracker) - let (rawActs, _) = ac.actorForward(state) + let (rawActs, logP) = ac.actorForward(state) + let value = ac.criticForward(state) let acts = mapActions(rawActs, getSpeed().float32, getGunHeat().float32) + # Compute tick reward from energy deltas + let curEnergy = getEnergy().float32 + let curEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: bot.prevEnemyE + let myDelta = curEnergy - bot.prevEnergy + let enemyDelta = curEnemyE - bot.prevEnemyE + let tickReward = computeTickReward(myDelta, enemyDelta) + + # Finalise previous transition with the reward from this tick's state change + if bot.hasLastTrans: + let tr = Transition( + state: bot.lastState, + action: bot.lastAction, + logProb: bot.lastLogP, + reward: tickReward, + value: bot.lastValue, + ) + bot.buffer.add(tr) + + # Store current for next tick + bot.lastState = state + bot.lastAction = rawActs + bot.lastLogP = logP + bot.lastValue = value + bot.prevEnergy = curEnergy + bot.prevEnemyE = curEnemyE + bot.hasLastTrans = true + setTargetSpeed(acts.targetSpeed.float) setTurnRate(acts.turnRate.float) setGunTurnRate(acts.gunTurnRate.float) @@ -52,5 +110,8 @@ method run(bot: PPOBot) = go() when isMainModule: - var bot = PPOBot(tracker: initEnemyTracker()) + var bot = PPOBot( + tracker: initEnemyTracker(), + buffer: initTrajectoryBuffer(), + ) start(bot, botJsonPath) diff --git a/PPO_Bot/PPO_Bot.nimble b/PPO_Bot/PPO_Bot.nimble new file mode 100644 index 0000000..ef8bc53 --- /dev/null +++ b/PPO_Bot/PPO_Bot.nimble @@ -0,0 +1,11 @@ +# Package +version = "0.1.0" +author = "Davide Cappellini" +description = "PPO-trained Tank Royale bot" +license = "MIT" +bin = @["PPO_Bot"] + +# Dependencies +requires "nim >= 2.0.0" +requires "tankroyale_botapi >= 1.0.0" +requires "arraymancer >= 0.7.0" diff --git a/PPO_Bot/PPO_Bot.sh b/PPO_Bot/PPO_Bot.sh new file mode 100755 index 0000000..5feff62 --- /dev/null +++ b/PPO_Bot/PPO_Bot.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# PPO_Bot — PPO-trained RL bot (compiled native binary) +cd -- "$(dirname -- "$0")" +exec "./PPO_Bot" diff --git a/PPO_Bot/config.nims b/PPO_Bot/config.nims new file mode 100644 index 0000000..b47a741 --- /dev/null +++ b/PPO_Bot/config.nims @@ -0,0 +1,8 @@ +# Static-link OpenBLAS for portable deployment +# ponytail: adjust path per machine, or use pkg-config +switch("passL", "-lopenblas") +switch("threads", "on") +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/PPO_Bot/network.nim b/PPO_Bot/network.nim index 614760e..5730ce6 100644 --- a/PPO_Bot/network.nim +++ b/PPO_Bot/network.nim @@ -58,3 +58,16 @@ proc criticForward*(ac: ActorCritic, state: Tensor[float32]): float32 = ## state: [42]. Returns scalar value estimate. let val = ac.critic.forward(state) result = val[0] + +proc computeLogProb*(ac: ActorCritic, state, action: Tensor[float32]): float32 = + ## Log-probability of action under current policy (no sampling). + let mean = ac.actor.forward(state) + let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32)) + let std = logStdClamped.map(proc(v: float32): float32 = exp(v)) + var logP = 0.0'f32 + for i in 0..<5: + let mu = mean[i] + let s = std[i] + let diff = (action[i] - mu) / s + logP += -0.5'f32 * diff * diff - ln(s) - 0.5'f32 * ln(2.0'f32 * PI.float32) + result = logP diff --git a/PPO_Bot/shell.nix b/PPO_Bot/shell.nix new file mode 100644 index 0000000..f5dcea5 --- /dev/null +++ b/PPO_Bot/shell.nix @@ -0,0 +1,7 @@ +{ pkgs ? import {} }: + +pkgs.mkShell { + buildInputs = with pkgs; [ + openblas + ]; +} diff --git a/PPO_Bot/tests/test_training.nim b/PPO_Bot/tests/test_training.nim new file mode 100644 index 0000000..90db3fd --- /dev/null +++ b/PPO_Bot/tests/test_training.nim @@ -0,0 +1,102 @@ +## test_training.nim — assert-based tests for training.nim +## Run: nim c tests/test_training.nim && ./tests/test_training + +import std/[math, random] +import arraymancer +import "../network" +import "../training" + +template check(cond: bool, msg: string) = + if not cond: + quit("FAIL: " & msg, 1) + +# ── computeTickReward ───────────────────────────────────────────────────────── + +block testTickReward: + # I lost 2, enemy lost 10 → reward = -2 - (-10) = 8 + let r = computeTickReward(-2.0'f32, -10.0'f32) + check abs(r - 8.0'f32) < 1e-6'f32, "computeTickReward(-2, -10) == 8, got " & $r + +# ── computeRoundReward ──────────────────────────────────────────────────────── + +block testRoundReward: + let r = computeRoundReward(350.0'f32) + check abs(r - 3.5'f32) < 1e-6'f32, "computeRoundReward(350) == 3.5, got " & $r + +# ── TrajectoryBuffer ────────────────────────────────────────────────────────── + +block testBuffer: + var buf = initTrajectoryBuffer() + check buf.len == 0, "empty buffer len == 0" + + let t1 = Transition(state: zeros[float32](42), action: zeros[float32](5), + logProb: -1.0'f32, reward: 0.5'f32, value: 0.3'f32) + buf.add(t1) + buf.add(t1) + buf.add(t1) + check buf.len == 3, "buffer len == 3 after 3 adds" + + buf.clear() + check buf.len == 0, "buffer len == 0 after clear" + +# ── computeGAE — hand-calculated 3-step ────────────────────────────────────── + +block testGAE: + # rewards = [1.0, 0.0, 1.0], values = [0.5, 0.5, 0.5], lastValue = 0.0 + # gamma = 0.99, lam = 0.95 + # delta_2 = 1.0 + 0.99*0.0 - 0.5 = 0.5 + # adv_2 = 0.5 + # delta_1 = 0.0 + 0.99*0.5 - 0.5 = -0.005 + # adv_1 = -0.005 + 0.99*0.95*0.5 ≈ -0.005 + 0.47025 = 0.46525 + # delta_0 = 1.0 + 0.99*0.5 - 0.5 = 0.995 + # adv_0 = 0.995 + 0.99*0.95*0.46525 ≈ 0.995 + 0.43744 = 1.43244 + let (adv, ret) = computeGAE( + rewards = @[1.0'f32, 0.0'f32, 1.0'f32], + values = @[0.5'f32, 0.5'f32, 0.5'f32], + lastValue = 0.0'f32, + gamma = 0.99'f32, + lam = 0.95'f32 + ) + + check abs(adv[2] - 0.5'f32) < 1e-4'f32, + "adv[2] should be ~0.5, got " & $adv[2] + check abs(adv[1] - 0.46525'f32) < 1e-3'f32, + "adv[1] should be ~0.46525, got " & $adv[1] + check abs(adv[0] - 1.43244'f32) < 1e-2'f32, + "adv[0] should be ~1.43244, got " & $adv[0] + + # returns = adv + values + check abs(ret[2] - (0.5'f32 + 0.5'f32)) < 1e-4'f32, "ret[2] = adv[2] + 0.5" + check abs(ret[0] - (adv[0] + 0.5'f32)) < 1e-4'f32, "ret[0] = adv[0] + 0.5" + +# ── ppoUpdate runs without crash; weights change ────────────────────────────── + +block testPpoUpdate: + randomize(42) + var ac = initActorCritic() + + # Save a copy of w1 before update + let w1Before = ac.actor.w1.clone() + + var buf = initTrajectoryBuffer() + for _ in 0..<10: + let s = randomNormalTensor[float32](42) + let a = randomNormalTensor[float32](5) + let lp = ac.computeLogProb(s, a) + let v = ac.criticForward(s) + buf.add(Transition(state: s, action: a, logProb: lp, reward: 0.1'f32, value: v)) + + ppoUpdate(ac, buf, lastValue = 0.0'f32, epochs = 2, miniBatchSize = 5) + + # Weights should have changed — compare flattened + let n = ac.actor.w1.shape[0] * ac.actor.w1.shape[1] + let w1After = ac.actor.w1.reshape(n) + let w1Flat = w1Before.reshape(n) + var changed = false + for i in 0.. 1e-9'f32: + changed = true + break + check changed, "actor w1 should change after ppoUpdate" + +echo "All tests passed" diff --git a/PPO_Bot/training.nim b/PPO_Bot/training.nim new file mode 100644 index 0000000..313f590 --- /dev/null +++ b/PPO_Bot/training.nim @@ -0,0 +1,347 @@ +## training.nim — Trajectory buffer, GAE, and PPO training loop. +## Uses manual backprop through the 3-layer tanh MLP + manual Adam. +## No external autograd dependencies — pure Arraymancer Tensor math. + +import arraymancer +import std/[math, random, sequtils] +import ./network + +# ── Types ───────────────────────────────────────────────────────────────────── + +type + Transition* = object + state*: Tensor[float32] # [42] + action*: Tensor[float32] # [5] + logProb*: float32 + reward*: float32 + value*: float32 # critic estimate at collection time + + TrajectoryBuffer* = object + transitions*: seq[Transition] + +# ── Buffer ───────────────────────────────────────────────────────────────────── + +proc initTrajectoryBuffer*(): TrajectoryBuffer = + result.transitions = @[] + +proc add*(buf: var TrajectoryBuffer, t: Transition) = + buf.transitions.add(t) + +proc clear*(buf: var TrajectoryBuffer) = + buf.transitions.setLen(0) + +proc len*(buf: TrajectoryBuffer): int = + buf.transitions.len + +# ── Reward helpers ───────────────────────────────────────────────────────────── + +proc computeTickReward*(myEnergyDelta, enemyEnergyDelta: float32): float32 = + ## Positive when we deal more damage than we receive. + result = myEnergyDelta - enemyEnergyDelta + +proc computeRoundReward*(roundScore: float32): float32 = + ## Normalise round-end score to a rough ±3 range. + result = roundScore / 100.0'f32 + +# ── GAE ─────────────────────────────────────────────────────────────────────── + +proc computeGAE*(rewards, values: seq[float32]; + lastValue: float32; + gamma: float32 = 0.99'f32; + lam: float32 = 0.95'f32): + tuple[advantages: seq[float32], returns: seq[float32]] = + ## Generalised Advantage Estimation — reverse sweep. + ## lastValue = 0 for natural episode end (death/win). + let n = rewards.len + var advantages = newSeq[float32](n) + var gaeAcc = 0.0'f32 + + for t in countdown(n - 1, 0): + let nextVal = if t == n - 1: lastValue else: values[t + 1] + let delta = rewards[t] + gamma * nextVal - values[t] + gaeAcc = delta + gamma * lam * gaeAcc + advantages[t] = gaeAcc + + var returns = newSeq[float32](n) + for t in 0.. -3, else 0) + # total loss gradient w.r.t. logStd: -entropyCoeff * d(entropy)/d(logStd) + # Also: d(newLogP)/d(logStd_i) when logStd_i > -3: + # = (action_i - mean_i)^2/std_i^2 - 1 + for i in 0..<5: + let isClamped = (ac.logStd[i] <= -3.0'f32) + if not isClamped: + let s = std[i] + let diff = (tr.action[i] - newMean[i]) / s + let dLogP_dLogStdI = diff * diff - 1.0'f32 + dLogStd[i] += dLoss_dNewLogP * dLogP_dLogStdI - + entropyCoeff / mbSize.float32 # entropy: d(-entropyCoeff*H)/d(logStd_i) = -entropyCoeff + + # Backprop actor gradients + let gradActorOut = dLoss_dNewLogP *. dLogP_dMean # [5] + let actorGrads = mlpBackward(ac.actor, actorFwd, tr.state, gradActorOut) + + dActorW1 += actorGrads.dw1 + dActorB1 += actorGrads.db1 + dActorW2 += actorGrads.dw2 + dActorB2 += actorGrads.db2 + dActorW3 += actorGrads.dw3 + dActorB3 += actorGrads.db3 + + # ── Critic forward + loss ── + let criticFwd = mlpForwardCached(ac.critic, tr.state) + let newVal = criticFwd.y[0] + # Value loss = (newVal - ret)^2; d/d(newVal) = 2*(newVal-ret) + let dVLoss_dVal = valueLossCoeff * 2.0'f32 * (newVal - ret) / mbSize.float32 + let gradCriticOut = [dVLoss_dVal].toTensor() # [1] + let criticGrads = mlpBackward(ac.critic, criticFwd, tr.state, gradCriticOut) + + dCriticW1 += criticGrads.dw1 + dCriticB1 += criticGrads.db1 + dCriticW2 += criticGrads.dw2 + dCriticB2 += criticGrads.db2 + dCriticW3 += criticGrads.dw3 + dCriticB3 += criticGrads.db3 + + # ── Gradient clipping ── + # Collect all grads into a seq for norm computation + var allGrads: seq[Tensor[float32]] = @[ + dActorW1, dActorB1, dActorW2, dActorB2, dActorW3, dActorB3, + dLogStd, + dCriticW1, dCriticB1, dCriticW2, dCriticB2, dCriticW3, dCriticB3 + ] + let norm = globalNorm(allGrads) + if norm > maxGradNorm: + let scale = maxGradNorm / norm + for g in allGrads.mitems: g = g *. scale + + # Unpack clipped grads + dActorW1 = allGrads[0]; dActorB1 = allGrads[1] + dActorW2 = allGrads[2]; dActorB2 = allGrads[3] + dActorW3 = allGrads[4]; dActorB3 = allGrads[5] + dLogStd = allGrads[6] + dCriticW1 = allGrads[7]; dCriticB1 = allGrads[8] + dCriticW2 = allGrads[9]; dCriticB2 = allGrads[10] + dCriticW3 = allGrads[11]; dCriticB3 = allGrads[12] + + # ── Adam updates ── + adamStep(ac.actor.w1, dActorW1, gAdamStates.aw1, lr) + adamStep(ac.actor.b1, dActorB1, gAdamStates.ab1, lr) + adamStep(ac.actor.w2, dActorW2, gAdamStates.aw2, lr) + adamStep(ac.actor.b2, dActorB2, gAdamStates.ab2, lr) + adamStep(ac.actor.w3, dActorW3, gAdamStates.aw3, lr) + adamStep(ac.actor.b3, dActorB3, gAdamStates.ab3, lr) + adamStep(ac.logStd, dLogStd, gAdamStates.logStd, lr) + adamStep(ac.critic.w1, dCriticW1, gAdamStates.cw1, lr) + adamStep(ac.critic.b1, dCriticB1, gAdamStates.cb1, lr) + adamStep(ac.critic.w2, dCriticW2, gAdamStates.cw2, lr) + adamStep(ac.critic.b2, dCriticB2, gAdamStates.cb2, lr) + adamStep(ac.critic.w3, dCriticW3, gAdamStates.cw3, lr) + adamStep(ac.critic.b3, dCriticB3, gAdamStates.cb3, lr) + + mbStart = mbEnd