diff --git a/GotoTest/.gitignore b/GotoTest/.gitignore new file mode 100644 index 0000000..f3685e2 --- /dev/null +++ b/GotoTest/.gitignore @@ -0,0 +1,3 @@ +nimble.develop +nimble.paths +nimbledeps diff --git a/GotoTest/GotoTest b/GotoTest/GotoTest new file mode 100755 index 0000000..344351c Binary files /dev/null and b/GotoTest/GotoTest differ diff --git a/GotoTest/GotoTest.json b/GotoTest/GotoTest.json new file mode 100644 index 0000000..75e28c1 --- /dev/null +++ b/GotoTest/GotoTest.json @@ -0,0 +1,11 @@ +{ + "name": "GotoTest", + "version": "0.1.0", + "authors": ["Davide Cappellini"], + "description": "Throwaway goto(x,y) controller prototype", + "homepage": "", + "countryCodes": ["IT"], + "gameTypes": ["classic", "melee", "1v1"], + "platform": "Nim", + "programmingLang": "Nim" +} diff --git a/GotoTest/GotoTest.nim b/GotoTest/GotoTest.nim new file mode 100644 index 0000000..a26aa25 --- /dev/null +++ b/GotoTest/GotoTest.nim @@ -0,0 +1,95 @@ +## GotoTest — throwaway prototype to validate goto(x,y) + aimTo(x,y) controllers. +## Diamond pattern with wall-smash north waypoint to test stuck recovery. +## Tank Royale: Y=0 is south, Y increases northward. Arena 800x600. + +import std/[strformat, os] +import tankroyale_botapi + +const botJsonPath = currentSourcePath().parentDir / "GotoTest.json" + +# Diamond waypoints. (400,600) is AT the north wall — physically impossible, tests wall recovery. +const waypoints = [ + (400.0, 300.0), # center + (400.0, 600.0), # north wall — AT wall, unreachable, tests wall-stuck recovery + ( 36.0, 300.0), # west wall (left) + (400.0, 36.0), # south wall (bottom) + (764.0, 300.0), # east wall (right) + (400.0, 300.0), # center +] + +type GotoBot = ref object of Bot + waypointIdx: int + oldX, oldY: float # position 3 ticks ago for stuck detection + tickCount: int # ticks since oldX/oldY was last updated + stuckTicks: int # remaining ticks of reverse override + + +method onRoundStarted*(bot: GotoBot, e: RoundStartedEvent) = + setAdjustGunForBodyTurn(true) + bot.waypointIdx = 0 + bot.oldX = 0.0; bot.oldY = 0.0 + bot.tickCount = 0; bot.stuckTicks = 0 + +method run(bot: GotoBot) = + while isRunning(): + let bx = getX() + let by = getY() + let (tx, ty) = waypoints[bot.waypointIdx] + let dist = distanceTo(bx, by, tx, ty) + + if dist < 40.0: + bot.waypointIdx = (bot.waypointIdx + 1) mod waypoints.len + + # goto controller — explicit forward/reverse proportional steering + # ponytail: no speed-taper on heading error; add when overshooting observed + let rawBearing = normalizeRelativeAngle(directionTo(tx, ty) - getDirection()) + let (dirSign, effBearing) = + if abs(rawBearing) > 90.0: + (-1.0, normalizeRelativeAngle(rawBearing + 180.0)) + else: + (1.0, rawBearing) + let maxTurn = calcMaxTurnRate(getSpeed()) + setTurnRate(effBearing.clamp(-maxTurn, maxTurn)) + + # stuck detector — ponytail: 3-tick sample, upgrade to wall-nav if needed + inc bot.tickCount + if bot.tickCount >= 3: + let moved = distanceTo(bot.oldX, bot.oldY, bx, by) + if moved < 1.0 and dist >= 30.0: + bot.stuckTicks = 6 + echo &"STUCK at ({bx:.1f},{by:.1f}) moved={moved:.2f} — reversing" + bot.oldX = bx; bot.oldY = by + bot.tickCount = 0 + + if bot.stuckTicks > 0: + # reverse current speed direction to unstick + let targetSpd = if getSpeed() >= 0.0: -8.0 else: 8.0 + setTargetSpeed(targetSpd) + dec bot.stuckTicks + else: + let rawSpeed = getNewTargetSpeed(MAX_SPEED, abs(getSpeed()), dist) + setTargetSpeed(dirSign * rawSpeed) + + # aimTo arena center — radarBearingTo reuses getX/getY/getGunDirection internally + let cx = getArenaWidth().float / 2.0 + let cy = getArenaHeight().float / 2.0 + let gunTurnNeeded = normalizeRelativeAngle(directionTo(cx, cy) - getGunDirection()) + setGunTurnRate(gunTurnNeeded.clamp(-MAX_GUN_TURN_RATE, MAX_GUN_TURN_RATE)) + + echo &"pos=({bx:.1f},{by:.1f}) target=({tx:.1f},{ty:.1f}) dist={dist:.1f} spd={getSpeed():.2f} stuck={bot.stuckTicks}" + + # Debug graphics: waypoint circles + line to current target + for i, (wx, wy) in waypoints: + if i == bot.waypointIdx: + setStrokeColor(RED) + else: + setStrokeColor(GRAY) + drawCircle(wx, wy, 10.0) + setStrokeColor(YELLOW) + drawLine(bx, by, tx, ty) + + go() + +when isMainModule: + var bot = GotoBot() + start(bot, botJsonPath) diff --git a/GotoTest/GotoTest.nimble b/GotoTest/GotoTest.nimble new file mode 100644 index 0000000..ef52289 --- /dev/null +++ b/GotoTest/GotoTest.nimble @@ -0,0 +1,10 @@ +# Package +version = "0.1.0" +author = "Davide Cappellini" +description = "GotoTest — throwaway goto(x,y) controller prototype" +license = "MIT" +bin = @["GotoTest"] + +# Dependencies +requires "nim >= 2.0.0" +requires "tankroyale_botapi >= 1.0.0" diff --git a/GotoTest/config.nims b/GotoTest/config.nims new file mode 100644 index 0000000..8ee48d2 --- /dev/null +++ b/GotoTest/config.nims @@ -0,0 +1,4 @@ +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/PPO_Bot/PPO_Bot b/PPO_Bot/PPO_Bot index 12b7567..47aa1be 100755 Binary files a/PPO_Bot/PPO_Bot and b/PPO_Bot/PPO_Bot differ diff --git a/PPO_Bot/PPO_Bot.nim b/PPO_Bot/PPO_Bot.nim index 675f9d5..7f5aed8 100644 --- a/PPO_Bot/PPO_Bot.nim +++ b/PPO_Bot/PPO_Bot.nim @@ -24,6 +24,7 @@ type PPOBot = ref object of Bot lastLogP: float32 lastValue: float32 hasLastTrans: bool + lastActions: BotActions # previous tick's decoded actions (for state vector) roundRewardSum: float32 # cumulative reward this round (for live display) roundTicks: int # ticks this round @@ -36,6 +37,7 @@ type TrainingResult = object ac: ActorCritic adamStates: ACAdamStates + metrics: PPOMetrics TrainingArgs = object ac: ActorCritic @@ -54,9 +56,9 @@ var proc trainingThreadProc(args: TrainingArgs) {.thread.} = var localAc = args.ac var localAdam = args.adamStates - ppoUpdate(localAc, args.buffer, lastValue = args.lastValue, adamStates = localAdam) + let m = ppoUpdate(localAc, args.buffer, lastValue = args.lastValue, adamStates = localAdam) saveCheckpoint(localAc, args.weightsRoot, args.roundNum) - resultChan.send(TrainingResult(ac: localAc, adamStates: localAdam)) + resultChan.send(TrainingResult(ac: localAc, adamStates: localAdam, metrics: m)) # ── Bot methods ─────────────────────────────────────────────────────────────── @@ -91,6 +93,7 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = let avgR = rewardSum / ticks.float32 let avgRStr = formatFloat(avgR.float, ffDecimal, 3) printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n") + echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}" # Pick up result from previous training thread if available; channel IS the sync if threadLaunched: @@ -99,6 +102,9 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = ac = trained.ac gAdamStates = trained.adamStates threadLaunched = false + let m = trained.metrics + printToStdOut(&" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n") + echo &" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}" if bot.buffer.len == 0: bot.hasLastTrans = false @@ -122,6 +128,8 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = bot.buffer.clear() bot.hasLastTrans = false + printToStdOut(&" train→ R:{roundCounter} ticks:{ticks}\n") + echo &" train→ R:{roundCounter} ticks:{ticks}" createThread(trainingThread, trainingThreadProc, args) threadLaunched = true @@ -147,10 +155,19 @@ method run(bot: PPOBot) = arenaHeight: float64(getArenaHeight()), ) - let state = buildStateVector(botData, bot.tracker) + let remainingGotoDistance = hypot(bot.lastActions.gotoX - botData.x, + bot.lastActions.gotoY - botData.y) + let remainingGunAngle = abs(normalizeRelativeAngle( + directionTo(botData.x, botData.y, bot.lastActions.aimToX, bot.lastActions.aimToY) - + botData.gunDirection)) + let state = buildStateVector(botData, bot.tracker, remainingGotoDistance, remainingGunAngle) let (rawActs, logP) = ac.actorForward(state) let value = ac.criticForward(state) - let acts = mapActions(rawActs, getSpeed().float32, getGunHeat().float32) + let acts = mapActions(rawActs, + getGunHeat().float, + botData.arenaWidth, botData.arenaHeight, + botData.x, botData.y, + botData.direction, botData.speed, botData.gunDirection) # Compute tick reward from energy deltas let curEnergy = getEnergy().float32 @@ -182,6 +199,7 @@ method run(bot: PPOBot) = bot.prevEnergy = curEnergy bot.prevEnemyE = curEnemyE bot.hasLastTrans = true + bot.lastActions = acts setTargetSpeed(acts.targetSpeed.float) setTurnRate(acts.turnRate.float) diff --git a/PPO_Bot/actions.nim b/PPO_Bot/actions.nim index 30e0cb7..8e87375 100644 --- a/PPO_Bot/actions.nim +++ b/PPO_Bot/actions.nim @@ -2,32 +2,44 @@ import arraymancer import std/math +import ./controllers + +func sigmoid(x: float): float = 1.0 / (1.0 + exp(-x)) type BotActions* = object - targetSpeed*: float32 - turnRate*: float32 - gunTurnRate*: float32 + targetSpeed*: float + turnRate*: float + gunTurnRate*: float shouldFire*: bool - firePower*: float32 + firePower*: float + gotoX*: float + gotoY*: float + aimToX*: float + aimToY*: float -proc mapActions*(rawActions: Tensor[float32], currentSpeed: float32, gunHeat: float32): BotActions = - ## rawActions: [5] tensor from actorForward. - ## Ranges: - ## targetSpeed ∈ [-8, 8] - ## turnRate ∈ [-(10 - 0.75|v|), (10 - 0.75|v|)] - ## gunTurnRate ∈ [-20, 20] - ## firePower ∈ [0.1, 3.0] - let r0 = tanh(rawActions[0].float64).float32 - let r1 = tanh(rawActions[1].float64).float32 - let r2 = tanh(rawActions[2].float64).float32 - let r3 = tanh(rawActions[3].float64).float32 - let r4 = rawActions[4] +proc mapActions*(rawActions: Tensor[float32], + gunHeat: float, + arenaWidth, arenaHeight: float, + botX, botY, direction, speed, gunDirection: float): BotActions = + ## rawActions: [6] tensor from actorForward. + ## Dims 0–1: goto x/y, 2–3: aimTo x/y, 4: fire decision, 5: fire power. + let gotoX = sigmoid(rawActions[0].float) * arenaWidth + let gotoY = sigmoid(rawActions[1].float) * arenaHeight + let aimToX = sigmoid(rawActions[2].float) * arenaWidth + let aimToY = sigmoid(rawActions[3].float) * arenaHeight + let fireDec = tanh(rawActions[4].float) + let fp = sigmoid(rawActions[5].float) * 2.9 + 0.1 - result.targetSpeed = r0 * 8.0'f32 - result.turnRate = r1 * (10.0'f32 - 0.75'f32 * abs(currentSpeed)) - result.gunTurnRate = r2 * 20.0'f32 - let fireDecision = r3 - result.shouldFire = fireDecision >= 0.0'f32 and gunHeat <= 0.0'f32 - # sigmoid(r4) * 2.9 + 0.1 → [0.1, 3.0] - result.firePower = (1.0'f32 / (1.0'f32 + exp(-r4.float64).float32)) * 2.9'f32 + 0.1'f32 + let (ts, tr) = gotoTick(gotoX, gotoY, botX, botY, direction, speed) + let gtr = aimToTick(aimToX, aimToY, botX, botY, gunDirection) + + result.gotoX = gotoX + result.gotoY = gotoY + result.aimToX = aimToX + result.aimToY = aimToY + result.targetSpeed = ts + result.turnRate = tr + result.gunTurnRate = gtr + result.shouldFire = fireDec >= 0.0 and gunHeat <= 0.0 + result.firePower = fp diff --git a/PPO_Bot/controllers.nim b/PPO_Bot/controllers.nim new file mode 100644 index 0000000..df44b98 --- /dev/null +++ b/PPO_Bot/controllers.nim @@ -0,0 +1,24 @@ +## Pure tick-level controllers for goto(x,y) and aimTo(x,y). +## No bot object needed — all inputs are explicit parameters. + +import tankroyale_botapi + +proc gotoTick*(targetX, targetY, botX, botY, direction, speed: float): (float, float) = + ## Returns (targetSpeed, turnRate) to drive toward (targetX, targetY). + ## Selects forward or reverse automatically based on bearing. + let bearing = normalizeRelativeAngle(directionTo(botX, botY, targetX, targetY) - direction) + let (dirSign, effBearing) = + if abs(bearing) > 90.0: + (-1.0, normalizeRelativeAngle(bearing + 180.0)) + else: + (1.0, bearing) + let dist = distanceTo(botX, botY, targetX, targetY) + let maxTurn = calcMaxTurnRate(speed) + let turnRate = effBearing.clamp(-maxTurn, maxTurn) + let rawSpeed = getNewTargetSpeed(MAX_SPEED, abs(speed), dist) + (dirSign * rawSpeed, turnRate) + +proc aimToTick*(targetX, targetY, botX, botY, gunDirection: float): float = + ## Returns gunTurnRate (clamped to ±MAX_GUN_TURN_RATE) to rotate gun toward target. + let bearing = normalizeRelativeAngle(directionTo(botX, botY, targetX, targetY) - gunDirection) + bearing.clamp(-MAX_GUN_TURN_RATE, MAX_GUN_TURN_RATE) diff --git a/PPO_Bot/network.nim b/PPO_Bot/network.nim index 5730ce6..cbbe3a9 100644 --- a/PPO_Bot/network.nim +++ b/PPO_Bot/network.nim @@ -3,6 +3,10 @@ import arraymancer import std/[math, random] +const + STATE_DIM* = 44 + ACTION_DIM* = 6 + type MLP* = object w1*, b1*: Tensor[float32] # [hidden, input], [hidden] @@ -12,7 +16,7 @@ type ActorCritic* = object actor*: MLP critic*: MLP - logStd*: Tensor[float32] # [5] — one per action dim + logStd*: Tensor[float32] # [ACTION_DIM] — one per action dim proc initMLP*(inputDim, hiddenDim, outputDim: int): MLP = # Xavier/He-style init: scale weights by sqrt(2/fan_in) @@ -24,9 +28,9 @@ proc initMLP*(inputDim, hiddenDim, outputDim: int): MLP = result.b3 = zeros[float32](outputDim) proc initActorCritic*(): ActorCritic = - result.actor = initMLP(42, 64, 5) - result.critic = initMLP(42, 64, 1) - result.logStd = zeros[float32](5) # init to 0 → std=1 + result.actor = initMLP(STATE_DIM, 64, ACTION_DIM) + result.critic = initMLP(STATE_DIM, 64, 1) + result.logStd = zeros[float32](ACTION_DIM) # init to 0 → std=1 proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] = ## x shape: [inputDim] (1D vector) @@ -35,15 +39,15 @@ proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] = result = mlp.w3 * h2 + mlp.b3 proc actorForward*(ac: ActorCritic, state: Tensor[float32]): tuple[actions: Tensor[float32], logProb: float32] = - ## state: [42]. Returns sampled actions [5] and sum log-prob. + ## state: [STATE_DIM]. Returns sampled actions [ACTION_DIM] and sum log-prob. let mean = ac.actor.forward(state) # Floor logStd at -3 before exp → min std ≈ 0.05 var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32)) let std = clampedLogStd.map(proc(v: float32): float32 = exp(v)) - var actions = newTensor[float32](5) + var actions = newTensor[float32](ACTION_DIM) var logP = 0.0'f32 - for i in 0..<5: + for i in 0..= 0.0, "gotoX >= 0" + # Large negative raw → sigmoid ≈ 0 → close to 0 + let rawLow = makeRaw([-100.0'f32, -100.0, -100.0, -100.0, 0.0, 0.0]) + let actsLow = mapActions(rawLow, 0.0, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + check actsLow.gotoX >= -1e-9, "gotoX >= 0 (low raw)" + check actsLow.gotoY >= -1e-9, "gotoY >= 0 (low raw)" + +# --- Fire triggers correctly --- +block fireTrigger: + # tanh(positive) >= 0 → fire when gunHeat = 0 + let rawFire = makeRaw([0.0'f32, 0.0, 0.0, 0.0, 1.0, 0.0]) + let actsFire = mapActions(rawFire, 0.0, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + check actsFire.shouldFire, "positive tanh → should fire when gun cool" + + # tanh(negative) < 0 → no fire + let rawNoFire = makeRaw([0.0'f32, 0.0, 0.0, 0.0, -1.0, 0.0]) + let actsNoFire = mapActions(rawNoFire, 0.0, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + check not actsNoFire.shouldFire, "negative tanh → no fire" + + # gunHeat > 0 → no fire even with positive decision + let actsHot = mapActions(rawFire, 0.5, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + check not actsHot.shouldFire, "positive tanh but gun hot → no fire" + +# --- Fire power in [0.1, 3.0] --- +block firePowerRange: + let rawMin = makeRaw([0.0'f32, 0.0, 0.0, 0.0, 0.0, -100.0]) + let rawMax = makeRaw([0.0'f32, 0.0, 0.0, 0.0, 0.0, 100.0]) + let actsMin = mapActions(rawMin, 0.0, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + let actsMax = mapActions(rawMax, 0.0, arenaW, arenaH, 600.0, 400.0, 0.0, 0.0, 0.0) + check actsMin.firePower >= 0.1 - 1e-6, "firePower >= 0.1" + check actsMax.firePower <= 3.0 + 1e-6, "firePower <= 3.0" + +echo "test_actions: all passed" diff --git a/PPO_Bot/tests/test_controllers b/PPO_Bot/tests/test_controllers new file mode 100755 index 0000000..9d93e25 Binary files /dev/null and b/PPO_Bot/tests/test_controllers differ diff --git a/PPO_Bot/tests/test_controllers.nim b/PPO_Bot/tests/test_controllers.nim new file mode 100644 index 0000000..adaaaf0 --- /dev/null +++ b/PPO_Bot/tests/test_controllers.nim @@ -0,0 +1,54 @@ +## Assert-based tests for controllers.nim. +## Run: nim c -r tests/test_controllers.nim + +import std/math +import "../controllers" + +template check(cond: bool, msg: string) = + if not cond: + quit("FAIL: " & msg, 1) + +# --------------------------------------------------------------------------- +# gotoTick tests +# --------------------------------------------------------------------------- + +block forwardMovement: + # Bot at (0,0) facing East (geometric 0°), target due east at (100,0) → bearing=0 → forward + let (spd2, turn2) = gotoTick(100.0, 0.0, 0.0, 0.0, 0.0, 0.0) + check spd2 > 0.0, "forward: targetSpeed should be positive" + check abs(turn2) < 1e-9, "forward: no turn needed when already aimed" + +block reverseMovement: + # Bot at (100,0) facing East (direction=0), target at (0,0) — directly behind + # bearing = normalizeRelativeAngle(180 - 0) = 180 → |bearing|>90 → reverse + let (spd, _) = gotoTick(0.0, 0.0, 100.0, 0.0, 0.0, 0.0) + check spd < 0.0, "reverse: targetSpeed should be negative when target is behind" + +block turnRateClamping: + # Bot at (0,0) facing North (game north = geometric 90°, so direction=90 in geometric) + # Target at (100,0) = East. bearing = normalizeRelativeAngle(0 - 90) = -90 → still ≤90 + # Use large perpendicular target so bearing is 89°, and high speed → small maxTurn + # At speed=8, calcMaxTurnRate = 10 - 0.75*8 = 4° + # Bot facing East (0°), target at angle 89° bearing (just under 90) + let (_, turn) = gotoTick(100.0 * cos(89.0 * PI / 180.0), 100.0 * sin(89.0 * PI / 180.0), 0.0, 0.0, 0.0, 8.0) + check abs(turn) <= 4.0 + 1e-9, "turn rate clamped to calcMaxTurnRate at speed=8 (max 4°)" + +# --------------------------------------------------------------------------- +# aimToTick tests +# --------------------------------------------------------------------------- + +block gunShortestArc: + # Gun facing East (0°), target due north (geometric 90°) → turn left +90° but clamped to 20 + let rate = aimToTick(0.0, 100.0, 0.0, 0.0, 0.0) + check rate > 0.0, "gun shortest arc: should turn toward target" + # Gun facing East (0°), target due south (geometric 270° → normalised -90°) + let rate2 = aimToTick(0.0, -100.0, 0.0, 0.0, 0.0) + check rate2 < 0.0, "gun shortest arc: should turn other way for target behind" + +block gunTurnRateClamping: + # 180° away → clamped to ±20 + let rate = aimToTick(-100.0, 0.0, 0.0, 0.0, 0.0) # target west, gun east + check abs(rate) <= 20.0 + 1e-9, "gun turn rate clamped to ±MAX_GUN_TURN_RATE" + check abs(abs(rate) - 20.0) < 1e-9, "gun turn rate at max when 180° away" + +echo "test_controllers: all passed" diff --git a/PPO_Bot/tests/test_network.nim b/PPO_Bot/tests/test_network.nim index 19dae4f..d0299ea 100644 --- a/PPO_Bot/tests/test_network.nim +++ b/PPO_Bot/tests/test_network.nim @@ -11,16 +11,16 @@ func isNaNF(x: float32): bool = classify(x) == fcNan when isMainModule: # ---- MLP forward shape ---- - let mlp = initMLP(42, 64, 5) - let inp = zeros[float32](42) + let mlp = initMLP(STATE_DIM, 64, ACTION_DIM) + let inp = zeros[float32](STATE_DIM) let mlpOut = mlp.forward(inp) - assert mlpOut.shape[0] == 5, &"MLP output shape wrong: {mlpOut.shape}" + assert mlpOut.shape[0] == ACTION_DIM, &"MLP output shape wrong: {mlpOut.shape}" # ---- ActorCritic actorForward ---- let ac = initActorCritic() - let state = zeros[float32](42) + let state = zeros[float32](STATE_DIM) let (acts, logP) = ac.actorForward(state) - assert acts.shape[0] == 5, &"actorForward actions shape wrong: {acts.shape}" + assert acts.shape[0] == ACTION_DIM, &"actorForward actions shape wrong: {acts.shape}" assert not isNaNF(logP), "logProb is NaN" assert isFiniteF(logP), &"logProb not finite: {logP}" @@ -31,23 +31,20 @@ when isMainModule: # ---- logStd floor: collapsing logStd should not break actorForward ---- var ac2 = initActorCritic() - for i in 0..<5: ac2.logStd[i] = -10.0'f32 + for i in 0..= -8.0'f32 and botActs.targetSpeed <= 8.0'f32, &"targetSpeed out of range: {botActs.targetSpeed}" - let maxTurn = 10.0'f32 - 0.75'f32 * abs(speed) # = 7.0 - assert botActs.turnRate >= -maxTurn and botActs.turnRate <= maxTurn, - &"turnRate out of range: {botActs.turnRate}" - assert botActs.gunTurnRate >= -20.0'f32 and botActs.gunTurnRate <= 20.0'f32, &"gunTurnRate out of range: {botActs.gunTurnRate}" @@ -55,7 +52,7 @@ when isMainModule: &"firePower out of range: {botActs.firePower}" # shouldFire=false when gunHeat > 0 - let noFire = mapActions(raw, speed, 1.0'f32) + let noFire = mapActions(raw, 1.0'f32, 800.0, 600.0, 400.0, 300.0, 0.0, speed.float, 0.0) assert not noFire.shouldFire, "shouldFire should be false when gunHeat > 0" echo "All tests passed" diff --git a/PPO_Bot/tests/test_radar_lock.nim b/PPO_Bot/tests/test_radar_lock.nim index e93de1e..75a6744 100644 --- a/PPO_Bot/tests/test_radar_lock.nim +++ b/PPO_Bot/tests/test_radar_lock.nim @@ -10,16 +10,16 @@ template check(cond: bool, msg: string) = # Bot at arena center; stationary enemy due north (same x, higher y). # Tank Royale: y increases northward. -# True bearing to enemy = 0° (north = 0° in game coords). +# Radar uses math convention (0°=east, CCW+); north = 90° in that system. let botX = 400.0 let botY = 300.0 let enemyX = 400.0 # same x → dx = 0 let enemyY = 500.0 # north of bot → dy > 0 -let trueBearing = 0.0 # north +let trueBearing = 90.0 # north in math convention (0=east, CCW+) -# Radar starts pointing at the enemy (radarDirection = 0°, due north). -var radarDir = 0.0 +# Radar starts pointing at the enemy (radarDirection = 90°, due north in math convention). +var radarDir = 90.0 var tracker = initEnemyTracker() # Prime with contact at the known position diff --git a/PPO_Bot/tests/test_state b/PPO_Bot/tests/test_state index d3be37f..04c93eb 100755 Binary files a/PPO_Bot/tests/test_state and b/PPO_Bot/tests/test_state differ diff --git a/PPO_Bot/tests/test_state.nim b/PPO_Bot/tests/test_state.nim index 401b8d3..58aa857 100644 --- a/PPO_Bot/tests/test_state.nim +++ b/PPO_Bot/tests/test_state.nim @@ -84,7 +84,7 @@ block testStateVectorLength: arenaWidth: 800.0, arenaHeight: 600.0, ) let sv = buildStateVector(bot, t) - check sv.shape == [42], "state vector has 42 elements" + check sv.shape == [44], "state vector has 44 elements" block testStateVectorRange: var t = initEnemyTracker() @@ -95,7 +95,7 @@ block testStateVectorRange: arenaWidth: 800.0, arenaHeight: 600.0, ) let sv = buildStateVector(bot, t) - for i in 0 ..< 42: + for i in 0 ..< 44: check sv[i] >= -2.0f32 and sv[i] <= 2.0f32, &"sv[{i}]={sv[i]} out of [-2,2] range" @@ -155,5 +155,8 @@ block testHistoryPaddedWhenEmpty: let sv = buildStateVector(bot, t) for i in 22 ..< 42: check sv[i] == 0.0f32, &"history slot {i} should be 0 when no contact" + # indices 42-43 (goto inputs) default to 0 when not provided + check sv[42] == 0.0f32, "sv[42] (remainingGotoDistance) should default to 0" + check sv[43] == 0.0f32, "sv[43] (remainingGunAngle) should default to 0" echo "All tests passed" diff --git a/PPO_Bot/tests/test_training.nim b/PPO_Bot/tests/test_training.nim index e96dce3..a3b70fc 100644 --- a/PPO_Bot/tests/test_training.nim +++ b/PPO_Bot/tests/test_training.nim @@ -29,7 +29,7 @@ block testBuffer: var buf = initTrajectoryBuffer() check buf.len == 0, "empty buffer len == 0" - let t1 = Transition(state: zeros[float32](42), action: zeros[float32](5), + let t1 = Transition(state: zeros[float32](STATE_DIM), action: zeros[float32](ACTION_DIM), logProb: -1.0'f32, reward: 0.5'f32, value: 0.3'f32) buf.add(t1) buf.add(t1) @@ -80,14 +80,14 @@ block testPpoUpdate: var buf = initTrajectoryBuffer() for _ in 0..<10: - let s = randomNormalTensor[float32](42) - let a = randomNormalTensor[float32](5) + let s = randomNormalTensor[float32](STATE_DIM) + let a = randomNormalTensor[float32](ACTION_DIM) let lp = ac.computeLogProb(s, a) let v = ac.criticForward(s) buf.add(Transition(state: s, action: a, logProb: lp, reward: 0.1'f32, value: v)) var adam: ACAdamStates - ppoUpdate(ac, buf, lastValue = 0.0'f32, adamStates = adam, epochs = 2, miniBatchSize = 5) + discard ppoUpdate(ac, buf, lastValue = 0.0'f32, adamStates = adam, epochs = 2, miniBatchSize = 5) # Weights should have changed — compare flattened let n = ac.actor.w1.shape[0] * ac.actor.w1.shape[1] diff --git a/PPO_Bot/training.nim b/PPO_Bot/training.nim index bf36684..5e8dcf4 100644 --- a/PPO_Bot/training.nim +++ b/PPO_Bot/training.nim @@ -10,8 +10,8 @@ import ./network type Transition* = object - state*: Tensor[float32] # [42] - action*: Tensor[float32] # [5] + state*: Tensor[float32] # [STATE_DIM] + action*: Tensor[float32] # [ACTION_DIM] logProb*: float32 reward*: float32 value*: float32 # critic estimate at collection time @@ -151,6 +151,13 @@ proc initACAdamStates*(ac: ActorCritic): ACAdamStates = result.logStd = initAdamState(ac.logStd) result.initialized = true +# ── Training metrics ────────────────────────────────────────────────────────── + +type PPOMetrics* = object + actorLoss*: float32 + valueLoss*: float32 + gradNorm*: float32 + # ── Gradient clipping ───────────────────────────────────────────────────────── proc globalNorm(grads: varargs[Tensor[float32]]): float32 = @@ -171,9 +178,14 @@ proc ppoUpdate*(ac: var ActorCritic; entropyCoeff: float32 = 0.01'f32; valueLossCoeff: float32 = 0.5'f32; lr: float32 = 3e-4'f32; - maxGradNorm: float32 = 0.5'f32) {.gcsafe.} = + maxGradNorm: float32 = 0.5'f32): PPOMetrics {.gcsafe.} = if buffer.len == 0: return + var totalActorLoss = 0.0'f32 + var totalValueLoss = 0.0'f32 + var totalGradNorm = 0.0'f32 + var totalMiniBatches = 0 + # Initialise Adam states once; caller persists them across rounds if not adamStates.initialized: adamStates = initACAdamStates(ac) @@ -230,14 +242,14 @@ proc ppoUpdate*(ac: var ActorCritic; # ── Actor forward ── let actorFwd = mlpForwardCached(ac.actor, tr.state) - let newMean = actorFwd.y # [5] + let newMean = actorFwd.y # [ACTION_DIM] let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32)) let std = logStdClamped.map(proc(v: float32): float32 = exp(v)) # New log prob var newLogP = 0.0'f32 - for i in 0..<5: + for i in 0.. -3: # = (action_i - mean_i)^2/std_i^2 - 1 - for i in 0..<5: + for i in 0.. maxGradNorm: let scale = maxGradNorm / norm for g in allGrads.mitems: g = g *. scale @@ -343,3 +359,9 @@ proc ppoUpdate*(ac: var ActorCritic; adamStep(ac.critic.b3, dCriticB3, adamStates.cb3, lr) mbStart = mbEnd + + let totalSamples = (epochs * bufLen).float32 + result.actorLoss = totalActorLoss / totalSamples + result.valueLoss = totalValueLoss / totalSamples + result.gradNorm = if totalMiniBatches > 0: totalGradNorm / totalMiniBatches.float32 + else: 0.0'f32 diff --git a/PPO_Bot/weights/latest/actor_b1.npy b/PPO_Bot/weights/latest/actor_b1.npy index 1c10c11..8409018 100644 Binary files a/PPO_Bot/weights/latest/actor_b1.npy and b/PPO_Bot/weights/latest/actor_b1.npy differ diff --git a/PPO_Bot/weights/latest/actor_b2.npy b/PPO_Bot/weights/latest/actor_b2.npy index 2f1d835..185c4f8 100644 Binary files a/PPO_Bot/weights/latest/actor_b2.npy and b/PPO_Bot/weights/latest/actor_b2.npy differ diff --git a/PPO_Bot/weights/latest/actor_b3.npy b/PPO_Bot/weights/latest/actor_b3.npy index acbb141..918d39d 100644 Binary files a/PPO_Bot/weights/latest/actor_b3.npy and b/PPO_Bot/weights/latest/actor_b3.npy differ diff --git a/PPO_Bot/weights/latest/actor_w1.npy b/PPO_Bot/weights/latest/actor_w1.npy index bc95bb4..0959b5c 100644 Binary files a/PPO_Bot/weights/latest/actor_w1.npy and b/PPO_Bot/weights/latest/actor_w1.npy differ diff --git a/PPO_Bot/weights/latest/actor_w2.npy b/PPO_Bot/weights/latest/actor_w2.npy index 1d68db7..b16aadc 100644 Binary files a/PPO_Bot/weights/latest/actor_w2.npy and b/PPO_Bot/weights/latest/actor_w2.npy differ diff --git a/PPO_Bot/weights/latest/actor_w3.npy b/PPO_Bot/weights/latest/actor_w3.npy index 42050b6..86eec7f 100644 Binary files a/PPO_Bot/weights/latest/actor_w3.npy and b/PPO_Bot/weights/latest/actor_w3.npy differ diff --git a/PPO_Bot/weights/latest/critic_b1.npy b/PPO_Bot/weights/latest/critic_b1.npy index 2445d45..e950f18 100644 Binary files a/PPO_Bot/weights/latest/critic_b1.npy and b/PPO_Bot/weights/latest/critic_b1.npy differ diff --git a/PPO_Bot/weights/latest/critic_b2.npy b/PPO_Bot/weights/latest/critic_b2.npy index df9777c..8a51c4d 100644 Binary files a/PPO_Bot/weights/latest/critic_b2.npy and b/PPO_Bot/weights/latest/critic_b2.npy differ diff --git a/PPO_Bot/weights/latest/critic_b3.npy b/PPO_Bot/weights/latest/critic_b3.npy index 80fdf4a..b2716b2 100644 Binary files a/PPO_Bot/weights/latest/critic_b3.npy and b/PPO_Bot/weights/latest/critic_b3.npy differ diff --git a/PPO_Bot/weights/latest/critic_w1.npy b/PPO_Bot/weights/latest/critic_w1.npy index 4bc8eb7..dc64755 100644 Binary files a/PPO_Bot/weights/latest/critic_w1.npy and b/PPO_Bot/weights/latest/critic_w1.npy differ diff --git a/PPO_Bot/weights/latest/critic_w2.npy b/PPO_Bot/weights/latest/critic_w2.npy index f595f9b..890e3e2 100644 Binary files a/PPO_Bot/weights/latest/critic_w2.npy and b/PPO_Bot/weights/latest/critic_w2.npy differ diff --git a/PPO_Bot/weights/latest/critic_w3.npy b/PPO_Bot/weights/latest/critic_w3.npy index ccb4d77..0bfff90 100644 Binary files a/PPO_Bot/weights/latest/critic_w3.npy and b/PPO_Bot/weights/latest/critic_w3.npy differ diff --git a/PPO_Bot/weights/latest/log_std.npy b/PPO_Bot/weights/latest/log_std.npy index e9ce775..61d2a07 100644 Binary files a/PPO_Bot/weights/latest/log_std.npy and b/PPO_Bot/weights/latest/log_std.npy differ diff --git a/tools/battle_runner/RunBattle.class b/tools/battle_runner/RunBattle.class new file mode 100644 index 0000000..6251411 Binary files /dev/null and b/tools/battle_runner/RunBattle.class differ diff --git a/tools/battle_runner/RunBattle.java b/tools/battle_runner/RunBattle.java index ea736c3..e14bb47 100644 --- a/tools/battle_runner/RunBattle.java +++ b/tools/battle_runner/RunBattle.java @@ -40,9 +40,9 @@ public class RunBattle { idToName.put(p.getId(), p.getName()); System.out.printf(" #%d %s%n", p.getId(), p.getName()); } - System.out.printf("%-6s %-4s %-30s %-30s %-20s%n", - "Turn", "Id", "Name", "RadarDir", "Event"); - System.out.println("-".repeat(95)); + System.out.printf("%-6s %-12s %-21s %-12s %-10s %-10s%n", + "Turn", "Name", "Position", "Direction", "Speed", "RadarDir"); + System.out.println("-".repeat(80)); }); handle.getOnTickEvent().on(owner, event -> { @@ -52,12 +52,16 @@ public class RunBattle { for (var state : event.getBotStates()) if (state.getName() != null) idToName.putIfAbsent(state.getId(), state.getName()); - // Print radar direction for every bot + // Print position/speed/direction for every bot for (var state : event.getBotStates()) { String name = idToName.getOrDefault(state.getId(), state.getName() != null ? state.getName() : "#" + state.getId()); - System.out.printf("%-6d %-4d %-30s %-30.2f%n", - turn, state.getId(), name, state.getRadarDirection()); + System.out.printf("%-6d %-12s pos=(%-6.1f,%-6.1f) dir=%-7.2f spd=%-6.2f radar=%-7.2f%n", + turn, name, + state.getX(), state.getY(), + state.getDirection(), + state.getSpeed(), + state.getRadarDirection()); } // Print any scan events on the same tick diff --git a/tools/battle_runner/RunGotoTest.class b/tools/battle_runner/RunGotoTest.class new file mode 100644 index 0000000..e1d6394 Binary files /dev/null and b/tools/battle_runner/RunGotoTest.class differ diff --git a/tools/battle_runner/RunGotoTest.java b/tools/battle_runner/RunGotoTest.java new file mode 100644 index 0000000..a231ade --- /dev/null +++ b/tools/battle_runner/RunGotoTest.java @@ -0,0 +1,76 @@ +import dev.robocode.tankroyale.runner.*; +import dev.robocode.tankroyale.client.model.*; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * GotoTest battle: GotoTest vs Target (1 round, 500 turns max via observation). + * Watches bot positions to verify diamond completion including wall-smash north waypoint. + */ +public class RunGotoTest { + + static final Map idToName = new ConcurrentHashMap<>(); + + public static void main(String[] args) { + Logger.getLogger("dev.robocode.tankroyale").setLevel(Level.WARNING); + + String gotoDir = requireEnv("GOTO_BOT_DIR"); + String sampleBots = requireEnv("SAMPLE_BOTS_DIR"); + + try (var runner = BattleRunner.create(b -> b.embeddedServer().suppressServerOutput())) { + var setup = BattleSetup.classic(s -> s.setNumberOfRounds(1)); + var bots = List.of( + BotEntry.of(gotoDir), + BotEntry.of(sampleBots + "/Target") + ); + + var owner = new Object(); + try (var handle = runner.startBattleAsync(setup, bots)) { + + handle.getOnGameStarted().on(owner, event -> { + System.out.println("=== GAME STARTED ==="); + for (var p : event.getParticipants()) { + idToName.put(p.getId(), p.getName()); + System.out.printf(" #%d %s%n", p.getId(), p.getName()); + } + }); + + handle.getOnTickEvent().on(owner, event -> { + int turn = event.getTurnNumber(); + for (var state : event.getBotStates()) { + if (state.getName() != null) idToName.putIfAbsent(state.getId(), state.getName()); + String name = idToName.getOrDefault(state.getId(), "#" + state.getId()); + if (name.equals("GotoTest")) { + System.out.printf("T%-5d pos=(%-6.1f,%-6.1f) dir=%-7.2f spd=%-5.2f%n", + turn, state.getX(), state.getY(), + state.getDirection(), state.getSpeed()); + } + } + }); + + handle.getOnRoundEnded().on(owner, event -> + System.out.printf("%n=== ROUND %d ENDED (turn %d) ===%n", + event.getRoundNumber(), event.getTurnNumber())); + + var results = handle.awaitResults(); + System.out.printf("%n=== RESULTS ===%n"); + for (var r : results.getResults()) { + System.out.printf(" #%d %-25s %d pts%n", + r.getRank(), r.getName(), r.getTotalScore()); + } + } + } + } + + static String requireEnv(String name) { + var v = System.getenv(name); + if (v == null || v.isBlank()) { + System.err.println("Error: " + name + " env var not set"); + System.exit(1); + } + return v; + } +}