diff --git a/PPO_Bot/PPO_Bot.nim b/PPO_Bot/PPO_Bot.nim index 97c4d66..3aa3fad 100644 --- a/PPO_Bot/PPO_Bot.nim +++ b/PPO_Bot/PPO_Bot.nim @@ -1,7 +1,7 @@ ## PPO_Bot — enemy tracker + state vector wired into the game loop. ## Training: trajectory collected per tick, PPO update in background thread. -import std/[os, strformat, strutils, math] +import std/[os, strformat, strutils, math, times] import arraymancer import tankroyale_botapi import network @@ -11,6 +11,51 @@ import weights import ./enemy_tracker import ./state_vector +# ── Hyperparameters from env vars (PPOB_ prefix) ───────────────────────────── +# All optional; defaults match ppoUpdate signature in training.nim. + +proc getEnvFloat(name: string, default: float32): float32 = + let v = getEnv(name) + if v.len == 0: default else: parseFloat(v).float32 + +proc getEnvInt(name: string, default: int): int = + let v = getEnv(name) + if v.len == 0: default else: parseInt(v) + +var + hpLr: float32 = getEnvFloat("PPOB_LR", 3e-4'f32) + hpClipEpsilon: float32 = getEnvFloat("PPOB_CLIP_EPSILON", 0.2'f32) + hpEntropyCoeff: float32 = getEnvFloat("PPOB_ENTROPY_COEFF", 0.01'f32) + hpValueLossCoeff: float32 = getEnvFloat("PPOB_VALUE_LOSS_COEFF", 0.5'f32) + hpMaxGradNorm: float32 = getEnvFloat("PPOB_MAX_GRAD_NORM", 0.5'f32) + hpGamma: float32 = getEnvFloat("PPOB_GAMMA", 0.99'f32) + hpLam: float32 = getEnvFloat("PPOB_LAM", 0.95'f32) + hpEpochs: int = getEnvInt("PPOB_EPOCHS", 4) + hpMiniBatchSize: int = getEnvInt("PPOB_MINI_BATCH_SIZE", 64) + +# Wire logStd tunable params into network module vars (read before initActorCritic) +logStdFloor = getEnvFloat("PPOB_LOG_STD_FLOOR", -3.0'f32) +initialLogStd = getEnvFloat("PPOB_INITIAL_LOG_STD", 0.0'f32) + +# ── Structured log output ───────────────────────────────────────────────────── + +let logFile = getEnv("PPOB_LOG_FILE") # empty → no JSON logging + +proc appendJsonLine(path, line: string) = + ## Append a JSON line to path; no-op if path is empty. + if path.len == 0: return + let f = open(path, fmAppend) + f.writeLine(line) + f.close() + +proc hyperparmSnapshot(): string = + ## Compact JSON object of current hyperparams (no outer braces). + &"\"lr\":{hpLr},\"clipEpsilon\":{hpClipEpsilon}," & + &"\"entropyCoeff\":{hpEntropyCoeff},\"valueLossCoeff\":{hpValueLossCoeff}," & + &"\"maxGradNorm\":{hpMaxGradNorm},\"gamma\":{hpGamma},\"lam\":{hpLam}," & + &"\"epochs\":{hpEpochs},\"miniBatchSize\":{hpMiniBatchSize}," & + &"\"logStdFloor\":{logStdFloor},\"initialLogStd\":{initialLogStd}" + const botJsonPath = currentSourcePath().parentDir / "PPO_Bot.json" const weightsRoot = currentSourcePath().parentDir / "weights" @@ -40,12 +85,22 @@ type metrics: PPOMetrics TrainingArgs = object - ac: ActorCritic - adamStates: ACAdamStates - buffer: TrajectoryBuffer - lastValue: float32 - roundNum: int - weightsRoot: string + ac: ActorCritic + adamStates: ACAdamStates + buffer: TrajectoryBuffer + lastValue: float32 + roundNum: int + weightsRoot: string + # hyperparams snapshot at launch time + lr: float32 + clipEpsilon: float32 + entropyCoeff: float32 + valueLossCoeff: float32 + maxGradNorm: float32 + gamma: float32 + lam: float32 + epochs: int + miniBatchSize: int var trainingThread: Thread[TrainingArgs] @@ -56,7 +111,18 @@ var proc trainingThreadProc(args: TrainingArgs) {.thread.} = var localAc = args.ac var localAdam = args.adamStates - let m = ppoUpdate(localAc, args.buffer, lastValue = args.lastValue, adamStates = localAdam) + let m = ppoUpdate(localAc, args.buffer, + lastValue = args.lastValue, + adamStates = localAdam, + epochs = args.epochs, + miniBatchSize = args.miniBatchSize, + clipEpsilon = args.clipEpsilon, + entropyCoeff = args.entropyCoeff, + valueLossCoeff = args.valueLossCoeff, + lr = args.lr, + maxGradNorm = args.maxGradNorm, + gamma = args.gamma, + lam = args.lam) saveCheckpoint(localAc, localAdam, args.weightsRoot, args.roundNum) resultChan.send(TrainingResult(ac: localAc, adamStates: localAdam, metrics: m)) @@ -87,10 +153,11 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = # Training progress display — one line per round in the UI console let ticks = bot.buffer.len + var avgR = 0.0'f32 if ticks > 0: var rewardSum = 0.0'f32 for tr in bot.buffer.transitions: rewardSum += tr.reward - let avgR = rewardSum / ticks.float32 + avgR = rewardSum / ticks.float32 let avgRStr = formatFloat(avgR.float, ffDecimal, 3) printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n") echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}" @@ -105,6 +172,11 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = let m = trained.metrics printToStdOut(&" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n") echo &" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}" + # Emit training-health JSON line + let hp = hyperparmSnapshot() + let ts = int(epochTime()) + let jline = &"""{{\"type\":\"train\",\"round\":{roundCounter-1},\"actorLoss\":{m.actorLoss},\"valueLoss\":{m.valueLoss},\"gradNorm\":{m.gradNorm},\"ts\":{ts},{hp}}}""" + appendJsonLine(logFile, jline) if bot.buffer.len == 0: bot.hasLastTrans = false @@ -117,13 +189,27 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = bot.hasLastTrans = false return + # Emit per-round game-stats JSON line (training health will follow when thread finishes) + let ts = int(epochTime()) + let jline = &"""{{\"type\":\"round\",\"round\":{roundCounter},\"ticks\":{ticks},\"avgReward\":{avgR},\"score\":{e.results.totalScore},\"ts\":{ts}}}""" + appendJsonLine(logFile, jline) + let args = TrainingArgs( - ac: ac, - adamStates: gAdamStates, - buffer: bot.buffer, - lastValue: 0.0'f32, - roundNum: roundCounter, - weightsRoot: weightsRoot, + ac: ac, + adamStates: gAdamStates, + buffer: bot.buffer, + lastValue: 0.0'f32, + roundNum: roundCounter, + weightsRoot: weightsRoot, + lr: hpLr, + clipEpsilon: hpClipEpsilon, + entropyCoeff: hpEntropyCoeff, + valueLossCoeff: hpValueLossCoeff, + maxGradNorm: hpMaxGradNorm, + gamma: hpGamma, + lam: hpLam, + epochs: hpEpochs, + miniBatchSize: hpMiniBatchSize, ) bot.buffer.clear() bot.hasLastTrans = false @@ -134,9 +220,13 @@ method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = threadLaunched = true method run(bot: PPOBot) = - # Seed energy on first tick + # Seed energy and goto/aimTo targets on first tick (remainingDistance = 0 initially) bot.prevEnergy = getEnergy().float32 bot.prevEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: 0.0'f32 + bot.lastActions.gotoX = getX() + bot.lastActions.gotoY = getY() + bot.lastActions.aimToX = getX() + bot.lastActions.aimToY = getY() while isRunning(): bot.tracker.deadReckon() @@ -163,18 +253,36 @@ method run(bot: PPOBot) = let state = buildStateVector(botData, bot.tracker, remainingGotoDistance, remainingGunAngle) let (rawActs, logP) = ac.actorForward(state) let value = ac.criticForward(state) + let ex = if bot.tracker.hasContact: bot.tracker.current.x else: botData.arenaWidth / 2.0 + let ey = if bot.tracker.hasContact: bot.tracker.current.y else: botData.arenaHeight / 2.0 let acts = mapActions(rawActs, getGunHeat().float, botData.arenaWidth, botData.arenaHeight, botData.x, botData.y, - botData.direction, botData.speed, botData.gunDirection) + botData.direction, botData.speed, botData.gunDirection, + ex, ey) - # Compute tick reward from energy deltas + # Compute tick reward from energy deltas + dense shaping let curEnergy = getEnergy().float32 let curEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: bot.prevEnemyE let myDelta = curEnergy - bot.prevEnergy let enemyDelta = curEnemyE - bot.prevEnemyE - let tickReward = computeTickReward(myDelta, enemyDelta) + let arenaDiag = float32(sqrt(botData.arenaWidth * botData.arenaWidth + + botData.arenaHeight * botData.arenaHeight)) + let distEnemy = if bot.tracker.hasContact: + float32(hypot(bot.tracker.current.x - botData.x, + bot.tracker.current.y - botData.y)) + else: arenaDiag + let gunToEnemy = if bot.tracker.hasContact: + abs(normalizeRelativeAngle( + arctan2(bot.tracker.current.y - botData.y, + bot.tracker.current.x - botData.x) * 180.0 / PI - + botData.gunDirection)).float32 + else: 180.0'f32 + let tickReward = computeTickReward(myDelta, enemyDelta, + distToEnemy = distEnemy, + maxDist = arenaDiag, + gunBearingAbs = gunToEnemy) # Track running reward for in-game display bot.roundRewardSum += tickReward diff --git a/PPO_Bot/actions.nim b/PPO_Bot/actions.nim index 8e87375..a2be5ea 100644 --- a/PPO_Bot/actions.nim +++ b/PPO_Bot/actions.nim @@ -21,13 +21,17 @@ type proc mapActions*(rawActions: Tensor[float32], gunHeat: float, arenaWidth, arenaHeight: float, - botX, botY, direction, speed, gunDirection: float): BotActions = + botX, botY, direction, speed, gunDirection: float, + enemyX, enemyY: float): BotActions = ## rawActions: [6] tensor from actorForward. - ## Dims 0–1: goto x/y, 2–3: aimTo x/y, 4: fire decision, 5: fire power. - let gotoX = sigmoid(rawActions[0].float) * arenaWidth - let gotoY = sigmoid(rawActions[1].float) * arenaHeight - let aimToX = sigmoid(rawActions[2].float) * arenaWidth - let aimToY = sigmoid(rawActions[3].float) * arenaHeight + ## Dims 0–1: goto x/y offset from enemy, 2–3: aimTo x/y offset from enemy, + ## 4: fire decision, 5: fire power. + ## Enemy-centred mapping: tanh gives [-1,1]; scale by arena/4 (goto) and + ## arena/8 (aimTo) so zero-init defaults the bot toward the enemy. + let gotoX = clamp(enemyX + tanh(rawActions[0].float) * arenaWidth * 0.25, 0.0, arenaWidth) + let gotoY = clamp(enemyY + tanh(rawActions[1].float) * arenaHeight * 0.25, 0.0, arenaHeight) + let aimToX = clamp(enemyX + tanh(rawActions[2].float) * arenaWidth * 0.125, 0.0, arenaWidth) + let aimToY = clamp(enemyY + tanh(rawActions[3].float) * arenaHeight * 0.125, 0.0, arenaHeight) let fireDec = tanh(rawActions[4].float) let fp = sigmoid(rawActions[5].float) * 2.9 + 0.1 diff --git a/PPO_Bot/network.nim b/PPO_Bot/network.nim index cbbe3a9..7c1517b 100644 --- a/PPO_Bot/network.nim +++ b/PPO_Bot/network.nim @@ -7,6 +7,10 @@ const STATE_DIM* = 44 ACTION_DIM* = 6 +var + logStdFloor*: float32 = -3.0'f32 # overridden by PPOB_LOG_STD_FLOOR + initialLogStd*: float32 = 0.0'f32 # overridden by PPOB_INITIAL_LOG_STD + type MLP* = object w1*, b1*: Tensor[float32] # [hidden, input], [hidden] @@ -30,7 +34,7 @@ proc initMLP*(inputDim, hiddenDim, outputDim: int): MLP = proc initActorCritic*(): ActorCritic = result.actor = initMLP(STATE_DIM, 64, ACTION_DIM) result.critic = initMLP(STATE_DIM, 64, 1) - result.logStd = zeros[float32](ACTION_DIM) # init to 0 → std=1 + result.logStd = newTensor[float32](ACTION_DIM).map(proc(v: float32): float32 = initialLogStd) proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] = ## x shape: [inputDim] (1D vector) @@ -41,8 +45,8 @@ proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] = proc actorForward*(ac: ActorCritic, state: Tensor[float32]): tuple[actions: Tensor[float32], logProb: float32] = ## state: [STATE_DIM]. Returns sampled actions [ACTION_DIM] and sum log-prob. let mean = ac.actor.forward(state) - # Floor logStd at -3 before exp → min std ≈ 0.05 - var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32)) + # Floor logStd at logStdFloor before exp → min std ≈ exp(logStdFloor) + var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, logStdFloor)) let std = clampedLogStd.map(proc(v: float32): float32 = exp(v)) var actions = newTensor[float32](ACTION_DIM) @@ -66,7 +70,7 @@ proc criticForward*(ac: ActorCritic, state: Tensor[float32]): float32 = proc computeLogProb*(ac: ActorCritic, state, action: Tensor[float32]): float32 = ## Log-probability of action under current policy (no sampling). let mean = ac.actor.forward(state) - let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32)) + let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, logStdFloor)) let std = logStdClamped.map(proc(v: float32): float32 = exp(v)) var logP = 0.0'f32 for i in 0.. -3: # = (action_i - mean_i)^2/std_i^2 - 1 for i in 0..