## PPO_Bot — enemy tracker + state vector wired into the game loop. ## Training: trajectory collected per tick, PPO update in background thread. import std/[os, strformat, strutils, math, times] import arraymancer import tankroyale_botapi import network import actions import training import weights import ./enemy_tracker import ./state_vector # ── Hyperparameters from env vars (PPOB_ prefix) ───────────────────────────── # All optional; defaults match ppoUpdate signature in training.nim. proc getEnvFloat(name: string, default: float32): float32 = let v = getEnv(name) if v.len == 0: default else: parseFloat(v).float32 proc getEnvInt(name: string, default: int): int = let v = getEnv(name) if v.len == 0: default else: parseInt(v) var hpLr: float32 = getEnvFloat("PPOB_LR", 3e-4'f32) hpClipEpsilon: float32 = getEnvFloat("PPOB_CLIP_EPSILON", 0.2'f32) hpEntropyCoeff: float32 = getEnvFloat("PPOB_ENTROPY_COEFF", 0.01'f32) hpValueLossCoeff: float32 = getEnvFloat("PPOB_VALUE_LOSS_COEFF", 0.5'f32) hpMaxGradNorm: float32 = getEnvFloat("PPOB_MAX_GRAD_NORM", 0.5'f32) hpGamma: float32 = getEnvFloat("PPOB_GAMMA", 0.99'f32) hpLam: float32 = getEnvFloat("PPOB_LAM", 0.95'f32) hpEpochs: int = getEnvInt("PPOB_EPOCHS", 4) hpMiniBatchSize: int = getEnvInt("PPOB_MINI_BATCH_SIZE", 64) # Wire logStd tunable params into network module vars (read before initActorCritic) logStdFloor = getEnvFloat("PPOB_LOG_STD_FLOOR", -3.0'f32) initialLogStd = getEnvFloat("PPOB_INITIAL_LOG_STD", 0.0'f32) # ── Structured log output ───────────────────────────────────────────────────── let logFile = getEnv("PPOB_LOG_FILE") # empty → no JSON logging proc appendJsonLine(path, line: string) = ## Append a JSON line to path; no-op if path is empty. if path.len == 0: return let f = open(path, fmAppend) f.writeLine(line) f.close() proc hyperparmSnapshot(): string = ## Compact JSON object of current hyperparams (no outer braces). &"\"lr\":{hpLr},\"clipEpsilon\":{hpClipEpsilon}," & &"\"entropyCoeff\":{hpEntropyCoeff},\"valueLossCoeff\":{hpValueLossCoeff}," & &"\"maxGradNorm\":{hpMaxGradNorm},\"gamma\":{hpGamma},\"lam\":{hpLam}," & &"\"epochs\":{hpEpochs},\"miniBatchSize\":{hpMiniBatchSize}," & &"\"logStdFloor\":{logStdFloor},\"initialLogStd\":{initialLogStd}" const botJsonPath = currentSourcePath().parentDir / "PPO_Bot.json" const weightsRoot = currentSourcePath().parentDir / "weights" type PPOBot = ref object of Bot tracker: EnemyTracker buffer: TrajectoryBuffer prevEnergy: float32 # own energy last tick prevEnemyE: float32 # enemy energy last tick (from tracker) lastState: Tensor[float32] lastAction: Tensor[float32] lastLogP: float32 lastValue: float32 hasLastTrans: bool lastActions: BotActions # previous tick's decoded actions (for state vector) roundRewardSum: float32 # cumulative reward this round (for live display) roundTicks: int # ticks this round var ac = initActorCritic() var gAdamStates: ACAdamStates # persists across rounds # ── Background training state ───────────────────────────────────────────────── type TrainingResult = object ac: ActorCritic adamStates: ACAdamStates metrics: PPOMetrics TrainingArgs = object ac: ActorCritic adamStates: ACAdamStates buffer: TrajectoryBuffer lastValue: float32 roundNum: int weightsRoot: string # hyperparams snapshot at launch time lr: float32 clipEpsilon: float32 entropyCoeff: float32 valueLossCoeff: float32 maxGradNorm: float32 gamma: float32 lam: float32 epochs: int miniBatchSize: int var trainingThread: Thread[TrainingArgs] resultChan: Channel[TrainingResult] threadLaunched: bool = false # true while training thread is running roundCounter: int = 0 proc trainingThreadProc(args: TrainingArgs) {.thread.} = var localAc = args.ac var localAdam = args.adamStates let m = ppoUpdate(localAc, args.buffer, lastValue = args.lastValue, adamStates = localAdam, epochs = args.epochs, miniBatchSize = args.miniBatchSize, clipEpsilon = args.clipEpsilon, entropyCoeff = args.entropyCoeff, valueLossCoeff = args.valueLossCoeff, lr = args.lr, maxGradNorm = args.maxGradNorm, gamma = args.gamma, lam = args.lam) saveCheckpoint(localAc, localAdam, args.weightsRoot, args.roundNum) resultChan.send(TrainingResult(ac: localAc, adamStates: localAdam, metrics: m)) # ── Bot methods ─────────────────────────────────────────────────────────────── method onScannedBot*(bot: PPOBot, e: ScannedBotEvent) = bot.tracker.update(e.x, e.y, e.direction, e.speed, e.energy) method onRoundStarted*(bot: PPOBot, e: RoundStartedEvent) = setAdjustGunForBodyTurn(true) setAdjustRadarForBodyTurn(true) setAdjustRadarForGunTurn(true) bot.tracker = initEnemyTracker() bot.buffer = initTrajectoryBuffer() bot.prevEnergy = 0.0'f32 bot.prevEnemyE = 0.0'f32 bot.hasLastTrans = false bot.roundRewardSum = 0.0'f32 bot.roundTicks = 0 method onRoundEnded*(bot: PPOBot, e: RoundEndedEventForBot) = inc roundCounter # Add round-end score bonus to last transition (if any) let roundReward = computeRoundReward(e.results.totalScore.float32) if bot.hasLastTrans and bot.buffer.len > 0: bot.buffer.transitions[^1].reward += roundReward # Training progress display — one line per round in the UI console let ticks = bot.buffer.len var avgR = 0.0'f32 if ticks > 0: var rewardSum = 0.0'f32 for tr in bot.buffer.transitions: rewardSum += tr.reward avgR = rewardSum / ticks.float32 let avgRStr = formatFloat(avgR.float, ffDecimal, 3) printToStdOut(&"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}\n") echo &"R:{roundCounter} ticks:{ticks} avgR:{avgRStr} score:{e.results.totalScore}" # Pick up result from previous training thread if available; channel IS the sync if threadLaunched: let (avail, trained) = resultChan.tryRecv() if avail: ac = trained.ac gAdamStates = trained.adamStates threadLaunched = false let m = trained.metrics printToStdOut(&" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}\n") echo &" trained R:{roundCounter-1} aLoss:{formatFloat(m.actorLoss.float, ffDecimal, 4)} vLoss:{formatFloat(m.valueLoss.float, ffDecimal, 4)} gNorm:{formatFloat(m.gradNorm.float, ffDecimal, 3)}" # Emit training-health JSON line let hp = hyperparmSnapshot() let ts = int(epochTime()) let jline = &"""{{\"type\":\"train\",\"round\":{roundCounter-1},\"actorLoss\":{m.actorLoss},\"valueLoss\":{m.valueLoss},\"gradNorm\":{m.gradNorm},\"ts\":{ts},{hp}}}""" appendJsonLine(logFile, jline) if bot.buffer.len == 0: bot.hasLastTrans = false return # If last thread still running, drop this pass — start fresh with newer data # ponytail: simple drop; queue if every round must train if threadLaunched: bot.buffer.clear() bot.hasLastTrans = false return # Emit per-round game-stats JSON line (training health will follow when thread finishes) let ts = int(epochTime()) let jline = &"""{{\"type\":\"round\",\"round\":{roundCounter},\"ticks\":{ticks},\"avgReward\":{avgR},\"score\":{e.results.totalScore},\"ts\":{ts}}}""" appendJsonLine(logFile, jline) let args = TrainingArgs( ac: ac, adamStates: gAdamStates, buffer: bot.buffer, lastValue: 0.0'f32, roundNum: roundCounter, weightsRoot: weightsRoot, lr: hpLr, clipEpsilon: hpClipEpsilon, entropyCoeff: hpEntropyCoeff, valueLossCoeff: hpValueLossCoeff, maxGradNorm: hpMaxGradNorm, gamma: hpGamma, lam: hpLam, epochs: hpEpochs, miniBatchSize: hpMiniBatchSize, ) bot.buffer.clear() bot.hasLastTrans = false printToStdOut(&" train→ R:{roundCounter} ticks:{ticks}\n") echo &" train→ R:{roundCounter} ticks:{ticks}" createThread(trainingThread, trainingThreadProc, args) threadLaunched = true method run(bot: PPOBot) = # Seed energy and goto/aimTo targets on first tick (remainingDistance = 0 initially) bot.prevEnergy = getEnergy().float32 bot.prevEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: 0.0'f32 bot.lastActions.gotoX = getX() bot.lastActions.gotoY = getY() bot.lastActions.aimToX = getX() bot.lastActions.aimToY = getY() while isRunning(): bot.tracker.deadReckon() setRadarTurnRate(bot.tracker.getRadarTurnRate(getX(), getY(), getDirection(), getRadarDirection())) let botData = BotStateData( x: getX(), y: getY(), direction: getDirection(), speed: getSpeed(), energy: getEnergy(), gunDirection: getGunDirection(), gunHeat: getGunHeat(), arenaWidth: float64(getArenaWidth()), arenaHeight: float64(getArenaHeight()), ) let remainingGotoDistance = hypot(bot.lastActions.gotoX - botData.x, bot.lastActions.gotoY - botData.y) let remainingGunAngle = abs(normalizeRelativeAngle( directionTo(botData.x, botData.y, bot.lastActions.aimToX, bot.lastActions.aimToY) - botData.gunDirection)) let state = buildStateVector(botData, bot.tracker, remainingGotoDistance, remainingGunAngle) let (rawActs, logP) = ac.actorForward(state) let value = ac.criticForward(state) let ex = if bot.tracker.hasContact: bot.tracker.current.x else: botData.arenaWidth / 2.0 let ey = if bot.tracker.hasContact: bot.tracker.current.y else: botData.arenaHeight / 2.0 let acts = mapActions(rawActs, getGunHeat().float, botData.arenaWidth, botData.arenaHeight, botData.x, botData.y, botData.direction, botData.speed, botData.gunDirection, ex, ey) # Compute tick reward from energy deltas + dense shaping let curEnergy = getEnergy().float32 let curEnemyE = if bot.tracker.hasContact: bot.tracker.current.energy.float32 else: bot.prevEnemyE let myDelta = curEnergy - bot.prevEnergy let enemyDelta = curEnemyE - bot.prevEnemyE let arenaDiag = float32(sqrt(botData.arenaWidth * botData.arenaWidth + botData.arenaHeight * botData.arenaHeight)) let distEnemy = if bot.tracker.hasContact: float32(hypot(bot.tracker.current.x - botData.x, bot.tracker.current.y - botData.y)) else: arenaDiag let gunToEnemy = if bot.tracker.hasContact: abs(normalizeRelativeAngle( arctan2(bot.tracker.current.y - botData.y, bot.tracker.current.x - botData.x) * 180.0 / PI - botData.gunDirection)).float32 else: 180.0'f32 let tickReward = computeTickReward(myDelta, enemyDelta, distToEnemy = distEnemy, maxDist = arenaDiag, gunBearingAbs = gunToEnemy) # Track running reward for in-game display bot.roundRewardSum += tickReward inc bot.roundTicks # Finalise previous transition with the reward from this tick's state change if bot.hasLastTrans: let tr = Transition( state: bot.lastState, action: bot.lastAction, logProb: bot.lastLogP, reward: tickReward, value: bot.lastValue, ) bot.buffer.add(tr) # Store current for next tick bot.lastState = state bot.lastAction = rawActs bot.lastLogP = logP bot.lastValue = value bot.prevEnergy = curEnergy bot.prevEnemyE = curEnemyE bot.hasLastTrans = true bot.lastActions = acts setTargetSpeed(acts.targetSpeed.float) setTurnRate(acts.turnRate.float) setGunTurnRate(acts.gunTurnRate.float) if acts.shouldFire: discard setFire(acts.firePower.float) # In-game training progress overlay let avgR = if bot.roundTicks > 0: bot.roundRewardSum / bot.roundTicks.float32 else: 0.0'f32 let avgRStr = formatFloat(avgR.float, ffDecimal, 2) drawText(&"R:{roundCounter} avg:{avgRStr}", getX(), getY() - 40.0) go() when isMainModule: resultChan.open() createDir(weightsRoot) cleanStaleTempDirs(weightsRoot) let loadResult = loadBestAvailable(ac, gAdamStates, weightsRoot) if loadResult.loaded: roundCounter = loadResult.roundNum var bot = PPOBot( tracker: initEnemyTracker(), buffer: initTrajectoryBuffer(), ) start(bot, botJsonPath)