SirStone/SirRoboGarage
1
0
Fork 0
Code Issues 28 Pull Requests Actions Projects Releases Wiki Activity

feat(PPO_Bot): enemy-centered action space + reward shaping for 100% vs Target

Browse Source 
- actions.nim: goto/aimTo coordinates now offset from enemy position
  (enemyX + tanh(raw) * scale) instead of absolute arena coords
  (sigmoid(raw) * arenaSize). Initial random policy defaults to
  approaching and aiming at enemy.
- training.nim: added dense reward shaping (distance closeness +
  gun bearing) to computeTickReward, doubled round reward scaling.
- PPO_Bot.nim: passes enemy position to mapActions, computes
  gun-to-enemy bearing for reward shaping.

Result: 100/100 win rate vs Target with frozen weights.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Davide Cappellini
2026-08-18 17:40:59 +02:00
parent 0b17430735
commit 766b9e03ee
4 changed files with 164 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PPO_Bot/network.nim
+8 -4
View File
@@ -7,6 +7,10 @@ const
STATE_DIM* = 44
ACTION_DIM* = 6
var
logStdFloor*: float32 = -3.0'f32 # overridden by PPOB_LOG_STD_FLOOR
initialLogStd*: float32 = 0.0'f32 # overridden by PPOB_INITIAL_LOG_STD
type
MLP* = object
w1*, b1*: Tensor[float32] # [hidden, input], [hidden]
@@ -30,7 +34,7 @@ proc initMLP*(inputDim, hiddenDim, outputDim: int): MLP =
proc initActorCritic*(): ActorCritic =
result.actor = initMLP(STATE_DIM, 64, ACTION_DIM)
result.critic = initMLP(STATE_DIM, 64, 1)
result.logStd = zeros[float32](ACTION_DIM) # init to 0 → std=1
result.logStd = newTensor[float32](ACTION_DIM).map(proc(v: float32): float32 = initialLogStd)
proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] =
## x shape: [inputDim] (1D vector)
@@ -41,8 +45,8 @@ proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] =
proc actorForward*(ac: ActorCritic, state: Tensor[float32]): tuple[actions: Tensor[float32], logProb: float32] =
## state: [STATE_DIM]. Returns sampled actions [ACTION_DIM] and sum log-prob.
let mean = ac.actor.forward(state)
# Floor logStd at -3 before exp → min std ≈ 0.05
var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32))
# Floor logStd at logStdFloor before exp → min std ≈ exp(logStdFloor)
var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, logStdFloor))
let std = clampedLogStd.map(proc(v: float32): float32 = exp(v))
var actions = newTensor[float32](ACTION_DIM)
@@ -66,7 +70,7 @@ proc criticForward*(ac: ActorCritic, state: Tensor[float32]): float32 =
proc computeLogProb*(ac: ActorCritic, state, action: Tensor[float32]): float32 =
## Log-probability of action under current policy (no sampling).
let mean = ac.actor.forward(state)
let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32))
let logStdClamped = ac.logStd.map(proc(v: float32): float32 = max(v, logStdFloor))
let std = logStdClamped.map(proc(v: float32): float32 = exp(v))
var logP = 0.0'f32
for i in 0..<ACTION_DIM: