SirStone
aa4bc77068
Two-hidden-layer MLP actor-critic (42→64→64→5/1) with stochastic actorForward, logStd floor at -3, and BotAction mapper wired into the run() loop. Assert-based test suite covers shapes, finiteness, logStd collapse, and all action range bounds. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
61 lines
2.3 KiB
Nim
61 lines
2.3 KiB
Nim
## network.nim — MLP and ActorCritic forward pass (inference only, no autograd).
|
|
|
|
import arraymancer
|
|
import std/[math, random]
|
|
|
|
type
|
|
MLP* = object
|
|
w1*, b1*: Tensor[float32] # [hidden, input], [hidden]
|
|
w2*, b2*: Tensor[float32] # [hidden, hidden], [hidden]
|
|
w3*, b3*: Tensor[float32] # [output, hidden], [output]
|
|
|
|
ActorCritic* = object
|
|
actor*: MLP
|
|
critic*: MLP
|
|
logStd*: Tensor[float32] # [5] — one per action dim
|
|
|
|
proc initMLP*(inputDim, hiddenDim, outputDim: int): MLP =
|
|
# Xavier/He-style init: scale weights by sqrt(2/fan_in)
|
|
result.w1 = randomNormalTensor[float32]([hiddenDim, inputDim]) *. sqrt(2.0'f32 / inputDim.float32)
|
|
result.b1 = zeros[float32](hiddenDim)
|
|
result.w2 = randomNormalTensor[float32]([hiddenDim, hiddenDim]) *. sqrt(2.0'f32 / hiddenDim.float32)
|
|
result.b2 = zeros[float32](hiddenDim)
|
|
result.w3 = randomNormalTensor[float32]([outputDim, hiddenDim]) *. sqrt(1.0'f32 / hiddenDim.float32)
|
|
result.b3 = zeros[float32](outputDim)
|
|
|
|
proc initActorCritic*(): ActorCritic =
|
|
result.actor = initMLP(42, 64, 5)
|
|
result.critic = initMLP(42, 64, 1)
|
|
result.logStd = zeros[float32](5) # init to 0 → std=1
|
|
|
|
proc forward*(mlp: MLP, x: Tensor[float32]): Tensor[float32] =
|
|
## x shape: [inputDim] (1D vector)
|
|
let h1 = tanh(mlp.w1 * x + mlp.b1)
|
|
let h2 = tanh(mlp.w2 * h1 + mlp.b2)
|
|
result = mlp.w3 * h2 + mlp.b3
|
|
|
|
proc actorForward*(ac: ActorCritic, state: Tensor[float32]): tuple[actions: Tensor[float32], logProb: float32] =
|
|
## state: [42]. Returns sampled actions [5] and sum log-prob.
|
|
let mean = ac.actor.forward(state)
|
|
# Floor logStd at -3 before exp → min std ≈ 0.05
|
|
var clampedLogStd = ac.logStd.map(proc(v: float32): float32 = max(v, -3.0'f32))
|
|
let std = clampedLogStd.map(proc(v: float32): float32 = exp(v))
|
|
|
|
var actions = newTensor[float32](5)
|
|
var logP = 0.0'f32
|
|
for i in 0..<5:
|
|
let mu = mean[i]
|
|
let s = std[i]
|
|
let z = gauss(0.0'f64, 1.0'f64).float32
|
|
actions[i] = mu + s * z
|
|
# log N(a; mu, s) = -0.5*((a-mu)/s)^2 - log(s) - 0.5*log(2π)
|
|
let diff = (actions[i] - mu) / s
|
|
logP += -0.5'f32 * diff * diff - ln(s) - 0.5'f32 * ln(2.0'f32 * PI.float32)
|
|
|
|
result = (actions: actions, logProb: logP)
|
|
|
|
proc criticForward*(ac: ActorCritic, state: Tensor[float32]): float32 =
|
|
## state: [42]. Returns scalar value estimate.
|
|
let val = ac.critic.forward(state)
|
|
result = val[0]
|