feat(SAC_LSTM_Bot): campaign v2 levers — aggression/anti-ram reward shaping + stability knob overrides (part 2)
Lever 2 (#59): x1.25 aggression mult on damage dealt, flat +0.5 hit bonus, -3.0 per bot-bot collision (server deals RAM_DAMAGE=0.6 to both parties but only notifies the hitter), escalating proximity deterrent below 12% arena diagonal suppressed while dealing damage. Win/loss terminals unchanged and dominant. All weights TUNABLE consts marked ponytail. SACLSTM_REWARD_DEBUG=1 env-gated reward_debug.log for calibration greps. Lever 5 (#59): no code needed — SACLSTM_LR_ACTOR/LR_CRITIC/LR_ALPHA (3e-4) and SACLSTM_TARGET_ENTROPY (-4.0) were already env-overridable in training.nim. Smoke vs RamFire+Crazy (hidden=32, random init, isolated weights): 75 ram penalties, 381 charge events, hit bonuses firing, 0 crashes, metrics JSONL flowing. Tests: 8/8 suites green incl. new assert-level term math.
This commit is contained in:
@@ -67,7 +67,7 @@ type SacBot = ref object of Bot
|
|||||||
ticksSinceScan: int
|
ticksSinceScan: int
|
||||||
# per-step reward accumulators (consumed by the next tick's transition)
|
# per-step reward accumulators (consumed by the next tick's transition)
|
||||||
dmgDealt, dmgTaken, wastedPower: float64
|
dmgDealt, dmgTaken, wastedPower: float64
|
||||||
wallHits: int
|
wallHits, hits, ramTaken: int
|
||||||
# pending transition (episode spans the whole battle; round end is NOT a boundary)
|
# pending transition (episode spans the whole battle; round end is NOT a boundary)
|
||||||
hasLastTrans: bool
|
hasLastTrans: bool
|
||||||
lastState: array[STATE_DIM, float32]
|
lastState: array[STATE_DIM, float32]
|
||||||
@@ -96,13 +96,36 @@ proc tensorToHidden(t: Tensor[float32]; arr: var array[MaxHidden, float32]) =
|
|||||||
|
|
||||||
proc takeReward(bot: SacBot; win = false, loss = false): float32 =
|
proc takeReward(bot: SacBot; win = false, loss = false): float32 =
|
||||||
## Consume accumulated step events -> Welford-normalized reward (#44).
|
## Consume accumulated step events -> Welford-normalized reward (#44).
|
||||||
|
## Lever 2 (#59): pass enemy distance (frac of arena diagonal) for the
|
||||||
|
## anti-charge term; sentinel 2.0 (> ChargeDistFrac) when no contact.
|
||||||
|
var distFrac = 2.0
|
||||||
|
if bot.hasContact:
|
||||||
|
let diag = hypot(getArenaWidth().float64, getArenaHeight().float64)
|
||||||
|
distFrac = hypot(bot.enemy.x - getX(), bot.enemy.y - getY()) / diag
|
||||||
let raw = computeReward(
|
let raw = computeReward(
|
||||||
damageInflicted = bot.dmgDealt,
|
damageInflicted = bot.dmgDealt,
|
||||||
damageReceived = bot.dmgTaken,
|
damageReceived = bot.dmgTaken,
|
||||||
wallHitTicks = bot.wallHits,
|
wallHitTicks = bot.wallHits,
|
||||||
wastedShotPower = bot.wastedPower,
|
wastedShotPower = bot.wastedPower,
|
||||||
|
hitCount = bot.hits,
|
||||||
|
ramTakenCount = bot.ramTaken,
|
||||||
|
enemyDistFrac = distFrac,
|
||||||
win = win, loss = loss)
|
win = win, loss = loss)
|
||||||
|
# Lever-2 (#59) observability: env-gated one-liner for smoke/calibration
|
||||||
|
# greps — proves hit/ram/charge terms fire and shows raw magnitudes. File
|
||||||
|
# (not stderr): the battle runner swallows bot process streams.
|
||||||
|
# ponytail: grows unbounded if left on; keep off outside smokes.
|
||||||
|
if getEnv("SACLSTM_REWARD_DEBUG") == "1" and
|
||||||
|
(bot.hits > 0 or bot.ramTaken > 0 or (distFrac < ChargeDistFrac and bot.dmgDealt <= 0.0)):
|
||||||
|
try:
|
||||||
|
let f = open(getWeightsPath().parentDir.parentDir / "reward_debug.log", fmAppend)
|
||||||
|
f.writeLine("raw=" & $raw & " hits=" & $bot.hits & " ram=" & $bot.ramTaken &
|
||||||
|
" distFrac=" & $distFrac)
|
||||||
|
f.close()
|
||||||
|
except CatchableError:
|
||||||
|
discard
|
||||||
bot.dmgDealt = 0; bot.dmgTaken = 0; bot.wastedPower = 0; bot.wallHits = 0
|
bot.dmgDealt = 0; bot.dmgTaken = 0; bot.wastedPower = 0; bot.wallHits = 0
|
||||||
|
bot.hits = 0; bot.ramTaken = 0
|
||||||
let norm = rewards.normalize(bot.rn, raw)
|
let norm = rewards.normalize(bot.rn, raw)
|
||||||
rewards.update(bot.rn, raw)
|
rewards.update(bot.rn, raw)
|
||||||
norm.float32
|
norm.float32
|
||||||
@@ -157,10 +180,20 @@ method onScannedBot*(bot: SacBot, e: ScannedBotEvent) =
|
|||||||
|
|
||||||
method onBulletHit*(bot: SacBot, e: BulletHitBotEvent) =
|
method onBulletHit*(bot: SacBot, e: BulletHitBotEvent) =
|
||||||
bot.dmgDealt += e.damage
|
bot.dmgDealt += e.damage
|
||||||
|
inc bot.hits
|
||||||
|
|
||||||
method onHitByBullet*(bot: SacBot, e: HitByBulletEvent) =
|
method onHitByBullet*(bot: SacBot, e: HitByBulletEvent) =
|
||||||
bot.dmgTaken += e.damage
|
bot.dmgTaken += e.damage
|
||||||
|
|
||||||
|
# Lever 2 (#59): anti-ram — every BotHitBotEvent receipt means a bot-bot
|
||||||
|
# collision happened and we ate RAM_DAMAGE (server deals 0.6 to both parties;
|
||||||
|
# only the hitter gets notified). Flat per-event penalty; being rammed without
|
||||||
|
# hitting back stays event-invisible.
|
||||||
|
# ponytail: enemy-initiated rams undetected — add energy-residual detection if
|
||||||
|
# v2 battle data shows ram-heavy losses.
|
||||||
|
method onHitBot*(bot: SacBot, e: BotHitBotEvent) =
|
||||||
|
inc bot.ramTaken
|
||||||
|
|
||||||
method onHitWall*(bot: SacBot, e: BotHitWallEvent) =
|
method onHitWall*(bot: SacBot, e: BotHitWallEvent) =
|
||||||
inc bot.wallHits
|
inc bot.wallHits
|
||||||
|
|
||||||
@@ -173,6 +206,7 @@ method onGameAborted*(bot: SacBot) =
|
|||||||
# next battle's data.
|
# next battle's data.
|
||||||
bot.hasLastTrans = false
|
bot.hasLastTrans = false
|
||||||
bot.dmgDealt = 0; bot.dmgTaken = 0; bot.wastedPower = 0; bot.wallHits = 0
|
bot.dmgDealt = 0; bot.dmgTaken = 0; bot.wastedPower = 0; bot.wallHits = 0
|
||||||
|
bot.hits = 0; bot.ramTaken = 0
|
||||||
|
|
||||||
method onRoundEnded*(bot: SacBot, e: RoundEndedEventForBot) =
|
method onRoundEnded*(bot: SacBot, e: RoundEndedEventForBot) =
|
||||||
## Harness liveness signal (#49): RunTraining.java watches round_counter.txt
|
## Harness liveness signal (#49): RunTraining.java watches round_counter.txt
|
||||||
|
|||||||
@@ -3,6 +3,25 @@
|
|||||||
|
|
||||||
import std/math
|
import std/math
|
||||||
|
|
||||||
|
# ── Lever-2 shaping constants (#59, campaign v2) — TUNABLE ────────────────────
|
||||||
|
# Scale discipline: commensurate with existing magnitudes (dealt p=1 was +4,
|
||||||
|
# wall tick -5/tick, win +20). Death/loss and win terms stay dominant; these
|
||||||
|
# only re-rank mid-band behaviors (fight vs outlive vs get-rammed).
|
||||||
|
|
||||||
|
const
|
||||||
|
# Multiplier on the bullet-damage-dealt term: p=1 hit +4 -> +5. Low-power
|
||||||
|
# spam stays unprofitable (6*0.1-2 = -1.4 < 0 even after x1.25).
|
||||||
|
AggressionMult* = 1.25 # ponytail: TUNABLE — raise toward 1.5 if v2 bot still passivity-leaning
|
||||||
|
# Flat per landed shot on top of damage: discrete accuracy signal.
|
||||||
|
HitBonus* = 0.5 # ponytail: TUNABLE — keep < 6p-2 at min viable power (~0.34)
|
||||||
|
# Per bot-bot collision (BotHitBotEvent): server deals RAM_DAMAGE=0.6 to
|
||||||
|
# both parties but only notifies the hitter — each receipt = damage taken.
|
||||||
|
RamTakenPenalty* = 3.0 # ponytail: TUNABLE — vs p=0.8 bullet received (-6.8)
|
||||||
|
# Enemy-charging deterrent: distance/diagonal below this => escalating
|
||||||
|
# negative (max at zero distance), suppressed while we deal damage that step.
|
||||||
|
ChargeDistFrac* = 0.12 # ponytail: TUNABLE — ~120u of 800x600 diag (1000)
|
||||||
|
ChargePenalty* = 2.0 # ponytail: TUNABLE — per-tick ceiling, milder than wall (-5/tick)
|
||||||
|
|
||||||
# ── Raw reward ────────────────────────────────────────────────────────────────
|
# ── Raw reward ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
proc computeReward*(
|
proc computeReward*(
|
||||||
@@ -10,6 +29,9 @@ proc computeReward*(
|
|||||||
damageReceived: float64 = 0.0, # fire power p_e of enemy shot that hit
|
damageReceived: float64 = 0.0, # fire power p_e of enemy shot that hit
|
||||||
wallHitTicks: int = 0, # ticks in wall contact this step
|
wallHitTicks: int = 0, # ticks in wall contact this step
|
||||||
wastedShotPower: float64 = 0.0, # fire power of shot that missed/hit wall
|
wastedShotPower: float64 = 0.0, # fire power of shot that missed/hit wall
|
||||||
|
hitCount: int = 0, # own bullets that hit the enemy this step (#59)
|
||||||
|
ramTakenCount: int = 0, # collisions where we were the victim (#59)
|
||||||
|
enemyDistFrac: float64 = 2.0, # enemy dist / arena diag; >ChargeDistFrac when no contact (#59)
|
||||||
win: bool = false,
|
win: bool = false,
|
||||||
loss: bool = false
|
loss: bool = false
|
||||||
): float64 =
|
): float64 =
|
||||||
@@ -17,8 +39,13 @@ proc computeReward*(
|
|||||||
## Damage formula: 4p + 2(p-1) = 6p - 2 (matches Tank Royale bullet rules).
|
## Damage formula: 4p + 2(p-1) = 6p - 2 (matches Tank Royale bullet rules).
|
||||||
let p = damageInflicted
|
let p = damageInflicted
|
||||||
let pe = damageReceived
|
let pe = damageReceived
|
||||||
if p > 0.0: result += 6.0 * p - 2.0
|
if p > 0.0:
|
||||||
|
result += AggressionMult * (6.0 * p - 2.0)
|
||||||
|
if hitCount > 0: result += HitBonus * hitCount.float64
|
||||||
if pe > 0.0: result -= 6.0 * pe - 2.0
|
if pe > 0.0: result -= 6.0 * pe - 2.0
|
||||||
|
result -= RamTakenPenalty * ramTakenCount.float64
|
||||||
|
if enemyDistFrac < ChargeDistFrac and p <= 0.0:
|
||||||
|
result -= ChargePenalty * (1.0 - enemyDistFrac / ChargeDistFrac)
|
||||||
result -= 5.0 * wallHitTicks.float64
|
result -= 5.0 * wallHitTicks.float64
|
||||||
if wastedShotPower > 0.0: result -= 0.1 * wastedShotPower
|
if wastedShotPower > 0.0: result -= 0.1 * wastedShotPower
|
||||||
if win: result += 20.0
|
if win: result += 20.0
|
||||||
|
|||||||
@@ -11,17 +11,46 @@ template check(cond: bool, msg: string) =
|
|||||||
# ── computeReward ─────────────────────────────────────────────────────────────
|
# ── computeReward ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
block damageInflicted:
|
block damageInflicted:
|
||||||
# p=1: 6*1 - 2 = 4
|
# p=1: 1.25 * (6*1 - 2) = 5 (lever 2 aggression mult, #59)
|
||||||
check abs(computeReward(damageInflicted = 1.0) - 4.0) < 1e-9, "p=1 damage = +4"
|
check abs(computeReward(damageInflicted = 1.0) - 5.0) < 1e-9, "p=1 damage = +5"
|
||||||
# p=3: 6*3 - 2 = 16
|
# p=3: 1.25 * (6*3 - 2) = 20
|
||||||
check abs(computeReward(damageInflicted = 3.0) - 16.0) < 1e-9, "p=3 damage = +16"
|
check abs(computeReward(damageInflicted = 3.0) - 20.0) < 1e-9, "p=3 damage = +20"
|
||||||
|
# low-power spam stays unprofitable: 1.25*(6*0.1-2) < 0
|
||||||
|
check computeReward(damageInflicted = 0.1) < 0.0, "p=0.1 spam still negative"
|
||||||
|
|
||||||
block damageReceived:
|
block damageReceived:
|
||||||
# p_e=1: -(6*1 - 2) = -4
|
# p_e=1: -(6*1 - 2) = -4 (unchanged by lever 2)
|
||||||
check abs(computeReward(damageReceived = 1.0) - (-4.0)) < 1e-9, "p_e=1 received = -4"
|
check abs(computeReward(damageReceived = 1.0) - (-4.0)) < 1e-9, "p_e=1 received = -4"
|
||||||
# p_e=3: -(6*3 - 2) = -16
|
# p_e=3: -(6*3 - 2) = -16
|
||||||
check abs(computeReward(damageReceived = 3.0) - (-16.0)) < 1e-9, "p_e=3 received = -16"
|
check abs(computeReward(damageReceived = 3.0) - (-16.0)) < 1e-9, "p_e=3 received = -16"
|
||||||
|
|
||||||
|
block hitBonus:
|
||||||
|
# flat +0.5 per landed shot: p=1 hit -> 5.0 + 0.5
|
||||||
|
check abs(computeReward(damageInflicted = 1.0, hitCount = 1) - 5.5) < 1e-9,
|
||||||
|
"p=1 hit = +5.5"
|
||||||
|
# two hits in one step: 1.25*(6*2-2) + 2*0.5 = 12.5 + 1.0 = 13.5
|
||||||
|
check abs(computeReward(damageInflicted = 2.0, hitCount = 2) - 13.5) < 1e-9,
|
||||||
|
"two hits = +13.5"
|
||||||
|
|
||||||
|
block ramTaken:
|
||||||
|
# flat per victim collision (#59)
|
||||||
|
check abs(computeReward(ramTakenCount = 1) - (-3.0)) < 1e-9, "ram taken x1 = -3"
|
||||||
|
check abs(computeReward(ramTakenCount = 2) - (-6.0)) < 1e-9, "ram taken x2 = -6"
|
||||||
|
|
||||||
|
block chargeDeterrent:
|
||||||
|
# zero-damage case at half threshold depth: -2 * (1 - 0.06/0.12) = -1
|
||||||
|
let rHalf = computeReward(enemyDistFrac = 0.06)
|
||||||
|
check abs(rHalf - (-1.0)) < 1e-9, "charge at frac 0.06 = -1"
|
||||||
|
# at zero distance: full ceiling
|
||||||
|
check abs(computeReward(enemyDistFrac = 0.0) - (-2.0)) < 1e-9, "charge at frac 0 = -2"
|
||||||
|
# at/beyond threshold and no-contact sentinel: no penalty
|
||||||
|
check abs(computeReward(enemyDistFrac = 0.12)) < 1e-9, "at threshold = 0"
|
||||||
|
check abs(computeReward(enemyDistFrac = 0.5)) < 1e-9, "beyond threshold = 0"
|
||||||
|
check abs(computeReward(enemyDistFrac = 2.0)) < 1e-9, "no-contact sentinel = 0"
|
||||||
|
# suppressed while dealing damage that step (fighting back at close range is fine)
|
||||||
|
let rFight = computeReward(damageInflicted = 1.0, enemyDistFrac = 0.06)
|
||||||
|
check abs(rFight - 5.0) < 1e-9, "dealing damage cancels charge penalty"
|
||||||
|
|
||||||
block wallHit:
|
block wallHit:
|
||||||
check abs(computeReward(wallHitTicks = 1) - (-5.0)) < 1e-9, "1 wall tick = -5"
|
check abs(computeReward(wallHitTicks = 1) - (-5.0)) < 1e-9, "1 wall tick = -5"
|
||||||
|
|
||||||
@@ -30,6 +59,7 @@ block wastedShot:
|
|||||||
check abs(computeReward(wastedShotPower = 2.0) - (-0.2)) < 1e-9, "missed p=2 = -0.2"
|
check abs(computeReward(wastedShotPower = 2.0) - (-0.2)) < 1e-9, "missed p=2 = -0.2"
|
||||||
|
|
||||||
block winLoss:
|
block winLoss:
|
||||||
|
# terminal terms stay dominant over shaping (#59 scale discipline)
|
||||||
check abs(computeReward(win = true) - 20.0) < 1e-9, "win = +20"
|
check abs(computeReward(win = true) - 20.0) < 1e-9, "win = +20"
|
||||||
check abs(computeReward(loss = true) - (-10.0)) < 1e-9, "loss = -10"
|
check abs(computeReward(loss = true) - (-10.0)) < 1e-9, "loss = -10"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user