diff --git a/PPO_Bot/PPO_Bot b/PPO_Bot/PPO_Bot index 2dab3c3..21b6ff8 100755 Binary files a/PPO_Bot/PPO_Bot and b/PPO_Bot/PPO_Bot differ diff --git a/PPO_Bot/tests/test_training b/PPO_Bot/tests/test_training index 49f8bba..bb1729a 100755 Binary files a/PPO_Bot/tests/test_training and b/PPO_Bot/tests/test_training differ diff --git a/PPO_Bot/tests/test_training.nim b/PPO_Bot/tests/test_training.nim index 4a9d4df..f611abd 100644 --- a/PPO_Bot/tests/test_training.nim +++ b/PPO_Bot/tests/test_training.nim @@ -23,6 +23,9 @@ block testTickReward: block testRoundReward: let r = computeRoundReward(350.0'f32) check abs(r - 7.0'f32) < 1e-6'f32, "computeRoundReward(350) == 7.0, got " & $r + # bounded: long-battle cumulative scores must saturate, not blow the value scale + check abs(computeRoundReward(89299.0'f32) - 8.0'f32) < 1e-6'f32, + "computeRoundReward(89299) == 8.0 (capped), got " & $computeRoundReward(89299.0'f32) # ── TrajectoryBuffer ────────────────────────────────────────────────────────── diff --git a/PPO_Bot/training.nim b/PPO_Bot/training.nim index 6cc4cc0..067e864 100644 --- a/PPO_Bot/training.nim +++ b/PPO_Bot/training.nim @@ -69,7 +69,9 @@ proc computeTickReward*(myEnergyDelta, enemyEnergyDelta: float32; proc computeRoundReward*(roundScore: float32): float32 = ## Normalise round-end score to a rough ±6 range (doubled win signal). - result = roundScore / 50.0'f32 + ## ponytail: cap cumulative score at 400 before /50 — bounded terminal bonus + ## keeps critic value scale stable across battle boundaries and long battles. + result = min(roundScore, 400.0'f32) / 50.0'f32 # ── GAE ─────────────────────────────────────────────────────────────────────── diff --git a/PPO_Bot/weights/latest/actor_b1.npy b/PPO_Bot/weights/latest/actor_b1.npy index 67bb044..db38cb3 100644 Binary files a/PPO_Bot/weights/latest/actor_b1.npy and b/PPO_Bot/weights/latest/actor_b1.npy differ diff --git a/PPO_Bot/weights/latest/actor_b2.npy b/PPO_Bot/weights/latest/actor_b2.npy index 3d15539..4b2cc2f 100644 Binary files a/PPO_Bot/weights/latest/actor_b2.npy and b/PPO_Bot/weights/latest/actor_b2.npy differ diff --git a/PPO_Bot/weights/latest/actor_b3.npy b/PPO_Bot/weights/latest/actor_b3.npy index 29067aa..6ac4cd3 100644 Binary files a/PPO_Bot/weights/latest/actor_b3.npy and b/PPO_Bot/weights/latest/actor_b3.npy differ diff --git a/PPO_Bot/weights/latest/actor_w1.npy b/PPO_Bot/weights/latest/actor_w1.npy index f4899ca..70e2cf7 100644 Binary files a/PPO_Bot/weights/latest/actor_w1.npy and b/PPO_Bot/weights/latest/actor_w1.npy differ diff --git a/PPO_Bot/weights/latest/actor_w2.npy b/PPO_Bot/weights/latest/actor_w2.npy index b932603..955dd63 100644 Binary files a/PPO_Bot/weights/latest/actor_w2.npy and b/PPO_Bot/weights/latest/actor_w2.npy differ diff --git a/PPO_Bot/weights/latest/actor_w3.npy b/PPO_Bot/weights/latest/actor_w3.npy index 25b4c06..ecc1836 100644 Binary files a/PPO_Bot/weights/latest/actor_w3.npy and b/PPO_Bot/weights/latest/actor_w3.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab1_m.npy b/PPO_Bot/weights/latest/adam_ab1_m.npy index 23b1ce2..8197a7b 100644 Binary files a/PPO_Bot/weights/latest/adam_ab1_m.npy and b/PPO_Bot/weights/latest/adam_ab1_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab1_v.npy b/PPO_Bot/weights/latest/adam_ab1_v.npy index 23ca074..b824118 100644 Binary files a/PPO_Bot/weights/latest/adam_ab1_v.npy and b/PPO_Bot/weights/latest/adam_ab1_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab2_m.npy b/PPO_Bot/weights/latest/adam_ab2_m.npy index 90afeed..f3f1a31 100644 Binary files a/PPO_Bot/weights/latest/adam_ab2_m.npy and b/PPO_Bot/weights/latest/adam_ab2_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab2_v.npy b/PPO_Bot/weights/latest/adam_ab2_v.npy index 93f80fd..61f0e20 100644 Binary files a/PPO_Bot/weights/latest/adam_ab2_v.npy and b/PPO_Bot/weights/latest/adam_ab2_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab3_m.npy b/PPO_Bot/weights/latest/adam_ab3_m.npy index 8880282..dd7a867 100644 Binary files a/PPO_Bot/weights/latest/adam_ab3_m.npy and b/PPO_Bot/weights/latest/adam_ab3_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_ab3_v.npy b/PPO_Bot/weights/latest/adam_ab3_v.npy index ef6d26e..4c2519f 100644 Binary files a/PPO_Bot/weights/latest/adam_ab3_v.npy and b/PPO_Bot/weights/latest/adam_ab3_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw1_m.npy b/PPO_Bot/weights/latest/adam_aw1_m.npy index 76de757..66f365d 100644 Binary files a/PPO_Bot/weights/latest/adam_aw1_m.npy and b/PPO_Bot/weights/latest/adam_aw1_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw1_v.npy b/PPO_Bot/weights/latest/adam_aw1_v.npy index 2234da8..a2943b3 100644 Binary files a/PPO_Bot/weights/latest/adam_aw1_v.npy and b/PPO_Bot/weights/latest/adam_aw1_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw2_m.npy b/PPO_Bot/weights/latest/adam_aw2_m.npy index 8a618b1..3ca74aa 100644 Binary files a/PPO_Bot/weights/latest/adam_aw2_m.npy and b/PPO_Bot/weights/latest/adam_aw2_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw2_v.npy b/PPO_Bot/weights/latest/adam_aw2_v.npy index 08c6f63..6b8531a 100644 Binary files a/PPO_Bot/weights/latest/adam_aw2_v.npy and b/PPO_Bot/weights/latest/adam_aw2_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw3_m.npy b/PPO_Bot/weights/latest/adam_aw3_m.npy index 0016bfb..6a6016f 100644 Binary files a/PPO_Bot/weights/latest/adam_aw3_m.npy and b/PPO_Bot/weights/latest/adam_aw3_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_aw3_v.npy b/PPO_Bot/weights/latest/adam_aw3_v.npy index 0b398ce..286bd52 100644 Binary files a/PPO_Bot/weights/latest/adam_aw3_v.npy and b/PPO_Bot/weights/latest/adam_aw3_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb1_m.npy b/PPO_Bot/weights/latest/adam_cb1_m.npy index dd35d5e..a54c3d2 100644 Binary files a/PPO_Bot/weights/latest/adam_cb1_m.npy and b/PPO_Bot/weights/latest/adam_cb1_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb1_v.npy b/PPO_Bot/weights/latest/adam_cb1_v.npy index c8b1753..f7e92b6 100644 Binary files a/PPO_Bot/weights/latest/adam_cb1_v.npy and b/PPO_Bot/weights/latest/adam_cb1_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb2_m.npy b/PPO_Bot/weights/latest/adam_cb2_m.npy index c44f57d..3d56c2f 100644 Binary files a/PPO_Bot/weights/latest/adam_cb2_m.npy and b/PPO_Bot/weights/latest/adam_cb2_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb2_v.npy b/PPO_Bot/weights/latest/adam_cb2_v.npy index 0cd1745..7f7f896 100644 Binary files a/PPO_Bot/weights/latest/adam_cb2_v.npy and b/PPO_Bot/weights/latest/adam_cb2_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb3_m.npy b/PPO_Bot/weights/latest/adam_cb3_m.npy index 32db93c..0b43ee7 100644 Binary files a/PPO_Bot/weights/latest/adam_cb3_m.npy and b/PPO_Bot/weights/latest/adam_cb3_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cb3_v.npy b/PPO_Bot/weights/latest/adam_cb3_v.npy index b6f6a7e..7abb6d2 100644 Binary files a/PPO_Bot/weights/latest/adam_cb3_v.npy and b/PPO_Bot/weights/latest/adam_cb3_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw1_m.npy b/PPO_Bot/weights/latest/adam_cw1_m.npy index 6021369..c6ac7b3 100644 Binary files a/PPO_Bot/weights/latest/adam_cw1_m.npy and b/PPO_Bot/weights/latest/adam_cw1_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw1_v.npy b/PPO_Bot/weights/latest/adam_cw1_v.npy index 90bc8b5..36abf53 100644 Binary files a/PPO_Bot/weights/latest/adam_cw1_v.npy and b/PPO_Bot/weights/latest/adam_cw1_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw2_m.npy b/PPO_Bot/weights/latest/adam_cw2_m.npy index 0c311be..e117a85 100644 Binary files a/PPO_Bot/weights/latest/adam_cw2_m.npy and b/PPO_Bot/weights/latest/adam_cw2_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw2_v.npy b/PPO_Bot/weights/latest/adam_cw2_v.npy index af91421..53985ea 100644 Binary files a/PPO_Bot/weights/latest/adam_cw2_v.npy and b/PPO_Bot/weights/latest/adam_cw2_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw3_m.npy b/PPO_Bot/weights/latest/adam_cw3_m.npy index 6b2b488..72b8e27 100644 Binary files a/PPO_Bot/weights/latest/adam_cw3_m.npy and b/PPO_Bot/weights/latest/adam_cw3_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_cw3_v.npy b/PPO_Bot/weights/latest/adam_cw3_v.npy index 9f39ba2..f8d0cd6 100644 Binary files a/PPO_Bot/weights/latest/adam_cw3_v.npy and b/PPO_Bot/weights/latest/adam_cw3_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_logstd_m.npy b/PPO_Bot/weights/latest/adam_logstd_m.npy index 0cb7a83..20fc342 100644 Binary files a/PPO_Bot/weights/latest/adam_logstd_m.npy and b/PPO_Bot/weights/latest/adam_logstd_m.npy differ diff --git a/PPO_Bot/weights/latest/adam_logstd_v.npy b/PPO_Bot/weights/latest/adam_logstd_v.npy index f5aaec2..1c64542 100644 Binary files a/PPO_Bot/weights/latest/adam_logstd_v.npy and b/PPO_Bot/weights/latest/adam_logstd_v.npy differ diff --git a/PPO_Bot/weights/latest/adam_t.txt b/PPO_Bot/weights/latest/adam_t.txt index a0fda86..dbe0475 100644 --- a/PPO_Bot/weights/latest/adam_t.txt +++ b/PPO_Bot/weights/latest/adam_t.txt @@ -1,13 +1,13 @@ -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 -134308 \ No newline at end of file +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 +200200 \ No newline at end of file diff --git a/PPO_Bot/weights/latest/critic_b1.npy b/PPO_Bot/weights/latest/critic_b1.npy index 40dbb47..a98fda0 100644 Binary files a/PPO_Bot/weights/latest/critic_b1.npy and b/PPO_Bot/weights/latest/critic_b1.npy differ diff --git a/PPO_Bot/weights/latest/critic_b2.npy b/PPO_Bot/weights/latest/critic_b2.npy index 0167325..1b08ce0 100644 Binary files a/PPO_Bot/weights/latest/critic_b2.npy and b/PPO_Bot/weights/latest/critic_b2.npy differ diff --git a/PPO_Bot/weights/latest/critic_b3.npy b/PPO_Bot/weights/latest/critic_b3.npy index d50abab..abb5a67 100644 Binary files a/PPO_Bot/weights/latest/critic_b3.npy and b/PPO_Bot/weights/latest/critic_b3.npy differ diff --git a/PPO_Bot/weights/latest/critic_w1.npy b/PPO_Bot/weights/latest/critic_w1.npy index 7cd3cff..c2e29b1 100644 Binary files a/PPO_Bot/weights/latest/critic_w1.npy and b/PPO_Bot/weights/latest/critic_w1.npy differ diff --git a/PPO_Bot/weights/latest/critic_w2.npy b/PPO_Bot/weights/latest/critic_w2.npy index 2f79f78..13ef3b2 100644 Binary files a/PPO_Bot/weights/latest/critic_w2.npy and b/PPO_Bot/weights/latest/critic_w2.npy differ diff --git a/PPO_Bot/weights/latest/critic_w3.npy b/PPO_Bot/weights/latest/critic_w3.npy index 0756f12..ac0caf2 100644 Binary files a/PPO_Bot/weights/latest/critic_w3.npy and b/PPO_Bot/weights/latest/critic_w3.npy differ diff --git a/PPO_Bot/weights/latest/log_std.npy b/PPO_Bot/weights/latest/log_std.npy index b98b483..258cd88 100644 Binary files a/PPO_Bot/weights/latest/log_std.npy and b/PPO_Bot/weights/latest/log_std.npy differ diff --git a/PPO_Bot/weights/round_counter.txt b/PPO_Bot/weights/round_counter.txt index db8eb44..774f1d5 100644 --- a/PPO_Bot/weights/round_counter.txt +++ b/PPO_Bot/weights/round_counter.txt @@ -1 +1 @@ -3159 \ No newline at end of file +5211 \ No newline at end of file diff --git a/tools/training_runner/run.sh b/tools/training_runner/run.sh index 8fe31a9..41affe2 100755 --- a/tools/training_runner/run.sh +++ b/tools/training_runner/run.sh @@ -79,13 +79,11 @@ while true; do sleep 2 fi - # ponytail: cap each battle at 60 rounds — matches the certified config. - # Round-end reward is cumulative totalScore/50; long battles grow it unboundedly - # and blow up the critic's value scale (seen at round 3160, vLoss 25745 by 3871). - # Short battles bound the reward, and fresh-process restarts clear thread state. - battle_rounds=$(( remaining > 60 ? 60 : remaining )) - echo ">>> Running $battle_rounds rounds (opponent: $OPPONENT)..." - java -cp ".:$JAR" RunTraining "$OPPONENT" "$battle_rounds" || { + # One battle for the whole remaining budget: round-end reward is a bounded + # terminal bonus (computeRoundReward caps totalScore at 400), so long battles + # keep the critic's value scale stable — no per-battle chunking needed. + echo ">>> Running $remaining rounds (opponent: $OPPONENT)..." + java -cp ".:$JAR" RunTraining "$OPPONENT" "$remaining" || { exit_code=$? echo ">>> Java runner exited with code $exit_code — checking if done..." remaining=$(remaining_rounds)