From ca3e3d2272c070c621b6f20a85b2da6b1c6b9cf3 Mon Sep 17 00:00:00 2001 From: Davide Cappellini Date: Thu, 20 Aug 2026 15:28:15 +0200 Subject: [PATCH] =?UTF-8?q?tune(PPO=5FBot):=20logStd=3D-2.0=20(std?= =?UTF-8?q?=E2=89=880.135),=20entropy=3D0,=20ceiling=3D-1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stochastic eval at std≈0.37 was 0/10 vs Corners (deterministic: 10/10). Warm-start policy is correct but brittle — any noise breaks it. - log_std initialized to -2.0 (std≈0.135) for moderate exploration - entropy_coeff=0.0 (no push toward exploration during fine-tuning) - logStd ceiling=-1.0 (cap at std≈0.37) --- tools/training_runner/training.env | 4 ++-- tools/warm_start.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/training_runner/training.env b/tools/training_runner/training.env index 40e1c84..c27d431 100644 --- a/tools/training_runner/training.env +++ b/tools/training_runner/training.env @@ -4,7 +4,7 @@ PPOB_LOG_FILE=/home/davide/Projects/SirRoboGarage/tools/training_runner/logs/fir PPOB_LR=1e-4 PPOB_UPDATE_INTERVAL=10 PPOB_CLIP_EPSILON=0.2 -PPOB_ENTROPY_COEFF=0.001 +PPOB_ENTROPY_COEFF=0.0 PPOB_VALUE_LOSS_COEFF=0.5 PPOB_MAX_GRAD_NORM=0.5 PPOB_GAMMA=0.99 @@ -12,7 +12,7 @@ PPOB_LAM=0.95 PPOB_EPOCHS=4 PPOB_MINI_BATCH_SIZE=64 PPOB_LOG_STD_FLOOR=-3.0 -PPOB_LOG_STD_CEILING=-0.5 +PPOB_LOG_STD_CEILING=-1.0 PPOB_INITIAL_LOG_STD=-0.5 # PPOB_EVAL_ONLY=1 → freeze training (pure evaluation) diff --git a/tools/warm_start.py b/tools/warm_start.py index 7c06f11..8ea1d00 100644 --- a/tools/warm_start.py +++ b/tools/warm_start.py @@ -37,10 +37,10 @@ for name in unchanged: np.save(DST / f"{name}.npy", data) print(f" {name}: {data.shape} copied") -# Initialize log_std to -1.0 (std ≈ 0.37) — snapshot values (2.27–4.68) are too high for fine-tuning -log_std = np.full(6, -1.0, dtype=np.float32) +# Initialize log_std to -2.0 (std ≈ 0.135) — tighter than -1.0, proven workable +log_std = np.full(6, -2.0, dtype=np.float32) np.save(DST / "log_std.npy", log_std) -print(f" log_std: initialized to -1.0 (std≈0.37), shape={log_std.shape}") +print(f" log_std: initialized to -2.0 (std≈0.135), shape={log_std.shape}") # Copy unchanged Adam moments (all except w1, which were handled above) unchanged_adam = [