feat(SAC_LSTM_Bot): live campaign dashboard — five panels, auto-reload, run-1 comparison dropped
This commit is contained in:
+344
-234
@@ -1,15 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot SAC training progress from live campaign logs.
|
||||
"""Build the single self-updating SAC training dashboard.
|
||||
|
||||
Pure-stdlib SVG output (matplotlib not available on this box).
|
||||
Generates:
|
||||
docs/graph_eval_winrates.svg - two stacked panels: Run 3 trend (raw dots +
|
||||
rolling-10 thick line) per opponent, and
|
||||
Run 1 vs Run 3 Corners on a fraction-of-run axis
|
||||
docs/graph_losses.svg - critic/actor/alpha loss curves from training_metrics.jsonl
|
||||
Generates ONE file:
|
||||
docs/campaign_dashboard.svg - five panels, current (v2) run only:
|
||||
1. test-match win % vs opponents (campaign_v2_stdout.log eval lines)
|
||||
2. real-fight win % per opponent (training_log.jsonl, ~100-game buckets)
|
||||
3. critic_loss / |actor_loss| (training_metrics.jsonl, shared log-y)
|
||||
4. alpha temperature (training_metrics.jsonl, linear)
|
||||
5. throughput, games/hour buckets (training_metrics.jsonl 'epoch' deltas;
|
||||
training_log.jsonl has NO timestamps -
|
||||
verified field names - and one metrics
|
||||
row == one 10-game chunk, counts match
|
||||
the stdout "=== Chunk N/N ===" markers)
|
||||
plus an embedded JS snippet that reloads the page every 60 s when the SVG is
|
||||
opened as a top-level document in Chrome.
|
||||
|
||||
Usage:
|
||||
python3 tools/plot_progress.py [campaign_log] [metrics_jsonl] [v1_log] [outdir]
|
||||
python3 tools/plot_progress.py [campaign_log] [metrics_jsonl] [games_jsonl] [outdir]
|
||||
All args optional; defaults relative to the SAC_LSTM_Bot/ root (parent of tools/).
|
||||
python3 tools/plot_progress.py --selftest # tiny built-in sanity check
|
||||
"""
|
||||
@@ -19,33 +27,53 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EVAL_RE = re.compile(r">>> \[eval\] win rate: (\d+)/(\d+) \(([\d.]+)%\) vs (\S+)")
|
||||
TREND_WINDOW = 10 # rolling mean shown as the thick trend line
|
||||
BUCKETS = 40 # v1 downsampling for the Run1-vs-Run3 panel
|
||||
COLORS = {"Corners": "#d62728", "Crazy": "#1f77b4", "Target": "#2ca02c"}
|
||||
V1_COLOR = "#999999"
|
||||
W, M_L, M_R, M_T, M_B = 1000, 70, 20, 40, 45 # graph 2 geometry (unchanged)
|
||||
GUIDE_H = 124 # bottom band holding the plain-English "How to read" panel
|
||||
WINRATE_GUIDE = [
|
||||
"One small dot = one test match. 0 means lost all ten fights.",
|
||||
"A thick line shows the recent trend. Up = learning.",
|
||||
"Watch the three colors. Red = Corners. Blue = Crazy. Green = Target.",
|
||||
"Gray line = old run. Red line = new run. Same enemy.",
|
||||
"If red ends higher than gray, the new training worked.",
|
||||
TREND_WINDOW = 10 # rolling mean shown as the thick trend line (panel 1)
|
||||
GAME_BUCKET = 100 # games per bucket, real-fight panel
|
||||
RATE_BUCKET = 20 # metric intervals per throughput bucket (~200 games)
|
||||
GAMES_PER_ROW = 10 # one training_metrics.jsonl row per 10-round chunk
|
||||
COLORS = {"Corners": "#d62728", "Crazy": "#1f77b4", "Target": "#2ca02c",
|
||||
"RamFire": "#ff7f0e", "SacTwin": "#9467bd"}
|
||||
CRITIC_C, ACTOR_C = "#1f77b4", "#ff7f0e"
|
||||
RELOAD_JS = ('<script type="text/javascript"><![CDATA[ '
|
||||
'setTimeout(function(){ location.reload(); }, 60000); ]]></script>')
|
||||
W, H = 1400, 2000
|
||||
TITLE_H, GUIDE_H = 80, 180
|
||||
# rows: (header_y, panel_top_y, panel_bottom_y, x_left, x_right)
|
||||
C1_L, C1_R = 70, 697
|
||||
C2_L, C2_R = 747, 1375
|
||||
ROWS = {
|
||||
1: (100, 120, 720, C1_L, C1_R),
|
||||
2: (100, 120, 720, C2_L, C2_R),
|
||||
3: (940, 958, 1428, C1_L, C1_R),
|
||||
4: (940, 958, 1428, C2_L, C2_R),
|
||||
5: (1600, 1618, 1780, C1_L, C2_R),
|
||||
}
|
||||
PANEL_TITLES = [
|
||||
"Test matches - win % vs opponents",
|
||||
"Real fights - win % per opponent",
|
||||
"Training losses (log scale)",
|
||||
"Alpha temperature",
|
||||
"Throughput - games per hour",
|
||||
]
|
||||
LOSSES_GUIDE = [
|
||||
"These are internal error numbers. They are not game scores.",
|
||||
"Critic and actor panels: big spikes are normal early.",
|
||||
"Steady growth forever is bad.",
|
||||
"Alpha near 1 = bot still experimenting.",
|
||||
"If alpha falls too fast, the bot freezes its habits.",
|
||||
GUIDE_LINES = [
|
||||
"Test matches: dots are single fights, thick line shows trend.",
|
||||
"Real battles only. Rising lines mean the bot improves.",
|
||||
"Loss spikes are normal early; endless growth is bad.",
|
||||
"Alpha high means experimenting; falling too fast freezes habits.",
|
||||
"Throughput flat is healthy; dips mean something slowed.",
|
||||
"This file reloads itself in Chrome every sixty seconds.",
|
||||
"Regenerate anytime with tools/watch_dashboard.sh or the python command.",
|
||||
]
|
||||
|
||||
|
||||
# ---------- parsing ----------
|
||||
|
||||
def parse_eval_series(path):
|
||||
"""Return {opponent: [win% per eval, in file order]}."""
|
||||
series = {}
|
||||
@@ -81,6 +109,39 @@ def parse_metrics(path):
|
||||
return rows
|
||||
|
||||
|
||||
def parse_games(path):
|
||||
"""Return [(opponent, won_bool)] for type=='game' rows, skipping junk."""
|
||||
out = []
|
||||
if not path.is_file():
|
||||
print(f"[skip] games log not found: {path}")
|
||||
return out
|
||||
for line in path.read_text(errors="replace").splitlines():
|
||||
try:
|
||||
r = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if r.get("type") != "game":
|
||||
continue
|
||||
opp, win = r.get("opponent"), r.get("win")
|
||||
if isinstance(opp, str) and isinstance(win, bool):
|
||||
out.append((opp, win))
|
||||
return out
|
||||
|
||||
|
||||
def metric_col(rows, key, positive=False):
|
||||
"""[(index, value)] for float-parseable rows; abs() applied; optional >0 filter."""
|
||||
out = []
|
||||
for i, r in enumerate(rows):
|
||||
try:
|
||||
v = abs(float(r[key]))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if positive and v <= 0:
|
||||
continue
|
||||
out.append((i, v))
|
||||
return out
|
||||
|
||||
|
||||
def rolling(vals, w=TREND_WINDOW):
|
||||
out, s = [], 0.0
|
||||
for i, v in enumerate(vals):
|
||||
@@ -91,6 +152,20 @@ def rolling(vals, w=TREND_WINDOW):
|
||||
return out
|
||||
|
||||
|
||||
def bucket_means(vals, n):
|
||||
"""Split vals into <=n contiguous buckets of near-equal size; per-bucket mean."""
|
||||
if not vals:
|
||||
return []
|
||||
n = min(n, len(vals))
|
||||
k, rem = divmod(len(vals), n)
|
||||
out, i = [], 0
|
||||
for b in range(n):
|
||||
size = k + (1 if b < rem else 0)
|
||||
out.append(sum(vals[i:i + size]) / size)
|
||||
i += size
|
||||
return out
|
||||
|
||||
|
||||
# ---------- tiny SVG helpers ----------
|
||||
|
||||
def esc(s):
|
||||
@@ -113,14 +188,6 @@ def write_svg(path, text):
|
||||
return True
|
||||
|
||||
|
||||
def svg_open(w, h, title):
|
||||
return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" '
|
||||
f'viewBox="0 0 {w} {h}" font-family="sans-serif">\n'
|
||||
f'<rect width="{w}" height="{h}" fill="white"/>\n'
|
||||
f'<text x="{w // 2}" y="22" text-anchor="middle" font-size="17" font-weight="bold">'
|
||||
f"{esc(title)}</text>\n")
|
||||
|
||||
|
||||
def polyline(pts, color, width=1.5, dash=None, opacity=1.0):
|
||||
if len(pts) < 2:
|
||||
return ""
|
||||
@@ -140,20 +207,6 @@ def hgrid(x0, x1, ys):
|
||||
f'stroke="#dddddd"/>\n' for y in ys)
|
||||
|
||||
|
||||
def bucket_means(vals, n):
|
||||
"""Split vals into <=n contiguous buckets of near-equal size; per-bucket mean."""
|
||||
if not vals:
|
||||
return []
|
||||
n = min(n, len(vals))
|
||||
k, rem = divmod(len(vals), n)
|
||||
out, i = [], 0
|
||||
for b in range(n):
|
||||
size = k + (1 if b < rem else 0)
|
||||
out.append(sum(vals[i:i + size]) / size)
|
||||
i += size
|
||||
return out
|
||||
|
||||
|
||||
def axis(x0, y0, x1, y1, xt, yt, xlabel, ylabel, ylog=False):
|
||||
"""Draw axes + ticks + labels. xt/yt are (value, px) tick lists."""
|
||||
s = (f'<line x1="{x0}" y1="{y0}" x2="{x1}" y2="{y0}" stroke="black"/>\n'
|
||||
@@ -203,172 +256,228 @@ def map_fn(p0, p1, vmin, vmax, log=False):
|
||||
|
||||
def guide_block(w, h, lines):
|
||||
"""Plain-English "How to read" band at the bottom of the canvas."""
|
||||
y = h - GUIDE_H
|
||||
y = H - GUIDE_H
|
||||
s = f'<rect x="0" y="{y}" width="{w}" height="{GUIDE_H}" fill="#f2f2f2"/>\n'
|
||||
s += (f'<text x="16" y="{y + 21}" font-size="15" font-weight="bold">'
|
||||
f"How to read</text>\n"
|
||||
f'<text x="{w - 16}" y="{y + 20}" text-anchor="end" font-size="11" '
|
||||
f'fill="#666">Regenerate anytime: python3 tools/plot_progress.py</text>\n')
|
||||
for i, ln in enumerate(lines):
|
||||
s += f'<text x="16" y="{y + 41 + i * 17}" font-size="13">{esc(ln)}</text>\n'
|
||||
s += f'<text x="16" y="{y + 43 + i * 17}" font-size="13">{esc(ln)}</text>\n'
|
||||
return s
|
||||
|
||||
|
||||
# ---------- graph 1: eval win rates ----------
|
||||
def header(x, y, title, sub=None):
|
||||
s = (f'<text x="{x}" y="{y}" font-size="15" font-weight="bold">'
|
||||
f"{esc(title)}</text>\n")
|
||||
if sub:
|
||||
s += (f'<text x="{x}" y="{y + 15}" font-size="11" fill="#555">'
|
||||
f"{esc(sub)}</text>\n")
|
||||
return s
|
||||
|
||||
def graph_eval(series_v2, series_v1, out):
|
||||
if not series_v2 and not series_v1:
|
||||
print("[skip] no eval data at all -> no win-rate graph")
|
||||
return 0
|
||||
# panel geometry (local: graph 2 keeps the module-level constants)
|
||||
CW, ML, MR = 1200, 64, 24
|
||||
TOP, PH, GAP, MB = 78, 310, 84, 56
|
||||
H = TOP + PH + GAP + PH + MB + GUIDE_H
|
||||
x0, x1 = ML, CW - MR
|
||||
s = svg_open(CW, H, "SAC-LSTM campaign: eval win rate vs opponents (Run 3 live, Run 1 archived)")
|
||||
|
||||
# ---- top panel: run 3 recent trend ----
|
||||
y1t, y0t = TOP, TOP + PH
|
||||
xmax = max(max((len(v) for v in series_v2.values()), default=1), 2)
|
||||
xm = map_fn(x0, x1, 1, xmax)
|
||||
ymt = map_fn(y0t, y1t, 0, 100)
|
||||
s += hgrid(x0, x1, [ymt(v) for v in range(0, 101, 20)])
|
||||
s += (f'<text x="{x0 + 6}" y="{TOP - 34}" font-size="14" font-weight="bold">'
|
||||
f"Run 3 — recent trend</text>\n"
|
||||
f'<text x="{x0 + 6}" y="{TOP - 16}" font-size="11" fill="#555">'
|
||||
f"win % per test match — thick line = trend</text>\n")
|
||||
xt = [(str(v), xm(v)) for v in range(10, xmax + 1, 10)] or \
|
||||
[("1", xm(1)), (str(xmax), xm(xmax))]
|
||||
s += axis(x0, y0t, x1, y1t, xt,
|
||||
ticks_linear(0, 100, y0t, y1t, n=6),
|
||||
"eval number", "win rate (%)")
|
||||
# ---------- panels ----------
|
||||
|
||||
def panel_test_matches(series, geo):
|
||||
_, pt, pb, x0, x1 = geo
|
||||
s = ""
|
||||
if not series:
|
||||
s += f'<text x="{x0 + 10}" y="{pt + 40}" font-size="12" fill="#a00">' \
|
||||
"no eval lines found</text>\n"
|
||||
return
|
||||
xmax = max(max(len(v) for v in series.values()), 2)
|
||||
xm, ym = map_fn(x0, x1, 1, xmax), map_fn(pb, pt, 0, 100)
|
||||
s += hgrid(x0, x1, [ym(v) for v in range(0, 101, 20)])
|
||||
step = max(1, (xmax // 8 // 10) * 10)
|
||||
xt = [(str(v), xm(v)) for v in range(step, xmax + 1, step)] or [("1", xm(1))]
|
||||
s += axis(x0, pb, x1, pt, xt, ticks_linear(0, 100, pb, pt, n=6),
|
||||
"test match number (each opponent)", "win rate (%)")
|
||||
items = []
|
||||
for name in ("Corners", "Crazy", "Target"):
|
||||
vals = series_v2.get(name, [])
|
||||
vals = series.get(name, [])
|
||||
if not vals:
|
||||
continue
|
||||
c = COLORS[name]
|
||||
s += dots([(xm(i + 1), ymt(v)) for i, v in enumerate(vals)], c)
|
||||
s += polyline([(xm(i + 1), ymt(v))
|
||||
for i, v in enumerate(rolling(vals, TREND_WINDOW))], c, 3.5)
|
||||
items.append((c, f"{name} — {len(vals)} evals"))
|
||||
s += legend(items, x0 + 12, y1t + 14)
|
||||
s += dots([(xm(i + 1), ym(v)) for i, v in enumerate(vals)], c)
|
||||
s += polyline([(xm(i + 1), ym(v))
|
||||
for i, v in enumerate(rolling(vals))], c, 3.5)
|
||||
items.append((c, f"{name} - {len(vals)} evals"))
|
||||
s += legend(items, x0 + 12, pb + 52)
|
||||
return s
|
||||
|
||||
# ---- bottom panel: run 1 vs run 3 on the same enemy ----
|
||||
tb = TOP + PH + GAP
|
||||
y1b, y0b = tb, tb + PH
|
||||
ymb = map_fn(y0b, y1b, 0, 100)
|
||||
|
||||
def xf(f):
|
||||
return x0 + (x1 - x0) * f
|
||||
|
||||
s += hgrid(x0, x1, [ymb(v) for v in range(0, 101, 20)])
|
||||
s += (f'<text x="{x0 + 6}" y="{tb - 22}" font-size="14" font-weight="bold">'
|
||||
f"Run 1 vs Run 3 (Corners) — same enemy — did learning stick?</text>\n")
|
||||
s += axis(x0, y0b, x1, y1b,
|
||||
[(("0", "0.2", "0.4", "0.6", "0.8", "1")[i], xf(i / 5)) for i in range(6)],
|
||||
ticks_linear(0, 100, y0b, y1b, n=6),
|
||||
"fraction of run (start → end)", "win rate (%)")
|
||||
items_b = []
|
||||
v1 = series_v1.get("Corners", [])
|
||||
bm = bucket_means(v1, BUCKETS)
|
||||
if bm:
|
||||
fb = [xf((i + 0.5) / len(bm)) for i in range(len(bm))]
|
||||
s += polyline(list(zip(fb, map(ymb, bm))), V1_COLOR, 1.5)
|
||||
items_b.append((V1_COLOR, f"Run 1 (old) — {len(v1)} evals in {len(bm)} buckets"))
|
||||
c3 = series_v2.get("Corners", [])
|
||||
if c3:
|
||||
rm = rolling(c3, TREND_WINDOW)
|
||||
fr = [xf((i + 1) / len(rm)) for i in range(len(rm))]
|
||||
s += polyline(list(zip(fr, map(ymb, rm))), COLORS["Corners"], 3.5)
|
||||
items_b.append((COLORS["Corners"], f"Run 3 trend (rolling-{TREND_WINDOW})"))
|
||||
s += legend(items_b, x0 + 12, y1b + 14)
|
||||
|
||||
s += guide_block(CW, H, WINRATE_GUIDE)
|
||||
s += "</svg>\n"
|
||||
return 1 if write_svg(out, s) else 0
|
||||
|
||||
|
||||
# ---------- graph 2: loss curves ----------
|
||||
|
||||
def graph_losses(rows, out):
|
||||
if not rows:
|
||||
print("[skip] no metric rows -> no loss graph")
|
||||
return 0
|
||||
|
||||
def col(key, log=False):
|
||||
vals = []
|
||||
for r in rows:
|
||||
try:
|
||||
v = abs(float(r[key])) # abs: log panels plot magnitude
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if log and v <= 0:
|
||||
continue
|
||||
vals.append(v)
|
||||
return vals
|
||||
|
||||
critic = col("critic_loss", log=True)
|
||||
actor = col("actor_loss", log=True)
|
||||
alpha = col("alpha")
|
||||
if not (critic or actor or alpha):
|
||||
print("[skip] metric rows lack critic_loss/actor_loss/alpha -> no loss graph")
|
||||
return 0
|
||||
|
||||
PH, GAP = 210, 55
|
||||
H = M_T + 3 * PH + 2 * GAP + M_B + GUIDE_H
|
||||
x0, x1 = M_L, W - M_R
|
||||
n = len(rows)
|
||||
s = svg_open(W, H, "SAC-LSTM training losses (x = metric line number)")
|
||||
|
||||
def panel(top, vals, title, log, ylab, fixed_range=None):
|
||||
nonlocal s
|
||||
y0, y1 = top + PH, top
|
||||
if not vals:
|
||||
s += (f'<text x="{x0 + 10}" y="{top + 20}" font-size="12" fill="#a00">'
|
||||
f"{esc(title)}: no valid points</text>\n")
|
||||
return
|
||||
if log:
|
||||
lo = math.floor(math.log10(min(vals)))
|
||||
hi = math.ceil(math.log10(max(vals)))
|
||||
if lo == hi:
|
||||
hi = lo + 1
|
||||
yt = ticks_log(lo, hi, y0, y1)
|
||||
else:
|
||||
lo, hi = fixed_range or (min(vals), max(vals))
|
||||
if lo == hi:
|
||||
hi = lo + 1
|
||||
yt = ticks_linear(lo, hi, y0, y1, n=5, fmt="{:.3g}")
|
||||
s += (f'<text x="{x0 + 6}" y="{top + 16}" font-size="13" font-weight="bold">'
|
||||
f"{esc(title)}</text>\n")
|
||||
s += axis(x0, y0, x1, y1, ticks_linear(1, n, x0, x1, fmt="{:.0f}"), yt,
|
||||
"", ylab)
|
||||
ym = map_fn(y0, y1, lo, hi, log=log)
|
||||
def panel_real_fights(games, geo):
|
||||
_, pt, pb, x0, x1 = geo
|
||||
s = ""
|
||||
if not games:
|
||||
s += f'<text x="{x0 + 10}" y="{pt + 40}" font-size="12" fill="#a00">' \
|
||||
"no game rows found</text>\n"
|
||||
return
|
||||
edges = list(range(0, len(games) + 1, GAME_BUCKET))
|
||||
if edges[-1] != len(games):
|
||||
edges.append(len(games))
|
||||
buckets = list(zip(edges[:-1], edges[1:]))
|
||||
xm, ym = map_fn(x0, x1, 1, max(len(games), 2)), map_fn(pb, pt, 0, 100)
|
||||
s += hgrid(x0, x1, [ym(v) for v in range(0, 101, 20)])
|
||||
step = max(GAME_BUCKET, GAME_BUCKET * (len(games) // GAME_BUCKET // 8 + 1))
|
||||
xt = [(str(v), xm(v)) for v in range(step, len(games) + 1, step)]
|
||||
s += axis(x0, pb, x1, pt, xt, ticks_linear(0, 100, pb, pt, n=6),
|
||||
f"game number ({GAME_BUCKET}-game buckets)", "win %")
|
||||
opponents = []
|
||||
for opp, _ in games:
|
||||
if opp not in opponents:
|
||||
opponents.append(opp)
|
||||
items = []
|
||||
for name in opponents:
|
||||
c = COLORS.get(name, "#7f7f7f")
|
||||
by_b = {}
|
||||
for bi, (lo, hi) in enumerate(buckets):
|
||||
sub = [w for o, w in games[lo:hi] if o == name]
|
||||
if sub:
|
||||
by_b[bi] = 100.0 * sum(sub) / len(sub)
|
||||
pts = []
|
||||
for i, r in enumerate(rows):
|
||||
try:
|
||||
v = abs(float(r[title_key(title)]))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if log and v <= 0:
|
||||
continue
|
||||
pts.append((x0 + (x1 - x0) * i / max(n - 1, 1), ym(v)))
|
||||
s += polyline(pts, "#1f77b4", 1.6)
|
||||
segs, prev = [], None
|
||||
for bi in sorted(by_b):
|
||||
if prev is not None and bi != prev + 1:
|
||||
segs.append(pts)
|
||||
pts = []
|
||||
center = (buckets[bi][0] + buckets[bi][1]) / 2
|
||||
pts.append((xm(center), ym(by_b[bi])))
|
||||
prev = bi
|
||||
if len(pts) >= 2:
|
||||
segs.append(pts)
|
||||
for seg in segs:
|
||||
s += polyline(seg, c, 3.5)
|
||||
n_played = sum(1 for o, _ in games if o == name)
|
||||
items.append((c, f"{name} ({n_played} games)"))
|
||||
s += legend(items, x0 + 12, pb + 52)
|
||||
return s
|
||||
|
||||
def title_key(title):
|
||||
return {"critic_loss (log scale)": "critic_loss",
|
||||
"actor_loss |abs| (log scale)": "actor_loss",
|
||||
"alpha (temperature)": "alpha"}[title]
|
||||
def panel_losses(rows, geo):
|
||||
_, pt, pb, x0, x1 = geo
|
||||
s = ""
|
||||
critic = metric_col(rows, "critic_loss", positive=True)
|
||||
actor = metric_col(rows, "actor_loss") # abs() applied; sign dropped
|
||||
actor = [(i, v) for i, v in actor if v > 0]
|
||||
if not (critic or actor):
|
||||
s += f'<text x="{x0 + 10}" y="{pt + 40}" font-size="12" fill="#a00">' \
|
||||
"no valid loss points</text>\n"
|
||||
return
|
||||
allv = [v for _, v in critic + actor]
|
||||
lo, hi = math.floor(math.log10(min(allv))), math.ceil(math.log10(max(allv)))
|
||||
if lo == hi:
|
||||
hi = lo + 1
|
||||
n = len(rows)
|
||||
xm = lambda i: x0 + (x1 - x0) * i / max(n - 1, 1)
|
||||
ym = map_fn(pb, pt, lo, hi, log=True)
|
||||
s += hgrid(x0, x1, [ym(10 ** e) for e in range(lo, hi + 1)])
|
||||
s += axis(x0, pb, x1, pt, ticks_linear(1, n, x0, x1, n=5),
|
||||
ticks_log(lo, hi, pb, pt), "metric line number", "loss (log)")
|
||||
s += polyline([(xm(i), ym(v)) for i, v in critic], CRITIC_C, 1.8)
|
||||
s += polyline([(xm(i), ym(v)) for i, v in actor], ACTOR_C, 1.8)
|
||||
s += legend([(CRITIC_C, "critic_loss"), (ACTOR_C, "|actor_loss|")],
|
||||
x0 + 12, pb + 52)
|
||||
return s
|
||||
|
||||
panel(M_T, critic, "critic_loss (log scale)", True, "critic_loss")
|
||||
panel(M_T + PH + GAP, actor, "actor_loss |abs| (log scale)", True, "|actor_loss|")
|
||||
panel(M_T + 2 * (PH + GAP), alpha, "alpha (temperature)", False, "alpha",
|
||||
fixed_range=(0, max(1.0, max(alpha))))
|
||||
s += (f'<text x="{x0 + (x1 - x0) // 2}" y="{H - GUIDE_H - 8}" text-anchor="middle" '
|
||||
f'font-size="12">metric line number ({n} rows)</text>\n')
|
||||
s += guide_block(W, H, LOSSES_GUIDE)
|
||||
def panel_alpha(rows, geo):
|
||||
_, pt, pb, x0, x1 = geo
|
||||
s = ""
|
||||
alpha = metric_col(rows, "alpha")
|
||||
if not alpha:
|
||||
s += f'<text x="{x0 + 10}" y="{pt + 40}" font-size="12" fill="#a00">' \
|
||||
"no alpha points</text>\n"
|
||||
return
|
||||
n = len(rows)
|
||||
hi = max(1.0, max(v for _, v in alpha))
|
||||
xm = lambda i: x0 + (x1 - x0) * i / max(n - 1, 1)
|
||||
ym = map_fn(pb, pt, 0, hi)
|
||||
s += hgrid(x0, x1, [ym(v) for v in
|
||||
[hi * k / 4 for k in range(5)]])
|
||||
s += axis(x0, pb, x1, pt, ticks_linear(1, n, x0, x1, n=5),
|
||||
ticks_linear(0, hi, pb, pt, n=5, fmt="{:.3g}"),
|
||||
"metric line number", "alpha")
|
||||
s += polyline([(xm(i), ym(v)) for i, v in alpha], "#9467bd", 1.8)
|
||||
return s
|
||||
|
||||
def panel_throughput(rows, geo):
|
||||
_, pt, pb, x0, x1 = geo
|
||||
s = ""
|
||||
eps = []
|
||||
for r in rows:
|
||||
try:
|
||||
eps.append(float(r["epoch"]))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
rates = [] # games/hour per inter-row interval
|
||||
for a, b in zip(eps, eps[1:]):
|
||||
dt = b - a
|
||||
if dt > 0:
|
||||
rates.append(3600.0 * GAMES_PER_ROW / dt)
|
||||
if not rates:
|
||||
s += f'<text x="{x0 + 10}" y="{pt + 40}" font-size="12" fill="#a00">' \
|
||||
"no usable epoch timestamps</text>\n"
|
||||
return
|
||||
bm = bucket_means(rates, RATE_BUCKET)
|
||||
xm = map_fn(x0, x1, 1, len(rates))
|
||||
ymax = max(max(rates), max(bm)) * 1.1
|
||||
ym = map_fn(pb, pt, 0, ymax)
|
||||
s += hgrid(x0, x1, [ym(ymax * k / 4) for k in range(5)])
|
||||
step = max(1, len(rates) // 10)
|
||||
xt = [(str(v), xm(v)) for v in range(step, len(rates) + 1, step)]
|
||||
s += axis(x0, pb, x1, pt, xt, ticks_linear(0, ymax, pb, pt, n=5, fmt="{:.0f}"),
|
||||
f"chunk interval ({GAMES_PER_ROW}-game chunks)", "games / hour")
|
||||
s += dots([(xm(i + 1), ym(v)) for i, v in enumerate(rates)], "#7f7f7f", r=1.6)
|
||||
if len(bm) >= 2:
|
||||
ctr = [xm(round((i + 0.5) * len(rates) / len(bm))) for i in range(len(bm))]
|
||||
s += polyline(list(zip(ctr, map(ym, bm))), "#2ca02c", 3.5)
|
||||
|
||||
|
||||
# ---------- assembly ----------
|
||||
return s
|
||||
|
||||
def build_dashboard(campaign, metrics_f, games_f, out):
|
||||
series = parse_eval_series(campaign)
|
||||
print("[info] evals parsed: " +
|
||||
(", ".join(f"{k}={len(v)}" for k, v in sorted(series.items())) or "(none)"))
|
||||
rows = parse_metrics(metrics_f)
|
||||
print(f"[info] metric rows parsed: {len(rows)}")
|
||||
games = parse_games(games_f)
|
||||
print(f"[info] game rows parsed: {len(games)}")
|
||||
|
||||
s = (f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" '
|
||||
f'viewBox="0 0 {W} {H}" font-family="sans-serif">\n'
|
||||
f'<rect width="{W}" height="{H}" fill="white"/>\n'
|
||||
f'<text x="{W // 2}" y="32" text-anchor="middle" font-size="21" '
|
||||
f'font-weight="bold">SAC-LSTM campaign dashboard - live run (current only)</text>\n'
|
||||
f'<text x="{W // 2}" y="56" text-anchor="middle" font-size="12" fill="#555">'
|
||||
f'generated {datetime.now():%Y-%m-%d %H:%M:%S} - auto-reloads every 60 s '
|
||||
f'(open this file in Chrome)</text>\n')
|
||||
|
||||
drawers = [
|
||||
(ROWS[1], PANEL_TITLES[0],
|
||||
"raw dots = single test matches, thick = rolling-mean-%d" % TREND_WINDOW,
|
||||
lambda: panel_test_matches(series, ROWS[1])),
|
||||
(ROWS[2], PANEL_TITLES[1],
|
||||
"training_log.jsonl only - learning in REAL battles, not tests",
|
||||
lambda: panel_real_fights(games, ROWS[2])),
|
||||
(ROWS[3], PANEL_TITLES[2],
|
||||
"training_metrics.jsonl - big early spikes are normal",
|
||||
lambda: panel_losses(rows, ROWS[3])),
|
||||
(ROWS[4], PANEL_TITLES[3],
|
||||
"training_metrics.jsonl - high = exploring, low = exploiting",
|
||||
lambda: panel_alpha(rows, ROWS[4])),
|
||||
(ROWS[5], PANEL_TITLES[4],
|
||||
"method: training_metrics.jsonl 'epoch' deltas (training_log.jsonl has "
|
||||
"no timestamps); 1 row = one 10-game chunk",
|
||||
lambda: panel_throughput(rows, ROWS[5])),
|
||||
]
|
||||
for geo, title, sub, drawer in drawers:
|
||||
s += header(geo[3], geo[0], title, sub)
|
||||
s += drawer()
|
||||
|
||||
s += guide_block(W, H, GUIDE_LINES)
|
||||
s += RELOAD_JS + "\n"
|
||||
s += "</svg>\n"
|
||||
return 1 if write_svg(out, s) else 0
|
||||
return write_svg(out, s)
|
||||
|
||||
|
||||
# ---------- selftest ----------
|
||||
@@ -381,33 +490,48 @@ def selftest():
|
||||
"garbage line\n"
|
||||
">>> [eval] win rate: 7/10 (70%) vs Crazy\n"
|
||||
">>> [eval] win rate: broken\n"
|
||||
">>> [eval] win rate: 5/10 (50%) vs Corners\n")
|
||||
">>> [eval] win rate: 5/10 (50%) vs Corners\n"
|
||||
">>> [eval] win rate: 4/10 (40%) vs Crazy\n")
|
||||
ser = parse_eval_series(td / "log")
|
||||
assert ser == {"Corners": [30.0, 50.0], "Crazy": [70.0]}, ser
|
||||
assert ser == {"Corners": [30.0, 50.0], "Crazy": [70.0, 40.0]}, ser
|
||||
assert rolling([10] * 25, 20)[-1] == 10.0
|
||||
assert rolling([1, 2, 3], 20) == [1.0, 1.5, 2.0]
|
||||
assert bucket_means(list(range(1287)), BUCKETS) is not None
|
||||
assert len(bucket_means(list(range(1287)), BUCKETS)) == BUCKETS
|
||||
bm = bucket_means([0, 10], BUCKETS)
|
||||
assert len(bucket_means(list(range(1287)), RATE_BUCKET)) == RATE_BUCKET
|
||||
bm = bucket_means([0, 10], RATE_BUCKET)
|
||||
assert bm == [0.0, 10.0], bm # fewer points than buckets -> no empty buckets
|
||||
(td / "m.jsonl").write_text(
|
||||
'{"critic_loss": 10, "actor_loss": -2, "alpha": 0.5}\n'
|
||||
'{"epoch": 1000.0, "critic_loss": 10, "actor_loss": -2, "alpha": 0.5}\n'
|
||||
"not json\n"
|
||||
'{"critic_loss": 100, "actor_loss": -4, "alpha": 0.25}\n')
|
||||
'{"epoch": 1060.0, "critic_loss": 100, "actor_loss": -4, "alpha": 0.25}\n'
|
||||
'{"epoch": 1090.0, "critic_loss": 50, "actor_loss": 3, "alpha": 0.2}\n')
|
||||
rows = parse_metrics(td / "m.jsonl")
|
||||
assert len(rows) == 2 and rows[1]["critic_loss"] == 100
|
||||
ok = graph_losses(rows, td / "g2.svg") and graph_eval(ser, {"Corners": [0, 10]}, td / "g1.svg")
|
||||
assert ok and (td / "g1.svg").stat().st_size > 500
|
||||
g1 = (td / "g1.svg").read_text()
|
||||
g2 = (td / "g2.svg").read_text()
|
||||
ET.fromstring(g1) # whole doc must parse -> closing tag present
|
||||
ET.fromstring(g2)
|
||||
assert "How to read" in g1 and "How to read" in g2, "reading guide missing"
|
||||
assert 'width="1200"' in g1, "canvas must be >=1200 wide"
|
||||
assert g1.count("<circle") == sum(len(v) for v in ser.values()) # one dot per raw eval
|
||||
n_trend = sum(1 for v in ser.values() if len(v) >= 2) + 2 # trends + v1 buckets + v3 trend
|
||||
assert g1.count("<polyline") == n_trend
|
||||
assert graph_eval({}, {}, td / "none.svg") == 0 # missing data handled
|
||||
assert len(rows) == 3 and rows[1]["critic_loss"] == 100
|
||||
glines = []
|
||||
for i in range(150): # 2 full GAME_BUCKETs, both opponents in both
|
||||
glines.append(json.dumps(
|
||||
{"type": "game", "round": i % 10 + 1, "ticks": 100,
|
||||
"score": i % 3, "total_score": i, "win": i % 3 == 0,
|
||||
"opponent": ("Corners", "Crazy")[i % 2]}))
|
||||
(td / "g.jsonl").write_text("\n".join(glines) + "\n")
|
||||
games = parse_games(td / "g.jsonl")
|
||||
assert len(games) == 150 and games[0] == ("Corners", True)
|
||||
assert games[-1] == ("Crazy", False) # i=149: odd -> Crazy; 149%3!=0 -> loss
|
||||
assert metric_col(rows, "actor_loss") == [(0, 2.0), (1, 4.0), (2, 3.0)]
|
||||
|
||||
dash = td / "dash.svg"
|
||||
assert build_dashboard(td / "log", td / "m.jsonl", td / "g.jsonl", dash)
|
||||
text = dash.read_text()
|
||||
ET.fromstring(text) # whole doc must parse -> closing tag present
|
||||
assert RELOAD_JS in text, "auto-reload script missing"
|
||||
for t in PANEL_TITLES:
|
||||
assert t in text, f"panel title missing: {t}"
|
||||
assert text.count(PANEL_TITLES[0]) == 1
|
||||
assert "How to read" in text, "reading guide missing"
|
||||
assert 'width="1400"' in text and 'height="2000"' in text
|
||||
# circles: 4 eval dots (panel 1) + 2 throughput rate dots (panel 5)
|
||||
assert text.count("<circle") == 6, text.count("<circle")
|
||||
# 2 trends + 2 real-fight opp lines + 2 losses + 1 alpha + 1 throughput
|
||||
assert text.count("<polyline") == 8, text.count("<polyline")
|
||||
print("selftest OK")
|
||||
|
||||
|
||||
@@ -418,33 +542,19 @@ def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("-")]
|
||||
campaign = Path(args[0]) if len(args) > 0 else ROOT / "campaign_v2_stdout.log"
|
||||
metrics = Path(args[1]) if len(args) > 1 else ROOT / "training_metrics.jsonl"
|
||||
v1log = Path(args[2]) if len(args) > 2 else ROOT / "weights_v1_archive" / "campaign_stdout.log"
|
||||
games = Path(args[2]) if len(args) > 2 else ROOT / "training_log.jsonl"
|
||||
outdir = Path(args[3]) if len(args) > 3 else ROOT / "docs"
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
made = 0
|
||||
series_v2 = parse_eval_series(campaign)
|
||||
print(f"[info] v2 evals parsed: " +
|
||||
", ".join(f"{k}={len(v)}" for k, v in sorted(series_v2.items())) or "(none)")
|
||||
series_v1 = parse_eval_series(v1log)
|
||||
print(f"[info] v1 evals parsed: " +
|
||||
", ".join(f"{k}={len(v)}" for k, v in sorted(series_v1.items())) or "(none)")
|
||||
try:
|
||||
made += graph_eval(series_v2, series_v1, outdir / "graph_eval_winrates.svg")
|
||||
ok = build_dashboard(campaign, metrics, games, outdir / "campaign_dashboard.svg")
|
||||
except Exception as e:
|
||||
print(f"[error] win-rate graph failed (loss graph still attempted): {e}")
|
||||
|
||||
rows = parse_metrics(metrics)
|
||||
print(f"[info] metric rows parsed: {len(rows)}")
|
||||
try:
|
||||
made += graph_losses(rows, outdir / "graph_losses.svg")
|
||||
except Exception as e:
|
||||
print(f"[error] loss graph failed: {e}")
|
||||
|
||||
if made:
|
||||
print(f"[done] {made} graph(s) written to {outdir}")
|
||||
print(f"[error] dashboard build failed: {e}")
|
||||
ok = False
|
||||
if ok:
|
||||
print(f"[done] dashboard written to {outdir / 'campaign_dashboard.svg'}")
|
||||
else:
|
||||
print("[error] nothing plotted - check paths above")
|
||||
print("[error] dashboard NOT updated - check paths above")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Keep docs/campaign_dashboard.svg fresh: regenerate every 60 s.
|
||||
# Errors go to stderr and never exit the loop silently.
|
||||
dir=$(dirname "$0")
|
||||
while :; do
|
||||
if ! python3 "$dir/plot_progress.py"; then
|
||||
echo "[watch_dashboard] $(date '+%F %T') regeneration failed (see error above)" >&2
|
||||
fi
|
||||
sleep 60
|
||||
done
|
||||
Reference in New Issue
Block a user