Python
#!/usr/bin/env python3
"""Plot the per-epoch curves in one or more runs' `history.jsonl`.
python tools/plot_history.py training/runs/g6bv_s
python tools/plot_history.py .../g6bv_s .../g6vv_s -o /tmp/gen6.png
python tools/plot_history.py $RUNS_DIR/g6*_s --tail 60
This answers "is it still going down, and is it healthy" — the question you
have to settle before relaunching a run for more epochs.
Six panels:
J14 PA-MPJPE the reported metric. EMA solid, RAW faint. The star is the
best epoch; if it sits at the right-hand edge the run has not
plateaued.
J14 MPJPE the same but without Procrustes, so it still sees body-size
and depth error that PA-MPJPE removes.
train total teacher agreement on the training mixture.
learning rate OneCycleLR peaks at 10% of the planned epochs and anneals to
~0 at the end. A run stopped mid-anneal has left the whole
low-LR refinement phase unspent, which is where the last few
mm usually come from — read this panel before concluding a
flat metric curve means "converged".
loss terms every term in the epoch record, normalised to its own epoch-1
value so terms three orders of magnitude apart share an axis.
`verts` appears only on --w-verts runs.
health EMA/RAW ratio (left) and skipped steps per epoch (right).
Ratio below 1 means the average is ahead of the raw weights,
which is the normal state. A spike above 1 is the cheapest
detector there is for an intra-epoch excursion the loss guard
never saw.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg") # write a file; there is no display on a cluster
import matplotlib.pyplot as plt
# DistillConfig.anomaly_skip_patience: consecutive anomalous steps that trigger
# an EMA rollback. An epoch reporting exactly this many skips took one.
ROLLBACK_SKIPS = 50
def find_history(p: Path) -> Path | None:
"""history.jsonl for a run, whether given the run dir or its parent.
A launcher that writes to $RUNS_DIR/<name> while train_ddp.py
creates <name>/ again underneath leaves so the file usually sits one level deeper
than the directory you name on the command line.
"""
if p.is_file():
return p
hits = sorted(p.glob("**/history.jsonl"))
return hits[0] if hits else None
def load(path: Path, tail: int | None):
rows = [json.loads(l) for l in path.open() if l.strip()]
return rows[-tail:] if tail else rows
def series(rows, *keys, default=None):
"""rows[i][k1][k2]... as a list, with None where the record is missing it."""
out = []
for r in rows:
v = r
for k in keys:
v = (v or {}).get(k) if isinstance(v, dict) else None
out.append(default if v is None else v)
return out
def plot(runs, out: Path, target_epochs: int | None):
fig, axes = plt.subplots(2, 3, figsize=(19, 9))
ax_pa, ax_mj, ax_tr, ax_lr, ax_terms, ax_health = axes.flat
colours = plt.rcParams["axes.prop_cycle"].by_key()["color"]
for i, (name, rows, cfg) in enumerate(runs):
c = colours[i % len(colours)]
ep = [r["epoch"] + 1 for r in rows]
label = name
if cfg:
label += f" ({cfg.get('preset', '?')}"
if cfg.get("w_verts"):
label += f", w_verts={cfg['w_verts']}"
label += ")"
pa_e = series(rows, "dpw_ema", "J14_PA_MPJPE")
pa_r = series(rows, "dpw_raw", "J14_PA_MPJPE")
if any(v is not None for v in pa_e):
ax_pa.plot(ep, pa_e, color=c, lw=1.8, label=label)
ax_pa.plot(ep, pa_r, color=c, lw=0.8, alpha=0.35)
best = min(v for v in pa_e if v is not None)
bi = pa_e.index(best)
ax_pa.plot(ep[bi], best, "*", color=c, ms=15, zorder=5)
ax_pa.annotate(f"{best:.2f} @{ep[bi]}", (ep[bi], best), color=c,
fontsize=8, xytext=(4, -11), textcoords="offset points")
mj = series(rows, "dpw_ema", "J14_MPJPE")
if any(v is not None for v in mj):
ax_mj.plot(ep, mj, color=c, lw=1.6, label=label)
ax_tr.plot(ep, series(rows, "train", "total"), color=c, lw=1.6, label=label)
ax_lr.plot(ep, series(rows, "lr"), color=c, lw=1.6, label=label)
# Terms live on wildly different scales (simcc ~2.2, reproj ~5e-4), so
# each is divided by its own first value: the shape is what matters.
for j, k in enumerate(sorted(rows[0].get("train", {}))):
if k == "total":
continue
v = series(rows, "train", k)
if not any(v) or v[0] in (None, 0):
continue
ax_terms.plot(ep, [x / v[0] for x in v], lw=1.3,
ls=["-", "--", ":", "-."][i % 4],
color=colours[(j + 2) % len(colours)],
label=f"{k} ({name})" if len(runs) > 1 else k)
ratio = [(a / b) if (a and b) else None for a, b in zip(pa_e, pa_r)]
if any(v is not None for v in ratio):
ax_health.plot(ep, ratio, color=c, lw=1.4, label=f"EMA/RAW {name}")
skipped = series(rows, "skipped", default=0)
ax_health.twinx().plot(ep, skipped, color=c, lw=0.9, alpha=0.4, ls=":")
# An EMA rollback halves the LR and clears the Adam moments, so it
# permanently reshapes the schedule -- the discontinuity it leaves in
# the LR panel is otherwise unexplainable. It fires at exactly
# anomaly_skip_patience (50) consecutive skips, which is why that count
# in a single epoch is its fingerprint.
for e, s in zip(ep, skipped):
if s >= ROLLBACK_SKIPS:
for ax in (ax_pa, ax_lr):
ax.axvline(e, color=c, ls=":", lw=1.2, alpha=0.7)
ax_lr.annotate(f"EMA rollback ep{e}\nLR x0.5", (e, max(series(rows, "lr"))),
color=c, fontsize=7, xytext=(4, -20),
textcoords="offset points")
for ax, title, ylab in (
(ax_pa, "3DPW-val J14 PA-MPJPE (EMA bold, RAW faint)", "mm"),
(ax_mj, "3DPW-val J14 MPJPE (EMA)", "mm"),
(ax_tr, "train total loss", "loss"),
(ax_lr, "learning rate", "lr"),
(ax_terms, "train loss terms, relative to epoch 1", "x epoch-1"),
(ax_health, "health: EMA/RAW ratio (solid), skipped steps (dotted, right)", "ratio")):
ax.set_title(title, fontsize=10)
ax.set_xlabel("epoch")
ax.set_ylabel(ylab)
ax.grid(alpha=0.3)
if target_epochs:
for ax in (ax_pa, ax_mj, ax_tr, ax_lr):
ax.axvline(target_epochs, color="k", ls="--", lw=0.8, alpha=0.5)
ax_lr.annotate(f"planned end (ep {target_epochs})", (target_epochs, 0),
fontsize=8, rotation=90, va="bottom", ha="right", alpha=0.6)
ax_health.axhline(1.0, color="k", lw=0.7, alpha=0.4)
for ax in (ax_pa, ax_mj, ax_tr, ax_lr, ax_health):
ax.legend(fontsize=8)
ax_terms.legend(fontsize=6, ncol=2)
fig.tight_layout()
fig.savefig(out, dpi=130)
print(f"wrote {out}")
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("runs", nargs="+", type=Path,
help="run directories (or history.jsonl paths)")
ap.add_argument("-o", "--out", type=Path, default=Path("history.png"))
ap.add_argument("--tail", type=int, default=None,
help="plot only the last N epochs")
args = ap.parse_args()
loaded = []
for p in args.runs:
h = find_history(p)
if h is None:
print(f" no history.jsonl under {p} — skipped")
continue
rows = load(h, args.tail)
cfg_p = h.parent / "run_config.json"
cfg = json.loads(cfg_p.read_text()) if cfg_p.is_file() else {}
loaded.append((p.name, rows, cfg))
pa = [r.get("dpw_ema", {}).get("J14_PA_MPJPE") for r in rows if r.get("dpw_ema")]
best = min(pa) if pa else float("nan")
print(f" {p.name:10s} {len(rows):4d} epochs, best EMA J14 PA {best:6.2f} mm"
f" (target {cfg.get('epochs', '?')})")
if not loaded:
raise SystemExit("nothing to plot")
targets = {c.get("epochs") for _, _, c in loaded if c.get("epochs")}
plot(loaded, args.out, targets.pop() if len(targets) == 1 else None)
if __name__ == "__main__":
main()