Skip to content
Python

feat: DDP training for Jean Zay + 3DPW held-out validation

mohamdev6f0173ac3a6b
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.py
new file mode 100755
index 0000000..f6eb157
--- /dev/null
+++ b/instanthmr_distill_train/train_distill_jz.py
@@ -0,0 +1,723 @@
+#!/usr/bin/env python3
+"""Multi-GPU (DDP) training of the InstantHMR baseline on Jean Zay.
+
+This is a thin driver around ``train_distill_mhr_only.py``: it imports the
+config, dataset, model, loss and metrics from there rather than copying them,
+so the changes queued in the change plan only ever have to be made once. What
+lives here is the four things the single-GPU script cannot do on a cluster.
+
+**1. DistributedDataParallel.** Ranks come from Slurm (``SLURM_PROCID`` /
+``SLURM_LOCALID`` / ``SLURM_NTASKS``) with ``MASTER_ADDR`` taken from the first
+node in the allocation, so one ``srun`` line launches the whole job.
+
+**2. Mixture control instead of caps.** SA-1B is 3.40 M of the 3.74 M crops —
+91% of the corpus — so uniform sampling trains almost exclusively on it. The
+existing ``per_dataset_caps`` is a ``random.sample`` applied once at dataset
+construction, which *deletes* crops for the whole run. Here a
+:class:`DistributedWeightedSampler` draws with replacement from per-dataset
+weights, so every crop stays reachable and the mixture is a knob (``--mix``),
+not a demolition. Default is weight proportional to sqrt(count), which pulls
+SA-1B from 91% to ~69% while keeping COCO and MPII visible.
+
+**3. Epochs decoupled from corpus size.** ``--samples-per-epoch`` fixes the
+epoch length, so "epoch" means a checkpoint interval rather than "one pass over
+3.74 M crops" (~58 k steps, ~4 h on one V100). Size it so an epoch is 30-60 min
+and a preemption costs at most that.
+
+**4. Resume by resubmitting.** Every epoch writes ``last.pth`` atomically with
+the *full* state — model, EMA, optimiser, scaler, scheduler, RNG, epoch, best
+metrics. Relaunching the same sbatch picks it up. Note this differs from the
+single-GPU script, which resumes from ``best_student_model_v3.pth`` and so
+silently discards every epoch since the last improvement.
+
+Offline caveat: Jean Zay compute nodes have no internet, and
+``InstantHMRStudent(pretrained=True)`` pulls timm weights from HuggingFace. Run
+``--warm-cache`` once on a login or ``prepost`` node to populate
+``$TORCH_HOME``/``$HF_HOME`` before the first training job.
+
+ # once, on a node with the proxy
+ python instanthmr_distill_train/train_distill_jz.py --warm-cache
+
+ # then, under srun
+ srun python instanthmr_distill_train/train_distill_jz.py \
+ --data_root $DATA_ROOT --output_dir $RUNS_DIR/baseline
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import json
+import math
+import os
+import random
+import shutil
+import socket
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.optim as optim
+import torch.optim.lr_scheduler as lr_scheduler
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.optim.swa_utils import AveragedModel
+from torch.utils.data import DataLoader, Sampler
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import train_distill_mhr_only as T # noqa: E402 (sets up matplotlib/ort at import)
+
+
+# ============================================================
+# Distributed plumbing
+# ============================================================
+def setup_distributed() -> tuple[int, int, int]:
+ """Initialise the process group from Slurm's environment.
+
+ Returns ``(rank, local_rank, world_size)``. Falls back to a single-process
+ group when not launched under ``srun``, so the same file runs on a laptop.
+ """
+ if "SLURM_PROCID" in os.environ and int(os.environ.get("SLURM_NTASKS", 1)) > 1:
+ rank = int(os.environ["SLURM_PROCID"])
+ local_rank = int(os.environ["SLURM_LOCALID"])
+ world_size = int(os.environ["SLURM_NTASKS"])
+ if "MASTER_ADDR" not in os.environ:
+ nodelist = os.environ["SLURM_JOB_NODELIST"]
+ head = subprocess.check_output(
+ ["scontrol", "show", "hostnames", nodelist], text=True
+ ).split()[0]
+ os.environ["MASTER_ADDR"] = head
+ os.environ.setdefault("MASTER_PORT", "29500")
+ else:
+ rank, local_rank, world_size = 0, 0, 1
+
+ torch.cuda.set_device(local_rank)
+ if world_size > 1:
+ dist.init_process_group(
+ backend="nccl", rank=rank, world_size=world_size,
+ # A cold Lustre cache can make the first epoch's data loading slow
+ # enough to trip the 30-minute default on the initial barrier.
+ timeout=_dt.timedelta(minutes=90),
+ )
+ return rank, local_rank, world_size
+
+
+def is_main() -> bool:
+ return (not dist.is_initialized()) or dist.get_rank() == 0
+
+
+def log(*a, **kw) -> None:
+ """Print from rank 0 only. Four ranks logging the same line helps nobody."""
+ if is_main():
+ print(*a, **kw, flush=True)
+
+
+def all_reduce_mean(values: dict[str, float], device) -> dict[str, float]:
+ """Average a dict of scalars across ranks so every rank sees one number."""
+ if not dist.is_initialized() or dist.get_world_size() == 1:
+ return values
+ keys = sorted(values)
+ t = torch.tensor([values[k] for k in keys], dtype=torch.float64, device=device)
+ dist.all_reduce(t, op=dist.ReduceOp.SUM)
+ t /= dist.get_world_size()
+ return {k: float(v) for k, v in zip(keys, t.tolist())}
+
+
+# ============================================================
+# Mixture-aware distributed sampling
+# ============================================================
+def dataset_of(npz_path: Path) -> str:
+ """``.../sam3d_gt_aic/annotations/xxx.npz`` -> ``sam3d_gt_aic``."""
+ return npz_path.parent.parent.name
+
+
+def build_mixture_weights(dataset, mix: str | None) -> tuple[torch.Tensor, str]:
+ """Per-sample sampling weights implementing a target dataset mixture.
+
+ ``mix`` is either ``"sqrt"`` / ``"uniform"`` / ``"equal"``, or an explicit
+ ``name=proportion`` list (``sa1b=0.4,aic=0.35,coco=0.15,mpii=0.1``, matched
+ as a substring of the split folder name). Proportions are renormalised, so
+ they need not sum to 1.
+
+ - ``uniform``: every crop equally likely — SA-1B ends up at 91%.
+ - ``sqrt``: proportion proportional to sqrt(count). The usual compromise for
+ a long-tailed corpus: keeps the big split dominant without erasing the
+ small ones.
+ - ``equal``: every split contributes the same number of samples per epoch.
+ With MPII at 7 k crops that means ~50x oversampling, which augmentation
+ cannot disguise — expect it to overfit MPII.
+ """
+ names = [dataset_of(npz) for _, npz in dataset.pairs]
+ counts: dict[str, int] = {}
+ for n in names:
+ counts[n] = counts.get(n, 0) + 1
+ splits = sorted(counts)
+
+ if mix in (None, "sqrt"):
+ target = {s: math.sqrt(counts[s]) for s in splits}
+ elif mix == "uniform":
+ target = {s: float(counts[s]) for s in splits}
+ elif mix == "equal":
+ target = {s: 1.0 for s in splits}
+ else:
+ req: dict[str, float] = {}
+ for tok in mix.split(","):
+ k, _, v = tok.partition("=")
+ req[k.strip()] = float(v)
+ target = {}
+ for s in splits:
+ hit = [v for k, v in req.items() if k in s]
+ if len(hit) > 1:
+ raise ValueError(f"--mix key matches '{s}' more than once: {req}")
+ target[s] = hit[0] if hit else 0.0
+ if sum(target.values()) <= 0:
+ raise ValueError(f"--mix matched no split. Splits present: {splits}")
+
+ tot = sum(target.values())
+ prop = {s: target[s] / tot for s in splits}
+ # Per-sample weight: a split's total probability mass divided evenly over
+ # its crops. WeightedRandomSampler normalises, so absolute scale is free.
+ per_sample = {s: (prop[s] / counts[s] if counts[s] else 0.0) for s in splits}
+ w = torch.tensor([per_sample[n] for n in names], dtype=torch.double)
+
+ rows = [f" {'split':<24}{'crops':>12}{'raw %':>9}{'sampled %':>11}"]
+ total = sum(counts.values())
+ for s in splits:
+ rows.append(f" {s:<24}{counts[s]:>12,}{100*counts[s]/total:>8.1f}%"
+ f"{100*prop[s]:>10.1f}%")
+ return w, "\n".join(rows)
+
+
+class DistributedWeightedSampler(Sampler[int]):
+ """Weighted sampling with replacement, sharded across ranks.
+
+ ``WeightedRandomSampler`` has no distributed variant. Rather than sample
+ per rank — which would give each rank a different mixture and a different
+ epoch length — every rank draws the *same* index list from a generator
+ seeded with ``(seed, epoch)``, then takes its own stride. So the global
+ mixture is exactly the requested one, the shards are disjoint, and the
+ sequence is reproducible from the epoch number alone, which is what makes
+ resume deterministic.
+ """
+
+ def __init__(self, weights: torch.Tensor, num_samples: int,
+ num_replicas: int, rank: int, seed: int = 0):
+ self.weights = weights.double()
+ self.num_replicas = num_replicas
+ self.rank = rank
+ self.seed = seed
+ self.epoch = 0
+ # Round up so every rank gets the same count; DDP deadlocks if one rank
+ # runs out of batches before the others.
+ self.per_rank = int(math.ceil(num_samples / num_replicas))
+ self.total = self.per_rank * num_replicas
+
+ def set_epoch(self, epoch: int) -> None:
+ self.epoch = int(epoch)
+
+ def __len__(self) -> int:
+ return self.per_rank
+
+ def __iter__(self):
+ g = torch.Generator()
+ g.manual_seed(self.seed * 100_003 + self.epoch)
+ idx = torch.multinomial(self.weights, self.total, replacement=True, generator=g)
+ return iter(idx[self.rank::self.num_replicas].tolist())
+
+
+# ============================================================
+# Data
+# ============================================================
+def build_jz_loaders(cfg, args, rank: int, world: int):
+ """Train loader with mixture sampling, plus a small held-out val loader.
+
+ The val split is a fixed slice of the *same* corpus, so it measures
+ agreement with the teacher rather than generalisation — it is a training
+ health signal, not a model-selection metric. Keep it small (``--val-frac``,
+ default 2%): with 3.74 M crops even 1% is 37 k samples, and evaluating it
+ every epoch is pure overhead. See ``--val-3dpw`` in the README notes for
+ the held-out alternative.
+ """
+ geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,
+ geom_scale_range=cfg.geom_scale_range,
+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)
+
+ train_ds = T.SAM3DStudentDataset(
+ cfg.data_root, augment=cfg.augment, max_images=cfg.max_images,
+ per_dataset_caps=None, **geom)
+ val_ds = T.SAM3DStudentDataset(
+ cfg.data_root, augment=False, max_images=cfg.max_images,
+ per_dataset_caps=None, **geom)
+
+ n = len(train_ds)
+ g = torch.Generator().manual_seed(42) # fixed: the split must not
+ perm = torch.randperm(n, generator=g).tolist() # move when --seed changes
+ n_val = max(1, int(n * args.val_frac))
+ val_idx, train_idx = perm[:n_val], perm[n_val:]
+
+ weights_all, report = build_mixture_weights(train_ds, args.mix)
+ log(f"\ndataset mixture ('{args.mix or 'sqrt'}'):\n{report}")
+
+ # Zero out the val indices so the sampler can never draw them.
+ w = weights_all.clone()
+ w[torch.tensor(val_idx, dtype=torch.long)] = 0.0
+
+ sampler = DistributedWeightedSampler(
+ w, num_samples=args.samples_per_epoch,
+ num_replicas=world, rank=rank, seed=args.seed)
+
+ loader_kw = dict(num_workers=cfg.num_workers, pin_memory=True, drop_last=True)
+ if cfg.num_workers > 0:
+ loader_kw.update(persistent_workers=True, prefetch_factor=args.prefetch_factor)
+
+ train_loader = DataLoader(train_ds, batch_size=cfg.batch_size,
+ sampler=sampler, **loader_kw)
+
+ # Val is sharded by plain striding — no weighting, no replacement.
+ val_subset = val_idx[rank::world]
+ val_loader = DataLoader(
+ torch.utils.data.Subset(val_ds, val_subset),
+ batch_size=cfg.batch_size, shuffle=False,
+ num_workers=max(1, cfg.num_workers // 2), pin_memory=True, drop_last=False)
+
+ log(f"train pool {len(train_idx):,} crops | val {n_val:,} "
+ f"({len(val_subset):,} on this rank) | "
+ f"{args.samples_per_epoch:,} samples/epoch")
+ return train_loader, val_loader, sampler, train_ds
+
+
+# ============================================================
+# Checkpointing
+# ============================================================
+def atomic_save(obj, path: Path) -> None:
+ """Write then rename. A job killed mid-``torch.save`` must not leave a
+ truncated ``last.pth`` — that would turn one lost epoch into a lost run."""
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ torch.save(obj, tmp)
+ os.replace(tmp, path)
+
+
+def save_state(path: Path, *, model, ema, optimizer, scaler, scheduler,
+ epoch, best, args, cfg) -> None:
+ atomic_save({
+ "epoch": epoch,
+ "model_state_dict": unwrap(model).state_dict(),
+ "ema_state_dict": ema.state_dict(),
+ "optimizer_state_dict": optimizer.state_dict(),
+ "scaler_state_dict": scaler.state_dict(),
+ "scheduler_state_dict": scheduler.state_dict(),
+ "best": best,
+ "rng": {
+ "python": random.getstate(),
+ "numpy": np.random.get_state(),
+ "torch": torch.get_rng_state(),
+ "cuda": torch.cuda.get_rng_state_all(),
+ },
+ "run": {"seed": args.seed, "epochs": args.epochs,
+ "samples_per_epoch": args.samples_per_epoch,
+ "batch_size": cfg.batch_size, "lr": cfg.lr, "mix": args.mix},
+ }, path)
+
+
+def unwrap(m):
+ return m.module if isinstance(m, (DDP, AveragedModel)) else m
+
+
+# ============================================================
+# Validation
+# ============================================================
+def run_validation(eval_model, loader, criterion, cfg, mhr_module, device):
+ eval_model.eval()
+ acc = {"loss": 0.0, "Mesh_MPJPE": 0.0, "Mesh_PA_MPJPE": 0.0}
+ n = 0
+ for batch in loader:
+ batch = {k: (v.to(device, non_blocking=True) if isinstance(v, torch.Tensor) else v)
+ for k, v in batch.items()}
+ with torch.amp.autocast(device_type="cuda", dtype=torch.float16, enabled=cfg.use_amp):
+ preds = eval_model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ lv = losses["total_loss"].item()
+ if math.isnan(lv) or math.isinf(lv):
+ continue
+ bm = T.evaluate_hmr_batch(preds, batch, cfg, mhr_module)
+ acc["loss"] += lv
+ acc["Mesh_MPJPE"] += bm.get("Mesh_MPJPE", 0.0)
+ acc["Mesh_PA_MPJPE"] += bm.get("Mesh_PA_MPJPE", 0.0)
+ n += 1
+ out = {k: v / max(n, 1) for k, v in acc.items()}
+ out["n"] = float(n)
+ # Average over ranks so the early-stopping decision is identical everywhere.
+ return all_reduce_mean(out, device)
+
+
+# ============================================================
+# Training
+# ============================================================
+def train(args, cfg, rank, local_rank, world):
+ device = torch.device(f"cuda:{local_rank}")
+ T.device = device # the module global some helpers read
+ out_dir = Path(cfg.log_dir)
+ if is_main():
+ out_dir.mkdir(parents=True, exist_ok=True)
+
+ W = np.load(cfg.kp_regressor_path)
+ mhr_module = T.MHRForwardPass(cfg.mhr_model_path, device, kp_regressor=W)
+ criterion = T.DistillationLoss(cfg, mhr_module)
+
+ train_loader, val_loader, sampler, _ = build_jz_loaders(cfg, args, rank, world)
+ steps_per_epoch = len(train_loader)
+
+ dpw_loader = None
+ if args.val_3dpw:
+ if not args.val_3dpw_images:
+ raise SystemExit("--val-3dpw needs --val-3dpw-images")
+ from val3dpw import ThreeDPWValSet
+ dpw = ThreeDPWValSet(args.val_3dpw, args.val_3dpw_images,
+ split=args.val_3dpw_split, stride=args.val_3dpw_stride)
+ # Stride-shard across ranks; the metric is all-reduced afterwards.
+ dpw_loader = torch.utils.data.DataLoader(
+ torch.utils.data.Subset(dpw, list(range(rank, len(dpw), world))),
+ batch_size=cfg.batch_size, shuffle=False, drop_last=False,
+ num_workers=max(1, cfg.num_workers // 2), pin_memory=True)
+ log(f"3DPW {args.val_3dpw_split}: {len(dpw):,} person-frames "
+ f"(stride {args.val_3dpw_stride}) -> selection metric is J12 PA-MPJPE")
+
+ model = T.InstantHMRStudent(cfg, pretrained=not args.no_pretrained).to(device)
+ if args.sync_bn and world > 1:
+ # repvit is 218 BatchNorm2d layers. At batch 64 per GPU the local stats
+ # are already reliable, so this is off by default — it costs an all-
+ # reduce per BN layer per step for no accuracy gain at this batch size.
+ model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
+ ddp_model = DDP(model, device_ids=[local_rank], output_device=local_rank,
+ find_unused_parameters=False) if world > 1 else model
+
+ param_groups = T.build_param_groups(model, cfg.weight_decay,
+ split_wd=cfg.split_weight_decay)
+ optimizer = optim.AdamW(param_groups, lr=cfg.lr, weight_decay=cfg.weight_decay)
+ scaler = torch.amp.GradScaler(device="cuda", enabled=cfg.use_amp)
+ scheduler = lr_scheduler.OneCycleLR(
+ optimizer, max_lr=cfg.lr, epochs=args.epochs,
+ steps_per_epoch=steps_per_epoch, pct_start=0.1)
+ ema = AveragedModel(model, multi_avg_fn=T.make_ema_avg_fn(cfg.ema_decay),
+ use_buffers=True)
+
+ start_epoch = 0
+ best = {"raw": float("inf"), "ema": float("inf"), "overall": float("inf"),
+ "no_improve": 0}
+ last_path = out_dir / "last.pth"
+ if last_path.exists() and not args.no_resume:
+ ck = torch.load(last_path, map_location=device, weights_only=False)
+ prev = ck.get("run", {})
+ for k in ("epochs", "samples_per_epoch", "batch_size"):
+ cur = {"epochs": args.epochs, "samples_per_epoch": args.samples_per_epoch,
+ "batch_size": cfg.batch_size}[k]
+ if prev.get(k) not in (None, cur):
+ log(f"⚠️ {k} changed {prev[k]} -> {cur} since the checkpoint. "
+ f"OneCycleLR's schedule is defined over the ORIGINAL total "
+ f"step count, so the LR curve will not match a fresh run.")
+ unwrap(model).load_state_dict(ck["model_state_dict"])
+ ema.load_state_dict(ck["ema_state_dict"])
+ optimizer.load_state_dict(ck["optimizer_state_dict"])
+ scaler.load_state_dict(ck["scaler_state_dict"])
+ scheduler.load_state_dict(ck["scheduler_state_dict"])
+ best = ck["best"]
+ start_epoch = ck["epoch"] + 1
+ if (r := ck.get("rng")):
+ random.setstate(r["python"]); np.random.set_state(r["numpy"])
+ torch.set_rng_state(r["torch"].cpu())
+ try:
+ torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]])
+ except Exception:
+ pass # a different GPU count than the run that saved it
+ log(f"🔄 resumed from {last_path} at epoch {start_epoch} "
+ f"(best PA-MPJPE {best['overall']:.1f} mm)")
+ else:
+ log("🚀 starting from scratch")
+
+ log(f"steps/epoch {steps_per_epoch} | world {world} | "
+ f"effective batch {cfg.batch_size * world} | peak lr {cfg.lr:.2e}")
+
+ for epoch in range(start_epoch, args.epochs):
+ sampler.set_epoch(epoch)
+ ddp_model.train()
+ run = {"total": 0.0, "3d": 0.0, "2d": 0.0, "simcc": 0.0}
+ nb = 0
+ criterion.set_step(epoch * steps_per_epoch)
+ for i, batch in enumerate(train_loader):
+ batch = {k: (v.to(device, non_blocking=True) if isinstance(v, torch.Tensor) else v)
+ for k, v in batch.items()}
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type="cuda", dtype=torch.float16, enabled=cfg.use_amp):
+ preds = ddp_model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ loss = losses["total_loss"]
+ lv = loss.item()
+
+ # An anomalous step is skipped, but every rank must agree or DDP's
+ # gradient all-reduce deadlocks: rank 0 skipping while rank 1 steps
+ # leaves rank 1 waiting forever. Vote, then act together.
+ bad = int(math.isnan(lv) or math.isinf(lv) or lv > cfg.anomaly_loss_threshold)
+ if world > 1:
+ flag = torch.tensor([bad], device=device)
+ dist.all_reduce(flag, op=dist.ReduceOp.MAX)
+ bad = int(flag.item())
+ if bad:
+ log(f"⚠️ skipping step {i} (loss {lv:.1f})")
+ continue
+
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(ddp_model.parameters(), cfg.grad_clip)
+ scaler.step(optimizer)
+ scaler.update()
+ scheduler.step()
+ ema.update_parameters(unwrap(ddp_model))
+ criterion.set_step(epoch * steps_per_epoch + i)
+
+ run["total"] += lv
+ run["3d"] += losses.get("loss_3d_native", torch.zeros(())).item()
+ run["2d"] += losses.get("loss_2d_native", torch.zeros(())).item()
+ run["simcc"] += losses.get("loss_2d_simcc", torch.zeros(())).item()
+ nb += 1
+ if is_main() and i % args.log_every == 0:
+ log(f" e{epoch+1}/{args.epochs} step {i}/{steps_per_epoch} "
+ f"loss {lv:.4f} lr {scheduler.get_last_lr()[0]:.2e}")
+
+ tr = all_reduce_mean({k: v / max(nb, 1) for k, v in run.items()}, device)
+ raw = run_validation(ddp_model, val_loader, criterion, cfg, mhr_module, device)
+ em = run_validation(ema, val_loader, criterion, cfg, mhr_module, device)
+
+ # 3DPW, when available, replaces the held-in metric for selection: it is
+ # the only signal here that measures generalisation rather than teacher
+ # agreement. The held-in numbers keep printing either way.
+ dpw_raw = dpw_em = None
+ if dpw_loader is not None:
+ import val3dpw
+ dpw_raw = all_reduce_mean(
+ val3dpw.evaluate(ddp_model, dpw_loader, mhr_module, device, cfg.use_amp),
+ device)
+ dpw_em = all_reduce_mean(
+ val3dpw.evaluate(ema, dpw_loader, mhr_module, device, cfg.use_amp),
+ device)
+ raw["Mesh_PA_MPJPE"], em["Mesh_PA_MPJPE"] = (
+ dpw_raw["J12_PA_MPJPE"], dpw_em["J12_PA_MPJPE"])
+ raw["Mesh_MPJPE"], em["Mesh_MPJPE"] = (
+ dpw_raw["J12_MPJPE"], dpw_em["J12_MPJPE"])
+
+ log(f"\n📈 epoch {epoch+1}/{args.epochs} | lr {scheduler.get_last_lr()[0]:.2e}")
+ log(f" train total {tr['total']:.4f} | 3d {tr['3d']:.4f} | "
+ f"2d {tr['2d']:.4f} | simcc {tr['simcc']:.3f}")
+ log(f" valRAW loss {raw['loss']:.4f} | PA-MPJPE {raw['Mesh_PA_MPJPE']:.1f} | "
+ f"MPJPE {raw['Mesh_MPJPE']:.1f} mm")
+ log(f" valEMA loss {em['loss']:.4f} | PA-MPJPE {em['Mesh_PA_MPJPE']:.1f} | "
+ f"MPJPE {em['Mesh_MPJPE']:.1f} mm")
+ if dpw_raw is not None:
+ log(f" 3DPW-{args.val_3dpw_split} RAW J12 PA {dpw_raw['J12_PA_MPJPE']:.1f} "
+ f"MPJPE {dpw_raw['J12_MPJPE']:.1f} | J14 PA {dpw_raw['J14_PA_MPJPE']:.1f} mm")
+ log(f" 3DPW-{args.val_3dpw_split} EMA J12 PA {dpw_em['J12_PA_MPJPE']:.1f} "
+ f"MPJPE {dpw_em['J12_MPJPE']:.1f} | J14 PA {dpw_em['J14_PA_MPJPE']:.1f} mm")
+
+ improved = False
+ if raw["Mesh_PA_MPJPE"] < best["raw"]:
+ best["raw"] = raw["Mesh_PA_MPJPE"]; improved = True
+ if is_main():
+ atomic_save({"model_state_dict": unwrap(model).state_dict(),
+ "val_pa_mpjpe": best["raw"], "source": "raw",
+ "epoch": epoch}, out_dir / "best_student_model_raw.pth")
+ if em["Mesh_PA_MPJPE"] < best["ema"]:
+ best["ema"] = em["Mesh_PA_MPJPE"]; improved = True
+ if is_main():
+ atomic_save({"model_state_dict": unwrap(ema).state_dict(),
+ "val_pa_mpjpe": best["ema"], "source": "ema",
+ "epoch": epoch}, out_dir / "best_student_model_ema.pth")
+ cur = min(raw["Mesh_PA_MPJPE"], em["Mesh_PA_MPJPE"])
+ if cur < best["overall"] - 1e-4:
+ best["overall"] = cur
+ use_ema = em["Mesh_PA_MPJPE"] <= raw["Mesh_PA_MPJPE"]
+ if is_main():
+ src = unwrap(ema) if use_ema else unwrap(model)
+ atomic_save({"model_state_dict": src.state_dict(),
+ "val_pa_mpjpe": cur, "epoch": epoch,
+ "source": "ema" if use_ema else "raw"},
+ out_dir / "best_student_model_v3.pth")
+ log(f"💾 new best {cur:.1f} mm ({'EMA' if use_ema else 'RAW'})")
+ best["no_improve"] = 0 if improved else best["no_improve"] + 1
+
+ if is_main():
+ save_state(out_dir / "last.pth", model=model, ema=ema,
+ optimizer=optimizer, scaler=scaler, scheduler=scheduler,
+ epoch=epoch, best=best, args=args, cfg=cfg)
+ with (out_dir / "history.jsonl").open("a") as fh:
+ fh.write(json.dumps({"epoch": epoch, "train": tr,
+ "val_raw": raw, "val_ema": em,
+ "dpw_raw": dpw_raw, "dpw_ema": dpw_em,
+ "lr": scheduler.get_last_lr()[0]}) + "\n")
+ if world > 1:
+ dist.barrier() # nobody starts epoch N+1 before last.pth is on disk
+
+ if best["no_improve"] >= cfg.early_stop_patience:
+ log(f"\n🛑 early stop: no improvement for {cfg.early_stop_patience} epochs")
+ break
+
+ # The sbatch resubmits itself unless this file exists, so it must be written
+ # on normal completion only — never in an exception path.
+ if is_main():
+ (out_dir / "DONE").write_text(
+ f"best_pa_mpjpe={best['overall']:.3f}\nepochs={args.epochs}\n")
+ log(f"\n🎉 done. best PA-MPJPE {best['overall']:.1f} mm -> {out_dir}")
+
+
+# ============================================================
+# CLI
+# ============================================================
+def parse_args():
+ p = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--data_root", default=os.environ.get("DATA_ROOT"))
+ p.add_argument("--output_dir", required=False,
+ default=os.environ.get("OUTPUT_DIR"))
+ p.add_argument("--mhr_model_path", default=None)
+ p.add_argument("--kp_regressor_path", default=None)
+
+ p.add_argument("--epochs", type=int, default=30)
+ p.add_argument("--batch_size", type=int, default=64,
+ help="PER GPU. Effective batch is this times the world size.")
+ p.add_argument("--lr", type=float, default=3e-4,
+ help="Peak LR for a single GPU; scaled by --lr-scaling.")
+ p.add_argument("--lr-scaling", choices=("linear", "sqrt", "none"), default="linear",
+ help="How to scale --lr with world size. 'linear' is the "
+ "standard rule for SGD-style large-batch training; "
+ "'sqrt' is gentler and sometimes better for Adam.")
+ p.add_argument("--num_workers", type=int, default=9)
+ p.add_argument("--prefetch-factor", type=int, default=4)
+ p.add_argument("--samples-per-epoch", type=int, default=400_000,
+ help="Epoch length in crops. Size it so one epoch is 30-60 "
+ "min: a preemption costs at most one epoch.")
+ p.add_argument("--val-frac", type=float, default=0.02,
+ help="Held-in val fraction. This is a training-health signal "
+ "(agreement with the teacher on the same distribution), "
+ "not a generalisation measure. 0 disables it.")
+ p.add_argument("--val-3dpw", default=os.environ.get("THREEDPW_SEQ"),
+ help="3DPW sequenceFiles/ directory. When given, PA-MPJPE on "
+ "J12 becomes the checkpoint-selection metric.")
+ p.add_argument("--val-3dpw-images", default=os.environ.get("THREEDPW_IMG"),
+ help="3DPW imageFiles/ directory.")
+ p.add_argument("--val-3dpw-split", default="validation",
+ choices=("validation", "test", "train"),
+ help="Keep this at 'validation'. Selecting on 'test' spends "
+ "the split benchmark/eval_3dpw.py reports.")
+ p.add_argument("--val-3dpw-stride", type=int, default=5,
+ help="3DPW is 30 fps; neighbouring frames are near-duplicates.")
+ p.add_argument("--mix", default="sqrt",
+ help="Dataset mixture: sqrt | uniform | equal | "
+ "'sa1b=0.4,aic=0.35,coco=0.15,mpii=0.1'.")
+ p.add_argument("--seed", type=int, default=0)
+ p.add_argument("--max_images", type=int, default=None)
+
+ p.add_argument("--w_keypoints3d", type=float, default=None)
+ p.add_argument("--w_simcc", type=float, default=None)
+ p.add_argument("--weight_decay", type=float, default=None)
+ p.add_argument("--backbone", default=None)
+ p.add_argument("--ema-decay", type=float, default=None)
+
+ p.add_argument("--sync-bn", action="store_true")
+ p.add_argument("--no-resume", action="store_true")
+ p.add_argument("--no-pretrained", action="store_true",
+ help="Skip the timm download. Compute nodes have no internet, "
+ "so run --warm-cache once on a login node instead.")
+ p.add_argument("--log-every", type=int, default=100)
+ p.add_argument("--warm-cache", action="store_true",
+ help="Instantiate the backbone to populate $TORCH_HOME / "
+ "$HF_HOME, then exit. Run on a login or prepost node.")
+ p.add_argument("--dry-run", action="store_true",
+ help="Build config, dataset and sampler, print the mixture, "
+ "then exit without training.")
+ return p.parse_args()
+
+
+def build_cfg(args):
+ cfg = T.DistillConfig()
+ if args.data_root: cfg.data_root = args.data_root
+ if args.output_dir: cfg.log_dir = args.output_dir
+ if args.mhr_model_path: cfg.mhr_model_path = args.mhr_model_path
+ if args.kp_regressor_path: cfg.kp_regressor_path = args.kp_regressor_path
+ cfg.batch_size = args.batch_size
+ cfg.num_workers = args.num_workers
+ cfg.max_images = args.max_images
+ cfg.epochs = args.epochs
+ cfg.per_dataset_caps = {} # replaced by the weighted sampler
+ if args.backbone: cfg.backbone = args.backbone
+ if args.w_keypoints3d is not None: cfg.w_keypoints3d = args.w_keypoints3d
+ if args.w_simcc is not None: cfg.w_simcc = args.w_simcc
+ if args.weight_decay is not None: cfg.weight_decay = args.weight_decay
+ if args.ema_decay is not None: cfg.ema_decay = args.ema_decay
+ return cfg
+
+
+def main():
+ args = parse_args()
+
+ if args.warm_cache:
+ cfg = build_cfg(args)
+ import timm
+ m = timm.create_model(cfg.backbone, pretrained=True, num_classes=0)
+ print(f"cached {cfg.backbone}: {sum(p.numel() for p in m.parameters())/1e6:.2f} M "
+ f"params under TORCH_HOME={os.environ.get('TORCH_HOME')} "
+ f"HF_HOME={os.environ.get('HF_HOME')}")
+ return
+
+ rank, local_rank, world = setup_distributed()
+
+ # Distinct per rank so workers do not draw identical augmentation streams,
+ # but derived from --seed so the run stays reproducible.
+ seed = args.seed * 1000 + rank
+ random.seed(seed); np.random.seed(seed)
+ torch.manual_seed(seed); torch.cuda.manual_seed_all(seed)
+
+ cfg = build_cfg(args)
+ if args.lr_scaling == "linear":
+ cfg.lr = args.lr * world
+ elif args.lr_scaling == "sqrt":
+ cfg.lr = args.lr * math.sqrt(world)
+ else:
+ cfg.lr = args.lr
+
+ if not args.output_dir:
+ raise SystemExit("--output_dir (or $OUTPUT_DIR) is required")
+ if not Path(cfg.kp_regressor_path).exists():
+ raise SystemExit(
+ f"keypoint regressor missing: {cfg.kp_regressor_path}\n"
+ f"It ships with the repo; if absent, fit it once with\n"
+ f" python instanthmr_distill_train/train_distill_mhr_only.py "
+ f"--fit-regressor --data_root {cfg.data_root}")
+ if not Path(cfg.mhr_model_path).exists():
+ raise SystemExit(f"MHR TorchScript rig missing: {cfg.mhr_model_path}")
+
+ log("=" * 66)
+ log(f"host {socket.gethostname()} | rank {rank}/{world} (local {local_rank})")
+ log(f"data {cfg.data_root}")
+ log(f"out {cfg.log_dir}")
+ log(f"backbone {cfg.backbone} | epochs {args.epochs} | "
+ f"batch {cfg.batch_size}/gpu | lr {cfg.lr:.2e} ({args.lr_scaling} x{world})")
+ log(f"seed {args.seed} | mix {args.mix} | "
+ f"samples/epoch {args.samples_per_epoch:,}")
+ log("=" * 66)
+
+ if args.dry_run:
+ build_jz_loaders(cfg, args, rank, world)
+ log("dry run OK")
+ else:
+ train(args, cfg, rank, local_rank, world)
+
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/instanthmr_distill_train/val3dpw.py b/instanthmr_distill_train/val3dpw.py
new file mode 100644
index 0000000..63c80ec
--- /dev/null
+++ b/instanthmr_distill_train/val3dpw.py
@@ -0,0 +1,165 @@
+"""3DPW as a held-out validation set during training.
+
+The default validation set is a random slice of the training corpus, which
+measures agreement with the teacher on data drawn from the same distribution —
+useful as a training-health signal, useless as a measure of generalisation.
+3DPW is real MoCap ground truth from a different distribution, so it answers a
+different and more interesting question.
+
+Two rules that matter more than the code:
+
+**Use the ``validation`` split (12 sequences), not ``test``.** The 24 test
+sequences are what ``benchmark/eval_3dpw.py`` reports for publication. Select a
+checkpoint on them and that number stops being held-out — it becomes a number
+you optimised against. The validation split exists precisely so you don't have
+to spend the test split on model selection.
+
+**Never build ``3dpw_train`` into the training corpus.** Neighbouring frames of
+the same person in the same scene make frame-level holdout meaningless. This is
+already the situation locally, where ``sam3d_distill_mix`` contains crops from
+all 60 sequences (see ``benchmark/README.md``). On Jean Zay it is not: the built
+corpus is coco/mpii/aic/sa1b only, so 3DPW there is genuinely unseen — which is
+what makes this worth doing on the cluster and not on the laptop.
+
+Selection metric is **PA-MPJPE on J12**. J12 is limbs only, where MHR and SMPL
+mean the same anatomical points; J14 adds neck and head, which the two rigs
+place differently and which therefore carry a systematic penalty unrelated to
+model quality. Procrustes alignment additionally removes any residual
+coordinate-frame convention mismatch between the rig-local prediction and 3DPW's
+camera space, so the number is comparable across runs without further care.
+"""
+
+from __future__ import annotations
+
+import math
+import sys
+from pathlib import Path
+
+import cv2
+import numpy as np
+import torch
+from torch.utils.data import Dataset
+
+_BENCH = Path(__file__).resolve().parent.parent / "benchmark"
+if str(_BENCH) not in sys.path:
+ sys.path.insert(0, str(_BENCH))
+
+from benchlib import joints as J # noqa: E402
+from benchlib import metrics as M # noqa: E402
+from benchlib import threedpw as P # noqa: E402
+
+INPUT_SIZE = 224
+CROP_EXPAND = 1.2 # must match instanthmr/inference.py
+IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+MM = 1000.0
+
+
+def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int):
+ """Square 1.2x crop + ImageNet normalise + CLIFF conditioning.
+
+ A transcription of ``instanthmr.inference.InstantHMR._preprocess``. It is
+ duplicated rather than imported because that module pulls in onnxruntime,
+ and this one runs inside the training loop on every rank. If the crop
+ convention there ever changes, this must follow or the validation numbers
+ will silently stop matching ``benchmark/eval_3dpw.py``.
+ """
+ x1, y1, x2, y2 = np.asarray(bbox, dtype=np.float64)
+ bw, bh = x2 - x1, y2 - y1
+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
+
+ cliff = np.array([2.0 * (cx / w) - 1.0,
+ 2.0 * (cy / h) - 1.0,
+ max(bw, bh) / max(w, h)], dtype=np.float32)
+
+ sq = max(bw, bh) * CROP_EXPAND
+ half = sq / 2.0
+ ix1, iy1 = int(math.floor(cx - half)), int(math.floor(cy - half))
+ ix2, iy2 = int(math.ceil(cx - half + sq)), int(math.ceil(cy - half + sq))
+
+ pl, pt = max(0, -ix1), max(0, -iy1)
+ pr, pb = max(0, ix2 - w), max(0, iy2 - h)
+ patch = image_rgb[max(0, iy1):min(h, iy2), max(0, ix1):min(w, ix2)]
+ if pl or pt or pr or pb:
+ patch = cv2.copyMakeBorder(patch, pt, pb, pl, pr,
+ cv2.BORDER_CONSTANT, value=(0, 0, 0))
+ crop = cv2.resize(patch, (INPUT_SIZE, INPUT_SIZE), interpolation=cv2.INTER_LINEAR)
+ crop = (crop.astype(np.float32) / 255.0 - IMAGENET_MEAN) / IMAGENET_STD
+ return np.ascontiguousarray(crop.transpose(2, 0, 1)), cliff
+
+
+class ThreeDPWValSet(Dataset):
+ """(crop, cliff_cond, GT SMPL-24 joints) for every evaluatable person-frame.
+
+ Person boxes come from the projected GT joints, so no detector is in the
+ loop and the number isolates the pose model. ``stride`` subsamples frames:
+ 3DPW is 30 fps and neighbouring frames are near-duplicates, so stride 5
+ keeps the signal at a fifth of the cost.
+ """
+
+ def __init__(self, sequence_dir: str, image_root: str,
+ split: str = "validation", stride: int = 5,
+ bbox_scale: float = 1.2):
+ self.samples, self.gt = P.build_samples(
+ sequence_dir, image_root, split=split,
+ stride=stride, bbox_scale=bbox_scale)
+ self.gt = np.asarray(self.gt, dtype=np.float32)
+
+ def __len__(self) -> int:
+ return len(self.samples)
+
+ def __getitem__(self, i: int):
+ s = self.samples[i]
+ bgr = cv2.imread(s["image_path"], cv2.IMREAD_COLOR)
+ if bgr is None:
+ # A missing frame must not kill a 20-hour job; return a zero crop
+ # and flag it so the metric can drop it.
+ return {"image": torch.zeros(3, INPUT_SIZE, INPUT_SIZE),
+ "cliff_cond": torch.zeros(3),
+ "gt": torch.from_numpy(self.gt[i]),
+ "ok": torch.tensor(0.0)}
+ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+ h, w = rgb.shape[:2]
+ crop, cliff = _preprocess(rgb, s["bbox"], h, w)
+ return {"image": torch.from_numpy(crop),
+ "cliff_cond": torch.from_numpy(cliff),
+ "gt": torch.from_numpy(self.gt[i]),
+ "ok": torch.tensor(1.0)}
+
+
+def evaluate(model, loader, mhr_module, device, use_amp: bool = True) -> dict:
+ """PA-MPJPE and MPJPE in mm on J12 and J14, from the MHR forward pass.
+
+ Predictions come from ``mhr_params`` through the same FK + (70, 127)
+ regressor the training loss uses, so this measures exactly the geometry the
+ model is being trained to produce.
+ """
+ model.eval()
+ preds, gts = [], []
+ for b in loader:
+ if float(b["ok"].sum()) == 0:
+ continue
+ img = b["image"].to(device, non_blocking=True)
+ cc = b["cliff_cond"].to(device, non_blocking=True)
+ with torch.amp.autocast(device_type="cuda", dtype=torch.float16, enabled=use_amp):
+ out = model(img, cc)
+ kp = mhr_module.get_native_keypoints(out["mhr_params"].float(),
+ out["shape_params"].float())
+ keep = b["ok"].bool()
+ preds.append(kp[keep.to(kp.device)].float().cpu().numpy())
+ gts.append(b["gt"][keep].numpy())
+
+ if not preds:
+ return {"n": 0.0}
+ pred = np.concatenate(preds) * MM # (N, 70, 3) rig-local, mm
+ gt = np.concatenate(gts) * MM # (N, 24, 3) camera space, mm
+
+ res: dict[str, float] = {"n": float(len(pred))}
+ for name in ("J12", "J14"):
+ spec = J.JOINT_SETS[name]
+ p, g = pred[:, spec["mhr"], :], gt[:, spec["smpl"], :]
+ rp, rg = J.pelvis(p), J.pelvis(g)
+ res[f"{name}_MPJPE"] = float(M.mpjpe(p, g, root=(rp, rg)).mean())
+ res[f"{name}_PA_MPJPE"] = float(M.pa_mpjpe(p, g).mean())
+ return res