Python
fix: drop the orphaned timm classifier head that aborts DDP on step 2, with tools/ddp_smoke.py to catch it before submitting
mohamdev57ad584efc96
diff --git a/CLAUDE.md b/CLAUDE.mdindex 2788db7..00bfc22 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -167,6 +167,26 @@ steps. Use `--anomaly-safe-fallback`, which steps on the non-FK terms insteaddivergence here by lowering the LR alone: `OneCycleLR(pct_start=0.1)` peaks at10% of the run, so a lower peak only moves *when* it dies.+**A single-process smoke test cannot clear a training change for the cluster.**+`52_train_ddp.slurm` wraps the student in `DistributedDataParallel` with the+default `find_unused_parameters=False`, and that Reducer exists **nowhere** in a+laptop run -- not in `train_distill_jz.py` on one GPU, not in `--self-test`, not+in `--overfit-test`. A parameter that receives no gradient therefore passes+every local check and then aborts the job on the SECOND optimiser step with+`Expected to have finished reduction in the prior iteration ... Parameter+indices which did not receive grad for rank N: <i>`, where `<i>` indexes+`list(model.parameters())`. On 2026-09-11 that burned all eight queued+generation-7 `hgnetv2_b4` links while the `repvit_m2_3` arms ran nine hours+beside them: `forward_features()` stops before the timm classifier head, and+`num_classes=0` empties that head for repvit but leaves hgnetv2_b4 a 2048x2048+`last_conv` -- 4,194,304 orphaned parameters, index 241. `InstantHMRStudent`+now replaces any head that still has parameters with `nn.Identity()` (a no-op+for a parameterless one, so every existing state_dict is byte-identical --+verified, 1542 keys, same sha256). **Run `python tools/ddp_smoke.py --backbone+<name> --w-verts <w>` before submitting any architecture change.** It builds a+real Reducer at world size 1 and takes under a minute; world 1 is enough,+because the unused-parameter check is world-size independent.+**The pair index is a host-RAM bomb, and it kills jobs with no traceback.**Four generation-6 jobs were OOM-killed at 74-76 GiB per node between epochs 14and 33 on 2026-09-08. Not CUDA -- `sacct` says `OUT_OF_ME+`, exit `0:125`,diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.pyindex 39f13f9..73b035d 100644--- a/instanthmr_distill_train/train_distill_mhr_only.py+++ b/instanthmr_distill_train/train_distill_mhr_only.py@@ -890,6 +890,18 @@ class InstantHMRStudent(nn.Module):self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)embed_dim = self.backbone.num_features+ # forward_features() stops before the classifier head, so any parameter+ # left in that head never receives a gradient and DDP (which runs with+ # find_unused_parameters=False) aborts on the second step with+ # "Expected to have finished reduction in the prior iteration".+ # num_classes=0 empties the head for repvit_m2_3 but NOT for hgnetv2_b4,+ # which keeps a 2048x2048 last_conv -- 4,194,304 parameters, DDP index+ # 241. That killed all four generation-7 hgnet jobs at step 2.+ # No-op for a head that is already parameterless, so the parameter list,+ # its order and the state_dict of every existing run are untouched.+ head = getattr(self.backbone, "head", None)+ if head is not None and any(p.numel() for p in head.parameters()):+ self.backbone.head = nn.Identity()self.feat_proj = nn.Linear(embed_dim, cfg.d_model)self.grid_size = cfg.image_size // 32diff --git a/tools/ddp_smoke.py b/tools/ddp_smoke.pynew file mode 100644index 0000000..42270d0--- /dev/null+++ b/tools/ddp_smoke.py@@ -0,0 +1,130 @@+#!/usr/bin/env python3+"""Run the student through a real DDP Reducer before submitting to the cluster.++ python tools/ddp_smoke.py --backbone hgnetv2_b4 --w-verts 0.35+ python tools/ddp_smoke.py --backbone repvit_m2_3 # the control++WHY THIS EXISTS. A plain single-process smoke run -- `train_distill_jz.py` on a+laptop, `--self-test`, `--overfit-test` -- builds no DDP Reducer at all, so it+cannot see the one class of failure that only exists under `srun`: a parameter+that never receives a gradient. `52_train_ddp.slurm` runs+`DistributedDataParallel` with the default `find_unused_parameters=False`, which+aborts on the SECOND optimiser step with++ Expected to have finished reduction in the prior iteration before starting+ a new one ... Parameter indices which did not receive grad for rank N: 241++and nothing before that step warns you. On 2026-09-11 that cost all four+generation-7 `hgnetv2_b4` jobs: `forward_features()` stops before the timm+classifier head, and `num_classes=0` empties that head for `repvit_m2_3` but+leaves `hgnetv2_b4` a 2048x2048 `last_conv` -- 4,194,304 parameters with no+gradient. The repvit arms ran for nine hours while every hgnet arm died at+step 2.++World size 1 is enough: the Reducer and its unused-parameter check are built and+run identically at any world size, so this reproduces the cluster error exactly,+on one GPU, in under a minute. It does NOT test NCCL, multi-node bring-up or+gradient bucketing across ranks -- only that every parameter participates.++Exit status is 0 when every parameter received a gradient, 1 otherwise, so it+can gate a submission.+"""++from __future__ import annotations++import argparse+import os+import sys+from pathlib import Path++import numpy as np+import torch+import torch.distributed as dist++sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "instanthmr_distill_train"))+import train_distill_mhr_only as T # noqa: E402+++def main() -> int:+ p = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument("--backbone", default=None, help="timm name; default is the config's")+ p.add_argument("--data_root", default="data")+ p.add_argument("--batch_size", type=int, default=8)+ p.add_argument("--losses", choices=("legacy", "rebalanced"), default="rebalanced")+ p.add_argument("--w-verts", dest="w_verts", type=float, default=0.0)+ p.add_argument("--bound-scales", dest="bound_scales", action="store_true", default=True)+ p.add_argument("--cliff-focal", dest="cliff_focal", action="store_true", default=True)+ p.add_argument("--crop-centre-fix", dest="crop_centre_fix", action="store_true", default=True)+ p.add_argument("--pretrained", action="store_true",+ help="Load the timm weights too. Off by default: the head "+ "geometry this checks does not depend on them.")+ args = p.parse_args()++ os.environ.setdefault("MASTER_ADDR", "127.0.0.1")+ os.environ.setdefault("MASTER_PORT", "29517")+ # gloo, not nccl: this needs no GPU-to-GPU transport and must run on a+ # laptop with one card (or none).+ dist.init_process_group(backend="gloo", rank=0, world_size=1)++ cfg = T.DistillConfig()+ T.cfg = cfg+ if args.losses == "rebalanced":+ T.apply_rebalanced_losses(cfg)+ cfg.data_root = args.data_root+ cfg.bound_scales = args.bound_scales+ cfg.cliff_focal = args.cliff_focal+ cfg.crop_centre_fix = args.crop_centre_fix+ cfg.w_verts = args.w_verts+ if args.backbone:+ cfg.backbone = args.backbone++ ds = T.SAM3DStudentDataset(cfg.data_root, augment=True, max_images=args.batch_size,+ per_dataset_caps={}, cliff_focal=cfg.cliff_focal,+ crop_centre_fix=cfg.crop_centre_fix)+ loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, num_workers=0)+ batch = {k: (v.to(T.device) if torch.is_tensor(v) else v)+ for k, v in next(iter(loader)).items()}++ mhr = T.MHRForwardPass(cfg.mhr_model_path, T.device,+ kp_regressor=np.load(cfg.kp_regressor_path),+ n_verts=cfg.n_verts if cfg.w_verts > 0 else 0)+ T.mhr_module = mhr++ model = T.InstantHMRStudent(cfg, pretrained=args.pretrained).to(T.device)+ n = sum(q.numel() for q in model.parameters())+ print(f"{cfg.backbone}: {n/1e6:.2f} M parameters, "+ f"w_verts={cfg.w_verts}, batch {args.batch_size}")++ ddp = torch.nn.parallel.DistributedDataParallel(model) # find_unused_parameters=False+ crit = T.DistillationLoss(cfg, mhr)+ crit.warmup_steps = 0 # full-strength FK terms++ # TWO iterations. The Reducer only reports unused parameters when a new+ # forward starts before the previous reduction completed, so one step passes+ # even on a broken model -- which is how this was missed the first time.+ try:+ for _ in range(2):+ out = ddp(batch["image"], batch["cliff_cond"].to(T.device))+ crit(out, batch)["total_loss"].backward()+ except RuntimeError as e:+ print(f"\n❌ DDP rejected {cfg.backbone}:\n{e}")+ print("\nThe named index is a position in list(model.parameters()); "+ "list(model.named_parameters())[i] gives the parameter.")+ return 1+ finally:+ dist.destroy_process_group()++ missing = [k for k, q in model.named_parameters() if q.grad is None]+ if missing:+ print(f"\n❌ {len(missing)} parameter(s) never received a gradient:")+ for k in missing:+ print(f" {k}")+ return 1++ print(f"✅ every parameter received a gradient over 2 DDP steps")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tools/pth_to_onnx.py b/tools/pth_to_onnx.pyindex bf540ee..89b174d 100644--- a/tools/pth_to_onnx.py+++ b/tools/pth_to_onnx.py@@ -35,7 +35,7 @@ Usage# or point at a specific checkpoint / outputpython ./tools/pth_to_onnx.py \- --ckpt ./instanthmr_distill_train/runs/g6bv_s/g6bv_s/best_student_model_v3.pth \+ --ckpt ./instanthmr_distill_train/runs/g7h_s0/g7h_s0/best_student_model_v3.pth \--output ./models/instanthmr.onnx"""import argparse