Skip to content
Python

fix: drop the orphaned timm classifier head that aborts DDP on step 2, with tools/ddp_smoke.py to catch it before submitting

mohamdev57ad584efc96
diff --git a/CLAUDE.md b/CLAUDE.md
index 2788db7..00bfc22 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -167,6 +167,26 @@ steps. Use `--anomaly-safe-fallback`, which steps on the non-FK terms instead
divergence here by lowering the LR alone: `OneCycleLR(pct_start=0.1)` peaks at
10% of the run, so a lower peak only moves *when* it dies.
+**A single-process smoke test cannot clear a training change for the cluster.**
+`52_train_ddp.slurm` wraps the student in `DistributedDataParallel` with the
+default `find_unused_parameters=False`, and that Reducer exists **nowhere** in a
+laptop run -- not in `train_distill_jz.py` on one GPU, not in `--self-test`, not
+in `--overfit-test`. A parameter that receives no gradient therefore passes
+every local check and then aborts the job on the SECOND optimiser step with
+`Expected to have finished reduction in the prior iteration ... Parameter
+indices which did not receive grad for rank N: <i>`, where `<i>` indexes
+`list(model.parameters())`. On 2026-09-11 that burned all eight queued
+generation-7 `hgnetv2_b4` links while the `repvit_m2_3` arms ran nine hours
+beside them: `forward_features()` stops before the timm classifier head, and
+`num_classes=0` empties that head for repvit but leaves hgnetv2_b4 a 2048x2048
+`last_conv` -- 4,194,304 orphaned parameters, index 241. `InstantHMRStudent`
+now replaces any head that still has parameters with `nn.Identity()` (a no-op
+for a parameterless one, so every existing state_dict is byte-identical --
+verified, 1542 keys, same sha256). **Run `python tools/ddp_smoke.py --backbone
+<name> --w-verts <w>` before submitting any architecture change.** It builds a
+real Reducer at world size 1 and takes under a minute; world 1 is enough,
+because the unused-parameter check is world-size independent.
+
**The pair index is a host-RAM bomb, and it kills jobs with no traceback.**
Four generation-6 jobs were OOM-killed at 74-76 GiB per node between epochs 14
and 33 on 2026-09-08. Not CUDA -- `sacct` says `OUT_OF_ME+`, exit `0:125`,
diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.py
index 39f13f9..73b035d 100644
--- a/instanthmr_distill_train/train_distill_mhr_only.py
+++ b/instanthmr_distill_train/train_distill_mhr_only.py
@@ -890,6 +890,18 @@ class InstantHMRStudent(nn.Module):
self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)
embed_dim = self.backbone.num_features
+ # forward_features() stops before the classifier head, so any parameter
+ # left in that head never receives a gradient and DDP (which runs with
+ # find_unused_parameters=False) aborts on the second step with
+ # "Expected to have finished reduction in the prior iteration".
+ # num_classes=0 empties the head for repvit_m2_3 but NOT for hgnetv2_b4,
+ # which keeps a 2048x2048 last_conv -- 4,194,304 parameters, DDP index
+ # 241. That killed all four generation-7 hgnet jobs at step 2.
+ # No-op for a head that is already parameterless, so the parameter list,
+ # its order and the state_dict of every existing run are untouched.
+ head = getattr(self.backbone, "head", None)
+ if head is not None and any(p.numel() for p in head.parameters()):
+ self.backbone.head = nn.Identity()
self.feat_proj = nn.Linear(embed_dim, cfg.d_model)
self.grid_size = cfg.image_size // 32
diff --git a/tools/ddp_smoke.py b/tools/ddp_smoke.py
new file mode 100644
index 0000000..42270d0
--- /dev/null
+++ b/tools/ddp_smoke.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Run the student through a real DDP Reducer before submitting to the cluster.
+
+ python tools/ddp_smoke.py --backbone hgnetv2_b4 --w-verts 0.35
+ python tools/ddp_smoke.py --backbone repvit_m2_3 # the control
+
+WHY THIS EXISTS. A plain single-process smoke run -- `train_distill_jz.py` on a
+laptop, `--self-test`, `--overfit-test` -- builds no DDP Reducer at all, so it
+cannot see the one class of failure that only exists under `srun`: a parameter
+that never receives a gradient. `52_train_ddp.slurm` runs
+`DistributedDataParallel` with the default `find_unused_parameters=False`, which
+aborts on the SECOND optimiser step with
+
+ Expected to have finished reduction in the prior iteration before starting
+ a new one ... Parameter indices which did not receive grad for rank N: 241
+
+and nothing before that step warns you. On 2026-09-11 that cost all four
+generation-7 `hgnetv2_b4` jobs: `forward_features()` stops before the timm
+classifier head, and `num_classes=0` empties that head for `repvit_m2_3` but
+leaves `hgnetv2_b4` a 2048x2048 `last_conv` -- 4,194,304 parameters with no
+gradient. The repvit arms ran for nine hours while every hgnet arm died at
+step 2.
+
+World size 1 is enough: the Reducer and its unused-parameter check are built and
+run identically at any world size, so this reproduces the cluster error exactly,
+on one GPU, in under a minute. It does NOT test NCCL, multi-node bring-up or
+gradient bucketing across ranks -- only that every parameter participates.
+
+Exit status is 0 when every parameter received a gradient, 1 otherwise, so it
+can gate a submission.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.distributed as dist
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "instanthmr_distill_train"))
+import train_distill_mhr_only as T # noqa: E402
+
+
+def main() -> int:
+ p = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--backbone", default=None, help="timm name; default is the config's")
+ p.add_argument("--data_root", default="data")
+ p.add_argument("--batch_size", type=int, default=8)
+ p.add_argument("--losses", choices=("legacy", "rebalanced"), default="rebalanced")
+ p.add_argument("--w-verts", dest="w_verts", type=float, default=0.0)
+ p.add_argument("--bound-scales", dest="bound_scales", action="store_true", default=True)
+ p.add_argument("--cliff-focal", dest="cliff_focal", action="store_true", default=True)
+ p.add_argument("--crop-centre-fix", dest="crop_centre_fix", action="store_true", default=True)
+ p.add_argument("--pretrained", action="store_true",
+ help="Load the timm weights too. Off by default: the head "
+ "geometry this checks does not depend on them.")
+ args = p.parse_args()
+
+ os.environ.setdefault("MASTER_ADDR", "127.0.0.1")
+ os.environ.setdefault("MASTER_PORT", "29517")
+ # gloo, not nccl: this needs no GPU-to-GPU transport and must run on a
+ # laptop with one card (or none).
+ dist.init_process_group(backend="gloo", rank=0, world_size=1)
+
+ cfg = T.DistillConfig()
+ T.cfg = cfg
+ if args.losses == "rebalanced":
+ T.apply_rebalanced_losses(cfg)
+ cfg.data_root = args.data_root
+ cfg.bound_scales = args.bound_scales
+ cfg.cliff_focal = args.cliff_focal
+ cfg.crop_centre_fix = args.crop_centre_fix
+ cfg.w_verts = args.w_verts
+ if args.backbone:
+ cfg.backbone = args.backbone
+
+ ds = T.SAM3DStudentDataset(cfg.data_root, augment=True, max_images=args.batch_size,
+ per_dataset_caps={}, cliff_focal=cfg.cliff_focal,
+ crop_centre_fix=cfg.crop_centre_fix)
+ loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, num_workers=0)
+ batch = {k: (v.to(T.device) if torch.is_tensor(v) else v)
+ for k, v in next(iter(loader)).items()}
+
+ mhr = T.MHRForwardPass(cfg.mhr_model_path, T.device,
+ kp_regressor=np.load(cfg.kp_regressor_path),
+ n_verts=cfg.n_verts if cfg.w_verts > 0 else 0)
+ T.mhr_module = mhr
+
+ model = T.InstantHMRStudent(cfg, pretrained=args.pretrained).to(T.device)
+ n = sum(q.numel() for q in model.parameters())
+ print(f"{cfg.backbone}: {n/1e6:.2f} M parameters, "
+ f"w_verts={cfg.w_verts}, batch {args.batch_size}")
+
+ ddp = torch.nn.parallel.DistributedDataParallel(model) # find_unused_parameters=False
+ crit = T.DistillationLoss(cfg, mhr)
+ crit.warmup_steps = 0 # full-strength FK terms
+
+ # TWO iterations. The Reducer only reports unused parameters when a new
+ # forward starts before the previous reduction completed, so one step passes
+ # even on a broken model -- which is how this was missed the first time.
+ try:
+ for _ in range(2):
+ out = ddp(batch["image"], batch["cliff_cond"].to(T.device))
+ crit(out, batch)["total_loss"].backward()
+ except RuntimeError as e:
+ print(f"\n❌ DDP rejected {cfg.backbone}:\n{e}")
+ print("\nThe named index is a position in list(model.parameters()); "
+ "list(model.named_parameters())[i] gives the parameter.")
+ return 1
+ finally:
+ dist.destroy_process_group()
+
+ missing = [k for k, q in model.named_parameters() if q.grad is None]
+ if missing:
+ print(f"\n❌ {len(missing)} parameter(s) never received a gradient:")
+ for k in missing:
+ print(f" {k}")
+ return 1
+
+ print(f"✅ every parameter received a gradient over 2 DDP steps")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/pth_to_onnx.py b/tools/pth_to_onnx.py
index bf540ee..89b174d 100644
--- a/tools/pth_to_onnx.py
+++ b/tools/pth_to_onnx.py
@@ -35,7 +35,7 @@ Usage
# or point at a specific checkpoint / output
python ./tools/pth_to_onnx.py \
- --ckpt ./instanthmr_distill_train/runs/g6bv_s/g6bv_s/best_student_model_v3.pth \
+ --ckpt ./instanthmr_distill_train/runs/g7h_s0/g7h_s0/best_student_model_v3.pth \
--output ./models/instanthmr.onnx
"""
import argparse