Python
feat: v2 training preset (truncation/occlusion/JPEG aug + absolute-pose supervision)
mohamdev8468be118c31
diff --git a/datasets_pipeline/verify_split.py b/datasets_pipeline/verify_split.pyindex 2eba390..ad18caa 100755--- a/datasets_pipeline/verify_split.py+++ b/datasets_pipeline/verify_split.py@@ -26,11 +26,13 @@ second encoder.from __future__ import annotationsimport argparse+import osimport randomimport sysfrom pathlib import Pathimport numpy as np+from concurrent.futures import ThreadPoolExecutorHERE = Path(__file__).resolve().parentREPO = HERE.parent@@ -46,7 +48,30 @@ LEFT_HAND_KPS = list(range(42, 63))RIGHT_WRIST, LEFT_WRIST = 41, 62-def structure(d: Path) -> None:+def count_files(root: Path) -> int:+ """Recursive file count that never follows a symlink.++ original_images/ can hold >1M symlinks into a read-only mirror on another+ filesystem; Path.is_file() resolves each one, turning a count into a+ million cross-filesystem stats. scandir's d_type answers dir-or-not+ without any stat at all.+ """+ n, stack = 0, [root]+ while stack:+ try:+ with os.scandir(stack.pop()) as it:+ for e in it:+ if e.is_dir(follow_symlinks=False):+ stack.append(Path(e.path))+ else:+ n += 1+ except OSError:+ pass+ return n+++def structure(d: Path, workers: int = 16, hand_scan_limit: int = 300_000,+ seed: int = 0) -> None:anns = sorted((d / "annotations").glob("*.npz"))body = {p.stem for p in (d / "body_crops").iterdir()} if (d / "body_crops").exists() else set()hands = {p.stem for p in (d / "hands_crops").iterdir()} if (d / "hands_crops").exists() else set()@@ -54,27 +79,51 @@ def structure(d: Path) -> None:missing_body = sorted(stems - body)[:5]orphan_body = sorted(body - stems)[:5]- want_hands, have_hands, missing_hands = 0, 0, []- for p in anns:- a = np.load(p, allow_pickle=False)++ # Reading every npz is the expensive half of this check: single-threaded it+ # runs at ~100/s on Lustre, so a 3.4M-crop split would take 9 h. Sample+ # above hand_scan_limit and scale, and thread the reads either way.+ scan = anns+ scaled = False+ if len(anns) > hand_scan_limit:+ scan = random.Random(seed).sample(anns, hand_scan_limit)+ scaled = True++ def probe(p: Path):+ try:+ a = np.load(p, allow_pickle=False)+ except Exception: # noqa: BLE001+ return ()+ out = []for side in ("l", "r"):key = f"hand_valid_{side}"- if key not in a.files or not bool(a[key]):- continue- want_hands += 1- if f"{p.stem}_{side}" in hands:- have_hands += 1- elif len(missing_hands) < 5:- missing_hands.append(f"{p.stem}_{side}")+ if key in a.files and bool(a[key]):+ out.append(f"{p.stem}_{side}")+ return tuple(out)++ want_hands, have_hands, missing_hands = 0, 0, []+ with ThreadPoolExecutor(workers) as ex:+ for names in ex.map(probe, scan):+ for nm in names:+ want_hands += 1+ if nm in hands:+ have_hands += 1+ elif len(missing_hands) < 5:+ missing_hands.append(nm)print(f" annotations : {len(anns):,}")print(f" body_crops : {len(body):,}"f" (missing {len(stems - body):,}, orphan {len(body - stems):,})")- print(f" hands_crops : {len(hands):,}"- f" (declared {want_hands:,}, present {have_hands:,})")+ if scaled:+ f = len(anns) / len(scan)+ print(f" hands_crops : {len(hands):,}"+ f" (declared ~{int(want_hands * f):,}, present ~{int(have_hands * f):,}"+ f" — estimated from {len(scan):,} of {len(anns):,} npz)")+ else:+ print(f" hands_crops : {len(hands):,}"+ f" (declared {want_hands:,}, present {have_hands:,})")if (d / "original_images").exists():- n = sum(1 for _ in (d / "original_images").rglob("*") if _.is_file())- print(f" original_images : {n:,} files")+ print(f" original_images : {count_files(d / 'original_images'):,} files")if missing_body:print(f" !! body crop missing for: {missing_body}")if orphan_body:@@ -149,6 +198,10 @@ def main() -> None:formatter_class=argparse.RawDescriptionHelpFormatter)p.add_argument("--dir", type=Path, required=True, help="data/sam3d_gt_<dataset>")p.add_argument("--sample", type=int, default=500)+ p.add_argument("--workers", type=int, default=16,+ help="threads for the npz hand-validity scan")+ p.add_argument("--hand-scan-limit", type=int, default=300_000,+ help="above this many npz, estimate hand counts from a sample instead of reading every file")p.add_argument("--seed", type=int, default=0)p.add_argument("--hand-signal", action="store_true")p.add_argument("--mhr", type=Path,@@ -162,7 +215,8 @@ def main() -> None:print(f"\n== {args.dir} ==")print("structure")- structure(args.dir)+ structure(args.dir, workers=args.workers,+ hand_scan_limit=args.hand_scan_limit, seed=args.seed)print("geometry")geometry(args.dir, args.sample, args.seed)if args.hand_signal:diff --git a/demo.py b/demo.pyindex 44ded50..5e98a14 100755--- a/demo.py+++ b/demo.py@@ -303,7 +303,8 @@ def build_all(args: argparse.Namespace):The rig comes first because an MHR-only graph cannot produce 3D keypointswithout it — the pipeline takes it as a constructor argument rather than- the visualizer alone.+ the visualizer alone. It is returned as well so the temporal filter can+ re-derive keypoints from filtered pose parameters."""model_path = resolve_model_path(args)mhr = build_mhr(args, required=_model_needs_mhr(model_path))@@ -315,7 +316,7 @@ def build_all(args: argparse.Namespace):mhr_renderer=mhr,mesh_alpha=args.mesh_alpha,)- return pipeline, viz+ return pipeline, viz, mhrdef _print_timings(@@ -343,21 +344,30 @@ def _print_timings(# ----------------------------------------------------------------------------def build_smoother(args: argparse.Namespace, live: bool = False):+def build_smoother(args: argparse.Namespace, live: bool = False, mhr=None):"""Construct the temporal filter, or ``None`` when --smoothing-filter is off.``live`` forces the adaptive filter: a camera cannot buffer future frames,and a trailing average would show visible phase lag on fast motion.++ Handing it the MHR backend lets it re-derive the 3D keypoints from the+ filtered pose parameters, so they stay welded to the mesh instead of+ drifting off it during fast motion."""if not args.smoothing_filter:return Nonemode = "1euro" if live else args.smoothing_mode+ kp_fn = mhr.keypoints if getattr(mhr, "has_keypoints", False) else Nonesm = TemporalSmoother(window=args.smoothing_window,mode=mode,beta_scale=args.smoothing_beta,mincutoff_scale=args.smoothing_mincutoff,+ keypoint_fn=kp_fn,)+ if kp_fn is None:+ print(" note: no MHR keypoint backend — keypoints are filtered on their "+ "own, so they may drift off the mesh during fast motion.")if sm.mode == "1euro":print(f"temporal smoothing: 1euro, beta x{sm.beta_scale:g}, "f"mincutoff x{sm.mincutoff_scale:g}, no latency")@@ -377,7 +387,7 @@ def run_image(args: argparse.Namespace) -> None:sys.exit(f"[error] cv2.imread failed for {image_path}")rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)- pipeline, viz = build_all(args)+ pipeline, viz, _ = build_all(args)result = pipeline.predict(rgb)@@ -408,8 +418,8 @@ def run_video(args: argparse.Namespace) -> None:total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))print(f"video: {video_path.name}, {total} frames @ {fps:.1f} fps")- pipeline, viz = build_all(args)- smoother = build_smoother(args)+ pipeline, viz, mhr = build_all(args)+ smoother = build_smoother(args, mhr=mhr)frame_idx = 0sent = 0@@ -511,8 +521,8 @@ def run_camera(args: argparse.Namespace) -> None:fps = cap.get(cv2.CAP_PROP_FPS) or 30.0print(f"camera {args.camera}: {width}x{height} @ {fps:.1f} fps — Ctrl+C to stop")- pipeline, viz = build_all(args)- smoother = build_smoother(args, live=True)+ pipeline, viz, mhr = build_all(args)+ smoother = build_smoother(args, live=True, mhr=mhr)frame_idx = 0det_times: list[float] = []diff --git a/docs/training.md b/docs/training.mdindex 3aeeaf1..4642e7c 100644--- a/docs/training.md+++ b/docs/training.md@@ -2,7 +2,15 @@The InstantHMR training notebook is[`notebooks/distill_transformer_decoder.ipynb`](../notebooks/distill_transformer_decoder.ipynb).-It is the source of truth — this page is a tour, not a re-implementation.+It is the source of truth for the notebook workflow — this page is a tour, not a+re-implementation.++**For cluster training, the notebook is not the source of truth.** That is+`instanthmr_distill_train/train_distill_mhr_only.py` (dataset, losses, model)+driven by `train_distill_jz.py` (DDP, mixture sampling, resume, 3DPW+validation), and the two have diverged from the notebook. See+[`instanthmr_distill_train/README.md`](../instanthmr_distill_train/README.md)+for which script to run, what `--preset v2` changes, and the pair-index cache.## Setupdiff --git a/instanthmr/smoothing.py b/instanthmr/smoothing.pyindex a49d8ee..3ac56eb 100644--- a/instanthmr/smoothing.py+++ b/instanthmr/smoothing.py@@ -54,7 +54,9 @@ from .inference import HMRPrediction#: linear mean is the right operation, except ``mhr_params`` — see the module#: docstring. ``joints_3d_cam`` is deliberately absent: it is recomputed from#: the smoothed ``joints_3d_local`` and ``cam_trans`` so the three stay-#: consistent with each other.+#: consistent with each other. ``joints_3d_local`` is filtered only when no+#: ``keypoint_fn`` is available; with one, it is re-derived from the filtered+#: ``mhr_params`` so the keypoints and the mesh cannot disagree.SMOOTHED_FIELDS = ("joints_3d_local","cam_trans",@@ -216,6 +218,10 @@ class TemporalSmoother:lower it if the output looks noisy while you hold still.mincutoff_scale: multiplies every per-field ``mincutoff``. Governs howhard the signal is smoothed at rest.+ keypoint_fn: optional ``f(mhr_params, shape_params) -> (70, 3)``. When+ given, ``joints_3d_local`` is *re-derived* from the filtered pose+ parameters instead of being filtered on its own. This is what keeps+ the keypoints welded to the mesh — see :meth:`_finalize`.iou_threshold: minimum IoU to associate a detection with a track.max_age: frames a track survives without a detection.@@ -236,6 +242,7 @@ class TemporalSmoother:mode: str = "centered",beta_scale: float = 1.0,mincutoff_scale: float = 1.0,+ keypoint_fn=None,iou_threshold: float = 0.3,max_age: int = 5,):@@ -251,6 +258,7 @@ class TemporalSmoother:self.lag = (window - 1) // 2 if mode == "centered" else 0self.beta_scale = float(beta_scale)self.mincutoff_scale = float(mincutoff_scale)+ self.keypoint_fn = keypoint_fnself._tracker = _Tracker(iou_threshold=iou_threshold, max_age=max_age)self._buf: deque[dict[int, HMRPrediction]] = deque(maxlen=window)@@ -340,10 +348,7 @@ class TemporalSmoother:name: bank[name](getattr(pred, name), t).astype(np.float32)for name in SMOOTHED_FIELDS}- fields["joints_3d_cam"] = (- fields["joints_3d_local"] + fields["cam_trans"][None, :]- ).astype(np.float32)- smoothed = replace(pred, **fields)+ smoothed = self._finalize(pred, fields)out.append(smoothed)emitted[track_id] = smoothed@@ -358,6 +363,31 @@ class TemporalSmoother:return out# ------------------------------------------------------------------+ def _finalize(self, anchor: HMRPrediction, fields: dict) -> HMRPrediction:+ """Assemble one smoothed prediction, keeping its parts self-consistent.++ Filtering ``mhr_params`` and ``joints_3d_local`` independently makes the+ green keypoints drift off the skin during fast motion, because the mesh+ is decoded from the former while the keypoints come from the latter and+ forward kinematics is *nonlinear* — ``filter(FK(p)) != FK(filter(p))``.+ Adaptive filtering makes it worse still, since each field then gets its+ own cutoff driven by its own speed in its own units.++ So when a keypoint backend is available we do not filter the keypoints+ at all: we filter the pose parameters and re-derive the keypoints from+ them, which is exactly how ``inference.py`` produces them for an+ MHR-only graph. The two agree by construction, in every mode.+ """+ if self.keypoint_fn is not None:+ fields["joints_3d_local"] = np.asarray(+ self.keypoint_fn(fields["mhr_params"], fields["shape_params"]),+ dtype=np.float32,+ ).reshape(anchor.joints_3d_local.shape)+ fields["joints_3d_cam"] = (+ fields["joints_3d_local"] + fields["cam_trans"][None, :]+ ).astype(np.float32)+ return replace(anchor, **fields)+def _emit(self, centre: int) -> list[HMRPrediction]:"""Average every track present in frame ``centre`` over the buffer."""out: list[HMRPrediction] = []@@ -372,10 +402,7 @@ class TemporalSmoother:).astype(np.float32)for name in SMOOTHED_FIELDS}- fields["joints_3d_cam"] = (- fields["joints_3d_local"] + fields["cam_trans"][None, :]- ).astype(np.float32)- smoothed = replace(anchor, **fields)+ smoothed = self._finalize(anchor, fields)out.append(smoothed)emitted[track_id] = smootheddiff --git a/instanthmr_distill_train/README.md b/instanthmr_distill_train/README.mdindex b9680b3..3537c41 100644--- a/instanthmr_distill_train/README.md+++ b/instanthmr_distill_train/README.md@@ -1,10 +1,81 @@-# InstantHMR Distillation — Standalone Cloud Training--Standalone version of `notebooks/distill_transformer_decoder.ipynb` for running-on the `pfcalcul` SLURM cluster. It bundles notebook cells **0, 1, 3, 7, 8, 10,-11 and 15** into a single script: setup, config, dataset, student architecture,-distillation loss, HMR metrics, the EMA + early-stopping training loop, and ONNX-export/quantization.+# InstantHMR Distillation — training scripts++## Which script++Four files, and the difference matters.++| file | what it is |+|---|---|+| `train_distill_mhr_only.py` | **the trainer.** Dataset, augmentation, student architecture, MHR forward pass, all losses, metrics, the single-GPU loop. Everything of substance lives here. |+| `train_distill_jz.py` | **the multi-GPU entry point.** `import train_distill_mhr_only as T` plus what a cluster needs: DDP, mixture sampling, resumable checkpointing, held-out 3DPW validation, the pair-index cache. Run this one on a cluster. |+| `val3dpw.py` | 3DPW held-out validation used by `train_distill_jz.py`. Cross-validated against `benchmark/eval_3dpw.py` — they agree exactly. |+| `train_distill.py` | **legacy.** The original notebook port. See the warning below before using it. |++They are one trainer in two layers, not two trainers. A change to the dataset or+a loss goes in `train_distill_mhr_only.py` and both entry points pick it up; only+cluster plumbing and the CLI belong in `train_distill_jz.py`. Do not fork the+base file — that is how the ONNX bug below happened.++> **`train_distill.py` predates the SimCC 2D head.** Exporting a SimCC+> checkpoint through its ONNX path silently zeroes the 2D head, and the symptom+> is keypoints collapsing to the bounding-box centre with no error raised. Use it+> for reference, not for exporting current checkpoints.++## The `--preset` switch++`train_distill_jz.py --preset {baseline,v2}` selects the model generation.+`baseline` is the default and is bit-for-bit identical to the pre-v2 trainer, so+the two are directly comparable on the same corpus and the same seeds.++`v2` (see `apply_v2_preset`) bundles four changes from MeTRAbs (2007.07227) and+NLF (2407.07532):++| item | change | why |+|---|---|---|+| truncation | zoom ceiling 2.0, `geom_trans` 0.20 | `bbox_square` is exactly 1.2x the tight box on every split, so the old symmetric ±0.25 shaved ~2% off each side and essentially never truncated. Measured effect: images with a truncated keypoint go from 17.2% to 57.0%. |+| prerequisite A | `mask_oob_2d` | SimCC spans ±`kp2d_range` and `pred_2d` is a soft-argmax over the same bins, so a target at 1.67 is unreachable. Clamping asserts it sits at the edge and leaves a permanent saturating gradient. Mask instead; the 3D loss still supervises those joints. Do **not** widen `kp2d_range` — `bin_w = 2*range/(bins-1)`, so that degrades every in-frame joint. |+| prerequisite B | `cliff_follows_aug` | CLIFF describes the box the crop came from. Flip and rotation already moved it; `dx/dy/scale` did not, which was harmless at ±0.08 and 1.25 and is not at truncation strength. |+| occlusion | `RandomErasing` p 0.2 → 0.7 | MeTRAbs Table 7: 52.8 → 49.3 mm on top of an already-strong colour pipeline. |+| JPEG | `RandomJPEG` p 0.3, q 30–90 | The one item on NLF's list that was missing, and the most deployment-relevant — phone camera stacks emit JPEG. |+| absolute pose | `w_reproj` 0.01 → 0.5, flipped half un-masked, `loss_cam` unsquared Euclidean | MeTRAbs Table 8 and NLF §3.2 both insist on supervising the camera-frame pose. At 0.01 the term was decorative and half the batch was masked out of it. |++Two things to know when reading v2 results:++- **PA-MPJPE and root-relative MPJPE are both blind to the translation half of+ the absolute-pose change.** Procrustes removes translation; root-relative+ alignment removes it too. The reprojection and flip-unmasking parts should+ still move them. An absolute metric needs 3DPW's real intrinsics and a+ `f_true/f_assumed` correction — the corpus uses a synthetic+ `f = sqrt(H² + W²)`, ~12% off 3DPW's real focal.+- **Truncation augmentation is not expected to show on a full-body benchmark.**+ MeTRAbs's 124.7 → 77.8 mm is measured on deliberately truncated crops. It is+ in v2 because the papers already established it works, not to be re-proved.++`w_reproj = 0.5` is the least-grounded number in the preset (a 50x jump, chosen+so the term stops being decorative). Sweep it first if v2 underperforms:+`--preset v2 --w_reproj 0.1`.++## The pair index++`SAM3DStudentDataset` enumerates the corpus with one `glob` plus a `stat()` per+crop. At 3.7M crops that is ~7.5M metadata operations, and `build_jz_loaders`+builds two dataset objects per rank — eight concurrent scans on a 4-GPU job, 78+minutes on Lustre before a single training step.++`build_pair_index()` caches that enumeration to+`<data_root>/.pair_index_v2.pkl`, rank 0 builds it behind a barrier while the+others wait, and train/val share one list. Build it ahead of time with+`--warm-cache` (which also populates `$TORCH_HOME` with the timm backbone, since+compute nodes have no internet). Cache validity is keyed on sub-folder mtimes;+`--rebuild-index` forces a rescan.++## Legacy: notebook port for `pfcalcul`++Everything below describes `train_distill.py`, the standalone version of+`notebooks/distill_transformer_decoder.ipynb`. It bundles notebook cells+**0, 1, 3, 7, 8, 10, 11 and 15** into a single script: setup, config, dataset,+student architecture, distillation loss, HMR metrics, the EMA + early-stopping+training loop, and ONNX export/quantization.## Folder layout (on the cluster)diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.pyindex f6eb157..62aa1bd 100755--- a/instanthmr_distill_train/train_distill_jz.py+++ b/instanthmr_distill_train/train_distill_jz.py@@ -50,11 +50,13 @@ import datetime as _dtimport jsonimport mathimport os+import pickleimport randomimport shutilimport socketimport subprocessimport sys+import timefrom pathlib import Pathimport numpy as np@@ -134,6 +136,76 @@ def dataset_of(npz_path: Path) -> str:return npz_path.parent.parent.name+INDEX_VERSION = 2+++def _index_fingerprint(data_root: Path) -> list:+ """Cheap identity for the corpus: the sub-folders and their mtimes.++ A directory's mtime changes when entries are added or removed, so this+ catches the cases that matter (a split rebuilt, a split added) at the cost+ of a handful of stat() calls rather than millions. It will NOT catch a file+ rewritten in place with the same name -- use --rebuild-index for that.+ """+ fp = []+ for ann, img in T.SAM3DStudentDataset._find_dataset_dirs(data_root):+ fp.append([ann.parent.name,+ int(ann.stat().st_mtime_ns), int(img.stat().st_mtime_ns)])+ return sorted(fp)+++def build_pair_index(data_root: str, cache: Path | None = None,+ rebuild: bool = False, log_fn=print) -> list:+ """The (image, npz) pair list, cached to disk.++ Scanning 3.7 M crops takes over an hour on Lustre and every rank of every+ job in a chained run would otherwise repeat it. The cache stores+ ``(sub_folder, stem, image_extension)`` rather than full paths -- an order+ of magnitude smaller than pickling Path objects -- and paths are rebuilt on+ load, which is seconds rather than minutes because it touches no metadata.+ """+ root = Path(data_root)+ cache = Path(cache) if cache else root / f".pair_index_v{INDEX_VERSION}.pkl"+ fp = _index_fingerprint(root)++ if cache.exists() and not rebuild:+ try:+ with cache.open("rb") as fh:+ blob = pickle.load(fh)+ if blob.get("version") == INDEX_VERSION and blob.get("fingerprint") == fp:+ pairs = [(root / sub / "images" / f"{stem}{ext}",+ root / sub / "annotations" / f"{stem}.npz")+ for sub, stem, ext in blob["entries"]]+ log_fn(f" pair index: {len(pairs):,} crops from cache {cache} "+ f"(built {blob.get('built', '?')})")+ return pairs+ log_fn(f" pair index: {cache} is stale (corpus changed) — rebuilding")+ except Exception as e: # truncated / half-written+ log_fn(f" pair index: {cache} unreadable ({e}) — rebuilding")++ t0 = time.time()+ log_fn(f" pair index: scanning {root} (slow — this is why it is cached)")+ ds = T.SAM3DStudentDataset(str(root), augment=False, per_dataset_caps=None)+ entries = [(npz.parent.parent.name, npz.stem, img.suffix)+ for img, npz in ds.pairs]+ log_fn(f" pair index: {len(entries):,} crops in {time.time() - t0:.0f}s")++ try:+ tmp = cache.with_suffix(cache.suffix + ".tmp")+ with tmp.open("wb") as fh:+ pickle.dump({"version": INDEX_VERSION, "fingerprint": fp,+ "built": time.strftime("%Y-%m-%d %H:%M"),+ "entries": entries}, fh, protocol=pickle.HIGHEST_PROTOCOL)+ os.replace(tmp, cache)+ log_fn(f" pair index: cached to {cache}")+ except OSError as e:+ # A read-only data root is not fatal — it just means we pay the scan+ # again next time. Say so rather than dying an hour into startup.+ log_fn(f" pair index: could NOT cache to {cache} ({e}); "+ f"every job will repeat the scan")+ return ds.pairs++def build_mixture_weights(dataset, mix: str | None) -> tuple[torch.Tensor, str]:"""Per-sample sampling weights implementing a target dataset mixture.@@ -243,14 +315,30 @@ def build_jz_loaders(cfg, args, rank: int, world: int):"""geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,geom_scale_range=cfg.geom_scale_range,- geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)-+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,+ geom_scale_max=cfg.geom_scale_max,+ cliff_follows_aug=cfg.cliff_follows_aug,+ occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)++ # Rank 0 builds the index (or reads the cache) while the others wait, then+ # everyone loads the same cache. Building on all ranks at once would put+ # world_size concurrent metadata storms on the same filesystem, which is+ # slower than doing it once and antisocial on a shared cluster.+ if world > 1 and rank != 0:+ dist.barrier()+ pairs = build_pair_index(cfg.data_root, args.index_cache,+ rebuild=args.rebuild_index, log_fn=log)+ if world > 1 and rank == 0:+ dist.barrier()+ if cfg.max_images is not None:+ pairs = pairs[:cfg.max_images]++ # Both datasets share one index; only the augmentation pipeline differs.train_ds = T.SAM3DStudentDataset(- cfg.data_root, augment=cfg.augment, max_images=cfg.max_images,- per_dataset_caps=None, **geom)+ cfg.data_root, augment=cfg.augment, pairs=pairs, **geom)val_ds = T.SAM3DStudentDataset(- cfg.data_root, augment=False, max_images=cfg.max_images,- per_dataset_caps=None, **geom)+ cfg.data_root, augment=False, pairs=pairs, **geom)n = len(train_ds)g = torch.Generator().manual_seed(42) # fixed: the split must not@@ -510,7 +598,8 @@ def train(args, cfg, rank, local_rank, world):raw["Mesh_MPJPE"], em["Mesh_MPJPE"] = (dpw_raw["J12_MPJPE"], dpw_em["J12_MPJPE"])- log(f"\n📈 epoch {epoch+1}/{args.epochs} | lr {scheduler.get_last_lr()[0]:.2e}")+ log(f"\n📈 epoch {epoch+1}/{args.epochs} | lr {scheduler.get_last_lr()[0]:.2e}"+ f" | peak GPU {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB")log(f" train total {tr['total']:.4f} | 3d {tr['3d']:.4f} | "f"2d {tr['2d']:.4f} | simcc {tr['simcc']:.3f}")log(f" valRAW loss {raw['loss']:.4f} | PA-MPJPE {raw['Mesh_PA_MPJPE']:.1f} | "@@ -632,6 +721,17 @@ def parse_args():help="Skip the timm download. Compute nodes have no internet, ""so run --warm-cache once on a login node instead.")p.add_argument("--log-every", type=int, default=100)+ p.add_argument("--preset", choices=["baseline", "v2"], default="baseline",+ help="'v2' enables the wave-1 bundle: truncation, occlusion "+ "and JPEG augmentation with the SimCC-mask and CLIFF "+ "prerequisites, plus absolute-pose supervision. "+ "'baseline' reproduces the pre-v2 trainer exactly.")+ p.add_argument("--w_reproj", type=float, default=None)+ p.add_argument("--index-cache", default=None,+ help="Where to cache the (image, npz) pair index. "+ "Default: <data_root>/.pair_index_v2.pkl")+ p.add_argument("--rebuild-index", action="store_true",+ help="Rescan the corpus even if the cache looks current.")p.add_argument("--warm-cache", action="store_true",help="Instantiate the backbone to populate $TORCH_HOME / ""$HF_HOME, then exit. Run on a login or prepost node.")@@ -657,9 +757,42 @@ def build_cfg(args):if args.w_simcc is not None: cfg.w_simcc = args.w_simccif args.weight_decay is not None: cfg.weight_decay = args.weight_decayif args.ema_decay is not None: cfg.ema_decay = args.ema_decay++ if args.preset == "v2":+ apply_v2_preset(cfg)+ # An explicit --w_reproj still wins over the preset.+ if args.w_reproj is not None: cfg.w_reproj = args.w_reprojreturn cfg+def apply_v2_preset(cfg) -> None:+ """Wave-1 bundle: augmentation (MeTRAbs items 1/2/4) + absolute-pose (item 3).++ Deliberately one preset rather than four independent flags. MeTRAbs measures+ the augmentations cumulatively (Table 7: 58.0 -> 52.8 -> 49.3), so splitting+ them into separate runs would spend three 3-seed sweeps reproducing an+ ablation that already exists in the literature. Absolute-pose supervision+ rides along because it touches disjoint machinery -- the reprojection and+ translation terms, not the input pipeline -- so a regression stays+ attributable from the per-term loss breakdown in history.jsonl.+ """+ # --- item 1: truncation -------------------------------------------------+ cfg.geom_scale_max = 2.0 # retains the central quarter by area+ cfg.geom_trans = 0.20 # off-centre, so truncation is not symmetric+ cfg.cliff_follows_aug = True # prerequisite B+ cfg.mask_oob_2d = True # prerequisite A+ # --- item 2: occlusion --------------------------------------------------+ cfg.occl_p = 0.7 # MeTRAbs Table 7+ cfg.occl_scale = (0.02, 0.30)+ # --- item 4: JPEG -------------------------------------------------------+ cfg.jpeg_p = 0.3+ cfg.jpeg_quality = (30, 90)+ # --- item 3: absolute pose ---------------------------------------------+ cfg.w_reproj = 0.5 # was 0.01, i.e. effectively off+ cfg.reproj_all_samples = True # the flipped half now contributes+ cfg.cam_loss = "euclid"++def main():args = parse_args()@@ -670,6 +803,10 @@ def main():print(f"cached {cfg.backbone}: {sum(p.numel() for p in m.parameters())/1e6:.2f} M "f"params under TORCH_HOME={os.environ.get('TORCH_HOME')} "f"HF_HOME={os.environ.get('HF_HOME')}")+ # The pair index too: this is the other thing a compute node should+ # never have to build, and a prepost node has the walltime for it.+ build_pair_index(cfg.data_root, args.index_cache,+ rebuild=args.rebuild_index)returnrank, local_rank, world = setup_distributed()@@ -705,6 +842,11 @@ def main():log(f"out {cfg.log_dir}")log(f"backbone {cfg.backbone} | epochs {args.epochs} | "f"batch {cfg.batch_size}/gpu | lr {cfg.lr:.2e} ({args.lr_scaling} x{world})")+ log(f"preset {args.preset}"+ + (f" | trunc<={cfg.geom_scale_max} trans={cfg.geom_trans} "+ f"occl_p={cfg.occl_p} jpeg_p={cfg.jpeg_p} "+ f"w_reproj={cfg.w_reproj} cam={cfg.cam_loss}"+ if args.preset != "baseline" else ""))log(f"seed {args.seed} | mix {args.mix} | "f"samples/epoch {args.samples_per_epoch:,}")log("=" * 66)diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.pyindex 2df05d2..5e71719 100644--- a/instanthmr_distill_train/train_distill_mhr_only.py+++ b/instanthmr_distill_train/train_distill_mhr_only.py@@ -44,6 +44,7 @@ import osos.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")import argparse+import ioimport mathimport randomimport gc@@ -149,6 +150,22 @@ class DistillConfig:geom_trans: float = 0.08geom_flip_p: float = 0.5+ # --- v2 knobs. Every default below reproduces the pre-v2 behaviour bit for+ # --- bit, so a baseline run started before these existed is unaffected.+ geom_scale_max: float | None = None # zoom-in ceiling; None => symmetric+ # U(1-range, 1+range) as before. 2.0 is+ # MeTRAbs's quarter-area truncation bound.+ cliff_follows_aug: bool = False # CLIFF cx/cy/b_scale track the augmented+ # crop box, as a real detector box would+ occl_p: float = 0.2 # RandomErasing probability (MeTRAbs: 0.7)+ occl_scale: tuple = (0.02, 0.15)+ jpeg_p: float = 0.0 # random JPEG re-encode probability+ jpeg_quality: tuple = (30, 90)+ mask_oob_2d: bool = False # mask 2D targets outside kp2d_range+ # instead of clamping them to the edge+ reproj_all_samples: bool = False # include flipped samples in loss_reproj+ cam_loss: str = "mse" # "mse" | "euclid" (unsquared L2 on xyz)+kp2d_bins: int = 96kp2d_range: float = 1.5@@ -236,6 +253,28 @@ class DistillConfig:# ============================================================# Cell 3 — Dataset and Data Augmentation# ============================================================+class RandomJPEG:+ """Re-encode through JPEG at a random quality.++ The one item on NLF's augmentation list that this pipeline lacked, and the+ most deployment-relevant of them: phone and tablet camera stacks emit JPEG,+ often aggressively compressed, while training crops are decoded from disk at+ whatever quality the source dataset happened to store. Blocking artefacts+ and chroma subsampling are a distribution the model otherwise never sees.+ """++ def __init__(self, p: float = 0.0, qmin: int = 30, qmax: int = 90):+ self.p, self.qmin, self.qmax = p, qmin, qmax++ def __call__(self, img):+ if self.p <= 0.0 or random.random() >= self.p:+ return img+ buf = io.BytesIO()+ img.save(buf, format="JPEG", quality=random.randint(self.qmin, self.qmax))+ buf.seek(0)+ return Image.open(buf).convert("RGB")++class RandomPixelate:def __init__(self, p=0.2, min_res=32, max_res=112):self.p = p@@ -255,9 +294,14 @@ class SAM3DStudentDataset(Dataset):def __init__(self, data_root: str,image_size: int = 224, max_images: int | None = None,augment: bool = True, per_dataset_caps: dict | None = None,+ pairs: list | None = None,geom_p: float = 0.8, geom_rot_deg: float = 30.0,geom_scale_range: float = 0.25, geom_trans: float = 0.08,- geom_flip_p: float = 0.5):+ geom_flip_p: float = 0.5,+ geom_scale_max: float | None = None,+ cliff_follows_aug: bool = False,+ occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),+ jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90)):super().__init__()self.image_size = image_sizeself.data_root = Path(data_root)@@ -267,6 +311,8 @@ class SAM3DStudentDataset(Dataset):self.geom_scale_range = geom_scale_rangeself.geom_trans = geom_transself.geom_flip_p = geom_flip_p+ self.geom_scale_max = geom_scale_max+ self.cliff_follows_aug = cliff_follows_augtfm_list = [transforms.Resize((image_size, image_size))]@@ -275,6 +321,7 @@ class SAM3DStudentDataset(Dataset):transforms.RandomApply([transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),RandomPixelate(p=0.2, min_res=48, max_res=112),+ RandomJPEG(p=jpeg_p, qmin=jpeg_quality[0], qmax=jpeg_quality[1]),transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),])@@ -285,10 +332,23 @@ class SAM3DStudentDataset(Dataset):])if augment:- tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))+ # MeTRAbs Table 7 puts synthetic occlusion at p=0.7 and credits it+ # with 52.8 -> 49.3 mm on top of an already-strong colour pipeline.+ tfm_list.append(transforms.RandomErasing(p=occl_p, scale=tuple(occl_scale)))self.transform = transforms.Compose(tfm_list)+ # A prebuilt index short-circuits the filesystem scan below. That scan+ # is one glob plus a stat() per crop -- ~7.5 M metadata operations on a+ # 3.7 M-crop corpus -- and on a parallel filesystem it dominates job+ # startup, so having every DDP rank redo it is the largest fixed cost of+ # a cluster run. See build_pair_index() in train_distill_jz.py.+ if pairs is not None:+ self.pairs = list(pairs)+ print(f" Prebuilt index: {len(self.pairs)} (image, npz) pairs "+ f"(augment={augment}).")+ return+self.pairs = []dataset_dirs = self._find_dataset_dirs(self.data_root)if not dataset_dirs:@@ -407,7 +467,16 @@ class SAM3DStudentDataset(Dataset):M_flip = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])angle = random.uniform(-self.geom_rot_deg, self.geom_rot_deg)- scale = random.uniform(1.0 - self.geom_scale_range, 1.0 + self.geom_scale_range)+ # Asymmetric by design. bbox_square is exactly 1.2x the tight box on+ # every split, so the body fills 83.3% of the crop and the old+ # symmetric ceiling of 1.25 shaved ~2% off each side -- it+ # essentially never truncated. A ceiling of 2.0 retains the central+ # quarter by area, which is MeTRAbs's bound. Zoom-OUT is still+ # capped at 1-geom_scale_range because there are no pixels outside+ # bbox_square; warpAffine would fill the margin with black.+ scale_hi = (1.0 + self.geom_scale_range if self.geom_scale_max is None+ else self.geom_scale_max)+ scale = random.uniform(1.0 - self.geom_scale_range, scale_hi)dx = random.uniform(-self.geom_trans, self.geom_trans)dy = random.uniform(-self.geom_trans, self.geom_trans)M_rst = cv2.getRotationMatrix2D((W / 2.0, H / 2.0), angle, scale)@@ -451,6 +520,19 @@ class SAM3DStudentDataset(Dataset):dxy = R_v.astype(np.float64) @ dxycx, cy = orig_w / 2.0 + dxy[0], orig_h / 2.0 + dxy[1]+ # CLIFF conditioning describes the box this crop came from. The flip+ # and rotation above already move it; dx/dy/scale did not, which was+ # harmless at +/-0.08 and 1.25 but not at truncation strength. A+ # crop-space shift of dx corresponds to the box moving+ # dx * crop_w / 2 original pixels, and a zoom of s shrinks the+ # visible box by s -- which is exactly what an off-centre or badly+ # sized detector box looks like at deployment.+ if self.cliff_follows_aug:+ crop_w = float(sq_bbox[2] - sq_bbox[0])+ cx -= dx * crop_w / 2.0+ cy -= dy * crop_w / 2.0+ b_scale = b_scale / scale+image = self.transform(img_pil)cx_norm = 2.0 * (cx / orig_w) - 1.0@@ -479,7 +561,11 @@ class SAM3DStudentDataset(Dataset):def build_dataloaders(cfg):geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,geom_scale_range=cfg.geom_scale_range,- geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,+ geom_scale_max=cfg.geom_scale_max,+ cliff_follows_aug=cfg.cliff_follows_aug,+ occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)full_dataset = SAM3DStudentDataset(cfg.data_root,augment=cfg.augment,@@ -811,7 +897,15 @@ class DistillationLoss(nn.Module):losses['loss_pose'] = self.m_smooth_l1(pred_pose, tgt_pose, m_ident) * self.cfg.w_poselosses['loss_scale'] = self.m_mse(pred_scale, tgt_scale, m_noflip) * self.cfg.w_scalelosses['loss_shape'] = self.m_mse(preds["shape_params"], targets["shape_params"], m_noflip) * self.cfg.w_shape- losses['loss_cam'] = self.mse(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam+ # NLF applies its translation loss as an unsquared Euclidean norm over the+ # xyz triple rather than elementwise MSE: MSE lets one badly-scaled axis+ # (in practice depth, which is an order of magnitude larger than x/y)+ # dominate the gradient of all three.+ if getattr(self.cfg, "cam_loss", "mse") == "euclid":+ e_cam = torch.linalg.norm(preds["cam_trans"] - targets["cam_trans"], dim=-1)+ losses['loss_cam'] = e_cam.mean() * self.cfg.w_cam+ else:+ losses['loss_cam'] = self.mse(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_campred_mhr_joints = self.mhr_module.get_joints(preds["mhr_params"], preds["shape_params"])[..., :3]with torch.no_grad():@@ -852,15 +946,31 @@ class DistillationLoss(nn.Module):if w_kp is None:w_kp = torch.ones_like(tgt_2d[..., 0])+ # Truncation augmentation pushes keypoints outside the crop. Both 2D+ # terms below are BOUNDED -- SimCC spans +/-kp2d_range and pred_2d is a+ # soft-argmax over those same bins -- so a target at 1.67 cannot be+ # reached. Clamping it to 1.5 asserts the joint is exactly at the crop+ # edge, which is false, and leaves a permanent saturating gradient at+ # w_keypoints2d = 10. Mask instead: the 3D loss still supervises those+ # joints, which is the whole MeTRo argument for predicting them at all.+ #+ # Do NOT widen kp2d_range instead: bin_w = 2*range/(bins-1), so at 96+ # bins that trades 3.5 px bins for 5.9 px, degrading every in-frame+ # joint to accommodate the out-of-frame ones.+ w_kp_2d = w_kp+ if getattr(self.cfg, "mask_oob_2d", False):+ in_crop = (tgt_2d.abs() <= self.cfg.kp2d_range).all(-1).to(w_kp.dtype)+ w_kp_2d = w_kp * in_crop+# 2D head — training-only auxiliary task, and the ONLY supervision that# survives a horizontal flip in full (targets["joints_2d"] is mirrored# AND FLIP_PERM-permuted by the dataset, so it stays self-consistent —# and so is joints_2d_vis).e2d = F.smooth_l1_loss(pred_2d[..., :2], tgt_2d, reduction='none').mean(-1)- losses['loss_2d_native'] = self._kp_weighted(e2d, w_kp) * self.cfg.w_keypoints2d+ losses['loss_2d_native'] = self._kp_weighted(e2d, w_kp_2d) * self.cfg.w_keypoints2dif "joints_2d_logits" in preds:losses['loss_2d_simcc'] = self._simcc_ce(- preds["joints_2d_logits"], tgt_2d, w_kp) * self.cfg.w_simcc+ preds["joints_2d_logits"], tgt_2d, w_kp_2d) * self.cfg.w_simcc# Main 3D loss, now on the MHR-derived keypoints. Deliberately UNMASKED:# targets["joints_3d"] is mirrored + permuted under flip, and the MHR rest@@ -883,14 +993,32 @@ class DistillationLoss(nn.Module):cam_cr = _counter_rot_cam(preds["cam_trans"])# Single re-projection (replaces loss_reproj_native + loss_reproj_mhr):- # one geometry, all 70 keypoints. Still m_noflip — the re-projection- # path undoes the rotation but not the mirror, so flipped samples would- # need an extra FLIP_PERM + cx mirror that is not worth the complexity- # for a 0.01-weight auxiliary term.- pred_reproj = self._project_3d_to_norm_2d(_counter_rot_pts(pred_kp3d), cam_cr, targets)+ # one geometry, all 70 keypoints.+ #+ # The projection runs in the ORIGINAL image frame (real focal, image+ # centre, un-augmented bbox_square) and only then applies aug_M, which+ # carries the mirror. So a flipped sample has to be un-mirrored first,+ # or it gets mirrored twice. The dataset applies flip THEN rotation, so+ # the inverse is counter-rotate THEN negate x -- on the keypoints and on+ # cam_trans alike, both of which the dataset negated. Index order needs+ # no fixing: pred_kp3d is already in FLIP_PERM order and aug_M maps it to+ # the same order as tgt_2d.+ #+ # Previously this whole term was masked to m_noflip, which was defensible+ # at w_reproj = 0.01 and is not once the weight matters: half the batch+ # would contribute nothing to the only loss tying 3D to pixels.+ pts_cr, cam_cr_r = _counter_rot_pts(pred_kp3d), cam_cr+ m_reproj = m_noflip+ if getattr(self.cfg, "reproj_all_samples", False):+ sgn = (1.0 - 2.0 * aug_flip).view(-1, 1) # +1 / -1+ pts_cr = torch.stack([pts_cr[..., 0] * sgn, pts_cr[..., 1],+ pts_cr[..., 2]], dim=-1)+ cam_cr_r = torch.cat([cam_cr[:, 0:1] * sgn, cam_cr[:, 1:]], dim=1)+ m_reproj = ones+ pred_reproj = self._project_3d_to_norm_2d(pts_cr, cam_cr_r, targets)e_reproj = (pred_reproj - tgt_2d).abs().mean(-1)losses['loss_reproj'] = self._kp_weighted(- e_reproj, w_kp, m_noflip) * self.cfg.w_reproj * self.fk_scale+ e_reproj, w_kp, m_reproj) * self.cfg.w_reproj * self.fk_scalelosses['total_loss'] = sum(losses.values())return losses