Skip to content
Python

feat: v2 training preset (truncation/occlusion/JPEG aug + absolute-pose supervision)

mohamdev8468be118c31
diff --git a/datasets_pipeline/verify_split.py b/datasets_pipeline/verify_split.py
index 2eba390..ad18caa 100755
--- a/datasets_pipeline/verify_split.py
+++ b/datasets_pipeline/verify_split.py
@@ -26,11 +26,13 @@ second encoder.
from __future__ import annotations
import argparse
+import os
import random
import sys
from pathlib import Path
import numpy as np
+from concurrent.futures import ThreadPoolExecutor
HERE = Path(__file__).resolve().parent
REPO = HERE.parent
@@ -46,7 +48,30 @@ LEFT_HAND_KPS = list(range(42, 63))
RIGHT_WRIST, LEFT_WRIST = 41, 62
-def structure(d: Path) -> None:
+def count_files(root: Path) -> int:
+ """Recursive file count that never follows a symlink.
+
+ original_images/ can hold >1M symlinks into a read-only mirror on another
+ filesystem; Path.is_file() resolves each one, turning a count into a
+ million cross-filesystem stats. scandir's d_type answers dir-or-not
+ without any stat at all.
+ """
+ n, stack = 0, [root]
+ while stack:
+ try:
+ with os.scandir(stack.pop()) as it:
+ for e in it:
+ if e.is_dir(follow_symlinks=False):
+ stack.append(Path(e.path))
+ else:
+ n += 1
+ except OSError:
+ pass
+ return n
+
+
+def structure(d: Path, workers: int = 16, hand_scan_limit: int = 300_000,
+ seed: int = 0) -> None:
anns = sorted((d / "annotations").glob("*.npz"))
body = {p.stem for p in (d / "body_crops").iterdir()} if (d / "body_crops").exists() else set()
hands = {p.stem for p in (d / "hands_crops").iterdir()} if (d / "hands_crops").exists() else set()
@@ -54,27 +79,51 @@ def structure(d: Path) -> None:
missing_body = sorted(stems - body)[:5]
orphan_body = sorted(body - stems)[:5]
- want_hands, have_hands, missing_hands = 0, 0, []
- for p in anns:
- a = np.load(p, allow_pickle=False)
+
+ # Reading every npz is the expensive half of this check: single-threaded it
+ # runs at ~100/s on Lustre, so a 3.4M-crop split would take 9 h. Sample
+ # above hand_scan_limit and scale, and thread the reads either way.
+ scan = anns
+ scaled = False
+ if len(anns) > hand_scan_limit:
+ scan = random.Random(seed).sample(anns, hand_scan_limit)
+ scaled = True
+
+ def probe(p: Path):
+ try:
+ a = np.load(p, allow_pickle=False)
+ except Exception: # noqa: BLE001
+ return ()
+ out = []
for side in ("l", "r"):
key = f"hand_valid_{side}"
- if key not in a.files or not bool(a[key]):
- continue
- want_hands += 1
- if f"{p.stem}_{side}" in hands:
- have_hands += 1
- elif len(missing_hands) < 5:
- missing_hands.append(f"{p.stem}_{side}")
+ if key in a.files and bool(a[key]):
+ out.append(f"{p.stem}_{side}")
+ return tuple(out)
+
+ want_hands, have_hands, missing_hands = 0, 0, []
+ with ThreadPoolExecutor(workers) as ex:
+ for names in ex.map(probe, scan):
+ for nm in names:
+ want_hands += 1
+ if nm in hands:
+ have_hands += 1
+ elif len(missing_hands) < 5:
+ missing_hands.append(nm)
print(f" annotations : {len(anns):,}")
print(f" body_crops : {len(body):,}"
f" (missing {len(stems - body):,}, orphan {len(body - stems):,})")
- print(f" hands_crops : {len(hands):,}"
- f" (declared {want_hands:,}, present {have_hands:,})")
+ if scaled:
+ f = len(anns) / len(scan)
+ print(f" hands_crops : {len(hands):,}"
+ f" (declared ~{int(want_hands * f):,}, present ~{int(have_hands * f):,}"
+ f" — estimated from {len(scan):,} of {len(anns):,} npz)")
+ else:
+ print(f" hands_crops : {len(hands):,}"
+ f" (declared {want_hands:,}, present {have_hands:,})")
if (d / "original_images").exists():
- n = sum(1 for _ in (d / "original_images").rglob("*") if _.is_file())
- print(f" original_images : {n:,} files")
+ print(f" original_images : {count_files(d / 'original_images'):,} files")
if missing_body:
print(f" !! body crop missing for: {missing_body}")
if orphan_body:
@@ -149,6 +198,10 @@ def main() -> None:
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--dir", type=Path, required=True, help="data/sam3d_gt_<dataset>")
p.add_argument("--sample", type=int, default=500)
+ p.add_argument("--workers", type=int, default=16,
+ help="threads for the npz hand-validity scan")
+ p.add_argument("--hand-scan-limit", type=int, default=300_000,
+ help="above this many npz, estimate hand counts from a sample instead of reading every file")
p.add_argument("--seed", type=int, default=0)
p.add_argument("--hand-signal", action="store_true")
p.add_argument("--mhr", type=Path,
@@ -162,7 +215,8 @@ def main() -> None:
print(f"\n== {args.dir} ==")
print("structure")
- structure(args.dir)
+ structure(args.dir, workers=args.workers,
+ hand_scan_limit=args.hand_scan_limit, seed=args.seed)
print("geometry")
geometry(args.dir, args.sample, args.seed)
if args.hand_signal:
diff --git a/demo.py b/demo.py
index 44ded50..5e98a14 100755
--- a/demo.py
+++ b/demo.py
@@ -303,7 +303,8 @@ def build_all(args: argparse.Namespace):
The rig comes first because an MHR-only graph cannot produce 3D keypoints
without it — the pipeline takes it as a constructor argument rather than
- the visualizer alone.
+ the visualizer alone. It is returned as well so the temporal filter can
+ re-derive keypoints from filtered pose parameters.
"""
model_path = resolve_model_path(args)
mhr = build_mhr(args, required=_model_needs_mhr(model_path))
@@ -315,7 +316,7 @@ def build_all(args: argparse.Namespace):
mhr_renderer=mhr,
mesh_alpha=args.mesh_alpha,
)
- return pipeline, viz
+ return pipeline, viz, mhr
def _print_timings(
@@ -343,21 +344,30 @@ def _print_timings(
# ---------------------------------------------------------------------------
-def build_smoother(args: argparse.Namespace, live: bool = False):
+def build_smoother(args: argparse.Namespace, live: bool = False, mhr=None):
"""Construct the temporal filter, or ``None`` when --smoothing-filter is off.
``live`` forces the adaptive filter: a camera cannot buffer future frames,
and a trailing average would show visible phase lag on fast motion.
+
+ Handing it the MHR backend lets it re-derive the 3D keypoints from the
+ filtered pose parameters, so they stay welded to the mesh instead of
+ drifting off it during fast motion.
"""
if not args.smoothing_filter:
return None
mode = "1euro" if live else args.smoothing_mode
+ kp_fn = mhr.keypoints if getattr(mhr, "has_keypoints", False) else None
sm = TemporalSmoother(
window=args.smoothing_window,
mode=mode,
beta_scale=args.smoothing_beta,
mincutoff_scale=args.smoothing_mincutoff,
+ keypoint_fn=kp_fn,
)
+ if kp_fn is None:
+ print(" note: no MHR keypoint backend — keypoints are filtered on their "
+ "own, so they may drift off the mesh during fast motion.")
if sm.mode == "1euro":
print(f"temporal smoothing: 1euro, beta x{sm.beta_scale:g}, "
f"mincutoff x{sm.mincutoff_scale:g}, no latency")
@@ -377,7 +387,7 @@ def run_image(args: argparse.Namespace) -> None:
sys.exit(f"[error] cv2.imread failed for {image_path}")
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
- pipeline, viz = build_all(args)
+ pipeline, viz, _ = build_all(args)
result = pipeline.predict(rgb)
@@ -408,8 +418,8 @@ def run_video(args: argparse.Namespace) -> None:
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print(f"video: {video_path.name}, {total} frames @ {fps:.1f} fps")
- pipeline, viz = build_all(args)
- smoother = build_smoother(args)
+ pipeline, viz, mhr = build_all(args)
+ smoother = build_smoother(args, mhr=mhr)
frame_idx = 0
sent = 0
@@ -511,8 +521,8 @@ def run_camera(args: argparse.Namespace) -> None:
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
print(f"camera {args.camera}: {width}x{height} @ {fps:.1f} fps — Ctrl+C to stop")
- pipeline, viz = build_all(args)
- smoother = build_smoother(args, live=True)
+ pipeline, viz, mhr = build_all(args)
+ smoother = build_smoother(args, live=True, mhr=mhr)
frame_idx = 0
det_times: list[float] = []
diff --git a/docs/training.md b/docs/training.md
index 3aeeaf1..4642e7c 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -2,7 +2,15 @@
The InstantHMR training notebook is
[`notebooks/distill_transformer_decoder.ipynb`](../notebooks/distill_transformer_decoder.ipynb).
-It is the source of truth — this page is a tour, not a re-implementation.
+It is the source of truth for the notebook workflow — this page is a tour, not a
+re-implementation.
+
+**For cluster training, the notebook is not the source of truth.** That is
+`instanthmr_distill_train/train_distill_mhr_only.py` (dataset, losses, model)
+driven by `train_distill_jz.py` (DDP, mixture sampling, resume, 3DPW
+validation), and the two have diverged from the notebook. See
+[`instanthmr_distill_train/README.md`](../instanthmr_distill_train/README.md)
+for which script to run, what `--preset v2` changes, and the pair-index cache.
## Setup
diff --git a/instanthmr/smoothing.py b/instanthmr/smoothing.py
index a49d8ee..3ac56eb 100644
--- a/instanthmr/smoothing.py
+++ b/instanthmr/smoothing.py
@@ -54,7 +54,9 @@ from .inference import HMRPrediction
#: linear mean is the right operation, except ``mhr_params`` — see the module
#: docstring. ``joints_3d_cam`` is deliberately absent: it is recomputed from
#: the smoothed ``joints_3d_local`` and ``cam_trans`` so the three stay
-#: consistent with each other.
+#: consistent with each other. ``joints_3d_local`` is filtered only when no
+#: ``keypoint_fn`` is available; with one, it is re-derived from the filtered
+#: ``mhr_params`` so the keypoints and the mesh cannot disagree.
SMOOTHED_FIELDS = (
"joints_3d_local",
"cam_trans",
@@ -216,6 +218,10 @@ class TemporalSmoother:
lower it if the output looks noisy while you hold still.
mincutoff_scale: multiplies every per-field ``mincutoff``. Governs how
hard the signal is smoothed at rest.
+ keypoint_fn: optional ``f(mhr_params, shape_params) -> (70, 3)``. When
+ given, ``joints_3d_local`` is *re-derived* from the filtered pose
+ parameters instead of being filtered on its own. This is what keeps
+ the keypoints welded to the mesh — see :meth:`_finalize`.
iou_threshold: minimum IoU to associate a detection with a track.
max_age: frames a track survives without a detection.
@@ -236,6 +242,7 @@ class TemporalSmoother:
mode: str = "centered",
beta_scale: float = 1.0,
mincutoff_scale: float = 1.0,
+ keypoint_fn=None,
iou_threshold: float = 0.3,
max_age: int = 5,
):
@@ -251,6 +258,7 @@ class TemporalSmoother:
self.lag = (window - 1) // 2 if mode == "centered" else 0
self.beta_scale = float(beta_scale)
self.mincutoff_scale = float(mincutoff_scale)
+ self.keypoint_fn = keypoint_fn
self._tracker = _Tracker(iou_threshold=iou_threshold, max_age=max_age)
self._buf: deque[dict[int, HMRPrediction]] = deque(maxlen=window)
@@ -340,10 +348,7 @@ class TemporalSmoother:
name: bank[name](getattr(pred, name), t).astype(np.float32)
for name in SMOOTHED_FIELDS
}
- fields["joints_3d_cam"] = (
- fields["joints_3d_local"] + fields["cam_trans"][None, :]
- ).astype(np.float32)
- smoothed = replace(pred, **fields)
+ smoothed = self._finalize(pred, fields)
out.append(smoothed)
emitted[track_id] = smoothed
@@ -358,6 +363,31 @@ class TemporalSmoother:
return out
# ------------------------------------------------------------------
+ def _finalize(self, anchor: HMRPrediction, fields: dict) -> HMRPrediction:
+ """Assemble one smoothed prediction, keeping its parts self-consistent.
+
+ Filtering ``mhr_params`` and ``joints_3d_local`` independently makes the
+ green keypoints drift off the skin during fast motion, because the mesh
+ is decoded from the former while the keypoints come from the latter and
+ forward kinematics is *nonlinear* — ``filter(FK(p)) != FK(filter(p))``.
+ Adaptive filtering makes it worse still, since each field then gets its
+ own cutoff driven by its own speed in its own units.
+
+ So when a keypoint backend is available we do not filter the keypoints
+ at all: we filter the pose parameters and re-derive the keypoints from
+ them, which is exactly how ``inference.py`` produces them for an
+ MHR-only graph. The two agree by construction, in every mode.
+ """
+ if self.keypoint_fn is not None:
+ fields["joints_3d_local"] = np.asarray(
+ self.keypoint_fn(fields["mhr_params"], fields["shape_params"]),
+ dtype=np.float32,
+ ).reshape(anchor.joints_3d_local.shape)
+ fields["joints_3d_cam"] = (
+ fields["joints_3d_local"] + fields["cam_trans"][None, :]
+ ).astype(np.float32)
+ return replace(anchor, **fields)
+
def _emit(self, centre: int) -> list[HMRPrediction]:
"""Average every track present in frame ``centre`` over the buffer."""
out: list[HMRPrediction] = []
@@ -372,10 +402,7 @@ class TemporalSmoother:
).astype(np.float32)
for name in SMOOTHED_FIELDS
}
- fields["joints_3d_cam"] = (
- fields["joints_3d_local"] + fields["cam_trans"][None, :]
- ).astype(np.float32)
- smoothed = replace(anchor, **fields)
+ smoothed = self._finalize(anchor, fields)
out.append(smoothed)
emitted[track_id] = smoothed
diff --git a/instanthmr_distill_train/README.md b/instanthmr_distill_train/README.md
index b9680b3..3537c41 100644
--- a/instanthmr_distill_train/README.md
+++ b/instanthmr_distill_train/README.md
@@ -1,10 +1,81 @@
-# InstantHMR Distillation — Standalone Cloud Training
-
-Standalone version of `notebooks/distill_transformer_decoder.ipynb` for running
-on the `pfcalcul` SLURM cluster. It bundles notebook cells **0, 1, 3, 7, 8, 10,
-11 and 15** into a single script: setup, config, dataset, student architecture,
-distillation loss, HMR metrics, the EMA + early-stopping training loop, and ONNX
-export/quantization.
+# InstantHMR Distillation — training scripts
+
+## Which script
+
+Four files, and the difference matters.
+
+| file | what it is |
+|---|---|
+| `train_distill_mhr_only.py` | **the trainer.** Dataset, augmentation, student architecture, MHR forward pass, all losses, metrics, the single-GPU loop. Everything of substance lives here. |
+| `train_distill_jz.py` | **the multi-GPU entry point.** `import train_distill_mhr_only as T` plus what a cluster needs: DDP, mixture sampling, resumable checkpointing, held-out 3DPW validation, the pair-index cache. Run this one on a cluster. |
+| `val3dpw.py` | 3DPW held-out validation used by `train_distill_jz.py`. Cross-validated against `benchmark/eval_3dpw.py` — they agree exactly. |
+| `train_distill.py` | **legacy.** The original notebook port. See the warning below before using it. |
+
+They are one trainer in two layers, not two trainers. A change to the dataset or
+a loss goes in `train_distill_mhr_only.py` and both entry points pick it up; only
+cluster plumbing and the CLI belong in `train_distill_jz.py`. Do not fork the
+base file — that is how the ONNX bug below happened.
+
+> **`train_distill.py` predates the SimCC 2D head.** Exporting a SimCC
+> checkpoint through its ONNX path silently zeroes the 2D head, and the symptom
+> is keypoints collapsing to the bounding-box centre with no error raised. Use it
+> for reference, not for exporting current checkpoints.
+
+## The `--preset` switch
+
+`train_distill_jz.py --preset {baseline,v2}` selects the model generation.
+`baseline` is the default and is bit-for-bit identical to the pre-v2 trainer, so
+the two are directly comparable on the same corpus and the same seeds.
+
+`v2` (see `apply_v2_preset`) bundles four changes from MeTRAbs (2007.07227) and
+NLF (2407.07532):
+
+| item | change | why |
+|---|---|---|
+| truncation | zoom ceiling 2.0, `geom_trans` 0.20 | `bbox_square` is exactly 1.2x the tight box on every split, so the old symmetric ±0.25 shaved ~2% off each side and essentially never truncated. Measured effect: images with a truncated keypoint go from 17.2% to 57.0%. |
+| prerequisite A | `mask_oob_2d` | SimCC spans ±`kp2d_range` and `pred_2d` is a soft-argmax over the same bins, so a target at 1.67 is unreachable. Clamping asserts it sits at the edge and leaves a permanent saturating gradient. Mask instead; the 3D loss still supervises those joints. Do **not** widen `kp2d_range` — `bin_w = 2*range/(bins-1)`, so that degrades every in-frame joint. |
+| prerequisite B | `cliff_follows_aug` | CLIFF describes the box the crop came from. Flip and rotation already moved it; `dx/dy/scale` did not, which was harmless at ±0.08 and 1.25 and is not at truncation strength. |
+| occlusion | `RandomErasing` p 0.2 → 0.7 | MeTRAbs Table 7: 52.8 → 49.3 mm on top of an already-strong colour pipeline. |
+| JPEG | `RandomJPEG` p 0.3, q 30–90 | The one item on NLF's list that was missing, and the most deployment-relevant — phone camera stacks emit JPEG. |
+| absolute pose | `w_reproj` 0.01 → 0.5, flipped half un-masked, `loss_cam` unsquared Euclidean | MeTRAbs Table 8 and NLF §3.2 both insist on supervising the camera-frame pose. At 0.01 the term was decorative and half the batch was masked out of it. |
+
+Two things to know when reading v2 results:
+
+- **PA-MPJPE and root-relative MPJPE are both blind to the translation half of
+ the absolute-pose change.** Procrustes removes translation; root-relative
+ alignment removes it too. The reprojection and flip-unmasking parts should
+ still move them. An absolute metric needs 3DPW's real intrinsics and a
+ `f_true/f_assumed` correction — the corpus uses a synthetic
+ `f = sqrt(H² + W²)`, ~12% off 3DPW's real focal.
+- **Truncation augmentation is not expected to show on a full-body benchmark.**
+ MeTRAbs's 124.7 → 77.8 mm is measured on deliberately truncated crops. It is
+ in v2 because the papers already established it works, not to be re-proved.
+
+`w_reproj = 0.5` is the least-grounded number in the preset (a 50x jump, chosen
+so the term stops being decorative). Sweep it first if v2 underperforms:
+`--preset v2 --w_reproj 0.1`.
+
+## The pair index
+
+`SAM3DStudentDataset` enumerates the corpus with one `glob` plus a `stat()` per
+crop. At 3.7M crops that is ~7.5M metadata operations, and `build_jz_loaders`
+builds two dataset objects per rank — eight concurrent scans on a 4-GPU job, 78
+minutes on Lustre before a single training step.
+
+`build_pair_index()` caches that enumeration to
+`<data_root>/.pair_index_v2.pkl`, rank 0 builds it behind a barrier while the
+others wait, and train/val share one list. Build it ahead of time with
+`--warm-cache` (which also populates `$TORCH_HOME` with the timm backbone, since
+compute nodes have no internet). Cache validity is keyed on sub-folder mtimes;
+`--rebuild-index` forces a rescan.
+
+## Legacy: notebook port for `pfcalcul`
+
+Everything below describes `train_distill.py`, the standalone version of
+`notebooks/distill_transformer_decoder.ipynb`. It bundles notebook cells
+**0, 1, 3, 7, 8, 10, 11 and 15** into a single script: setup, config, dataset,
+student architecture, distillation loss, HMR metrics, the EMA + early-stopping
+training loop, and ONNX export/quantization.
## Folder layout (on the cluster)
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.py
index f6eb157..62aa1bd 100755
--- a/instanthmr_distill_train/train_distill_jz.py
+++ b/instanthmr_distill_train/train_distill_jz.py
@@ -50,11 +50,13 @@ import datetime as _dt
import json
import math
import os
+import pickle
import random
import shutil
import socket
import subprocess
import sys
+import time
from pathlib import Path
import numpy as np
@@ -134,6 +136,76 @@ def dataset_of(npz_path: Path) -> str:
return npz_path.parent.parent.name
+INDEX_VERSION = 2
+
+
+def _index_fingerprint(data_root: Path) -> list:
+ """Cheap identity for the corpus: the sub-folders and their mtimes.
+
+ A directory's mtime changes when entries are added or removed, so this
+ catches the cases that matter (a split rebuilt, a split added) at the cost
+ of a handful of stat() calls rather than millions. It will NOT catch a file
+ rewritten in place with the same name -- use --rebuild-index for that.
+ """
+ fp = []
+ for ann, img in T.SAM3DStudentDataset._find_dataset_dirs(data_root):
+ fp.append([ann.parent.name,
+ int(ann.stat().st_mtime_ns), int(img.stat().st_mtime_ns)])
+ return sorted(fp)
+
+
+def build_pair_index(data_root: str, cache: Path | None = None,
+ rebuild: bool = False, log_fn=print) -> list:
+ """The (image, npz) pair list, cached to disk.
+
+ Scanning 3.7 M crops takes over an hour on Lustre and every rank of every
+ job in a chained run would otherwise repeat it. The cache stores
+ ``(sub_folder, stem, image_extension)`` rather than full paths -- an order
+ of magnitude smaller than pickling Path objects -- and paths are rebuilt on
+ load, which is seconds rather than minutes because it touches no metadata.
+ """
+ root = Path(data_root)
+ cache = Path(cache) if cache else root / f".pair_index_v{INDEX_VERSION}.pkl"
+ fp = _index_fingerprint(root)
+
+ if cache.exists() and not rebuild:
+ try:
+ with cache.open("rb") as fh:
+ blob = pickle.load(fh)
+ if blob.get("version") == INDEX_VERSION and blob.get("fingerprint") == fp:
+ pairs = [(root / sub / "images" / f"{stem}{ext}",
+ root / sub / "annotations" / f"{stem}.npz")
+ for sub, stem, ext in blob["entries"]]
+ log_fn(f" pair index: {len(pairs):,} crops from cache {cache} "
+ f"(built {blob.get('built', '?')})")
+ return pairs
+ log_fn(f" pair index: {cache} is stale (corpus changed) — rebuilding")
+ except Exception as e: # truncated / half-written
+ log_fn(f" pair index: {cache} unreadable ({e}) — rebuilding")
+
+ t0 = time.time()
+ log_fn(f" pair index: scanning {root} (slow — this is why it is cached)")
+ ds = T.SAM3DStudentDataset(str(root), augment=False, per_dataset_caps=None)
+ entries = [(npz.parent.parent.name, npz.stem, img.suffix)
+ for img, npz in ds.pairs]
+ log_fn(f" pair index: {len(entries):,} crops in {time.time() - t0:.0f}s")
+
+ try:
+ tmp = cache.with_suffix(cache.suffix + ".tmp")
+ with tmp.open("wb") as fh:
+ pickle.dump({"version": INDEX_VERSION, "fingerprint": fp,
+ "built": time.strftime("%Y-%m-%d %H:%M"),
+ "entries": entries}, fh, protocol=pickle.HIGHEST_PROTOCOL)
+ os.replace(tmp, cache)
+ log_fn(f" pair index: cached to {cache}")
+ except OSError as e:
+ # A read-only data root is not fatal — it just means we pay the scan
+ # again next time. Say so rather than dying an hour into startup.
+ log_fn(f" pair index: could NOT cache to {cache} ({e}); "
+ f"every job will repeat the scan")
+ return ds.pairs
+
+
def build_mixture_weights(dataset, mix: str | None) -> tuple[torch.Tensor, str]:
"""Per-sample sampling weights implementing a target dataset mixture.
@@ -243,14 +315,30 @@ def build_jz_loaders(cfg, args, rank: int, world: int):
"""
geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,
geom_scale_range=cfg.geom_scale_range,
- geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)
-
+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,
+ geom_scale_max=cfg.geom_scale_max,
+ cliff_follows_aug=cfg.cliff_follows_aug,
+ occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)
+
+ # Rank 0 builds the index (or reads the cache) while the others wait, then
+ # everyone loads the same cache. Building on all ranks at once would put
+ # world_size concurrent metadata storms on the same filesystem, which is
+ # slower than doing it once and antisocial on a shared cluster.
+ if world > 1 and rank != 0:
+ dist.barrier()
+ pairs = build_pair_index(cfg.data_root, args.index_cache,
+ rebuild=args.rebuild_index, log_fn=log)
+ if world > 1 and rank == 0:
+ dist.barrier()
+ if cfg.max_images is not None:
+ pairs = pairs[:cfg.max_images]
+
+ # Both datasets share one index; only the augmentation pipeline differs.
train_ds = T.SAM3DStudentDataset(
- cfg.data_root, augment=cfg.augment, max_images=cfg.max_images,
- per_dataset_caps=None, **geom)
+ cfg.data_root, augment=cfg.augment, pairs=pairs, **geom)
val_ds = T.SAM3DStudentDataset(
- cfg.data_root, augment=False, max_images=cfg.max_images,
- per_dataset_caps=None, **geom)
+ cfg.data_root, augment=False, pairs=pairs, **geom)
n = len(train_ds)
g = torch.Generator().manual_seed(42) # fixed: the split must not
@@ -510,7 +598,8 @@ def train(args, cfg, rank, local_rank, world):
raw["Mesh_MPJPE"], em["Mesh_MPJPE"] = (
dpw_raw["J12_MPJPE"], dpw_em["J12_MPJPE"])
- log(f"\n📈 epoch {epoch+1}/{args.epochs} | lr {scheduler.get_last_lr()[0]:.2e}")
+ log(f"\n📈 epoch {epoch+1}/{args.epochs} | lr {scheduler.get_last_lr()[0]:.2e}"
+ f" | peak GPU {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB")
log(f" train total {tr['total']:.4f} | 3d {tr['3d']:.4f} | "
f"2d {tr['2d']:.4f} | simcc {tr['simcc']:.3f}")
log(f" valRAW loss {raw['loss']:.4f} | PA-MPJPE {raw['Mesh_PA_MPJPE']:.1f} | "
@@ -632,6 +721,17 @@ def parse_args():
help="Skip the timm download. Compute nodes have no internet, "
"so run --warm-cache once on a login node instead.")
p.add_argument("--log-every", type=int, default=100)
+ p.add_argument("--preset", choices=["baseline", "v2"], default="baseline",
+ help="'v2' enables the wave-1 bundle: truncation, occlusion "
+ "and JPEG augmentation with the SimCC-mask and CLIFF "
+ "prerequisites, plus absolute-pose supervision. "
+ "'baseline' reproduces the pre-v2 trainer exactly.")
+ p.add_argument("--w_reproj", type=float, default=None)
+ p.add_argument("--index-cache", default=None,
+ help="Where to cache the (image, npz) pair index. "
+ "Default: <data_root>/.pair_index_v2.pkl")
+ p.add_argument("--rebuild-index", action="store_true",
+ help="Rescan the corpus even if the cache looks current.")
p.add_argument("--warm-cache", action="store_true",
help="Instantiate the backbone to populate $TORCH_HOME / "
"$HF_HOME, then exit. Run on a login or prepost node.")
@@ -657,9 +757,42 @@ def build_cfg(args):
if args.w_simcc is not None: cfg.w_simcc = args.w_simcc
if args.weight_decay is not None: cfg.weight_decay = args.weight_decay
if args.ema_decay is not None: cfg.ema_decay = args.ema_decay
+
+ if args.preset == "v2":
+ apply_v2_preset(cfg)
+ # An explicit --w_reproj still wins over the preset.
+ if args.w_reproj is not None: cfg.w_reproj = args.w_reproj
return cfg
+def apply_v2_preset(cfg) -> None:
+ """Wave-1 bundle: augmentation (MeTRAbs items 1/2/4) + absolute-pose (item 3).
+
+ Deliberately one preset rather than four independent flags. MeTRAbs measures
+ the augmentations cumulatively (Table 7: 58.0 -> 52.8 -> 49.3), so splitting
+ them into separate runs would spend three 3-seed sweeps reproducing an
+ ablation that already exists in the literature. Absolute-pose supervision
+ rides along because it touches disjoint machinery -- the reprojection and
+ translation terms, not the input pipeline -- so a regression stays
+ attributable from the per-term loss breakdown in history.jsonl.
+ """
+ # --- item 1: truncation -------------------------------------------------
+ cfg.geom_scale_max = 2.0 # retains the central quarter by area
+ cfg.geom_trans = 0.20 # off-centre, so truncation is not symmetric
+ cfg.cliff_follows_aug = True # prerequisite B
+ cfg.mask_oob_2d = True # prerequisite A
+ # --- item 2: occlusion --------------------------------------------------
+ cfg.occl_p = 0.7 # MeTRAbs Table 7
+ cfg.occl_scale = (0.02, 0.30)
+ # --- item 4: JPEG -------------------------------------------------------
+ cfg.jpeg_p = 0.3
+ cfg.jpeg_quality = (30, 90)
+ # --- item 3: absolute pose ---------------------------------------------
+ cfg.w_reproj = 0.5 # was 0.01, i.e. effectively off
+ cfg.reproj_all_samples = True # the flipped half now contributes
+ cfg.cam_loss = "euclid"
+
+
def main():
args = parse_args()
@@ -670,6 +803,10 @@ def main():
print(f"cached {cfg.backbone}: {sum(p.numel() for p in m.parameters())/1e6:.2f} M "
f"params under TORCH_HOME={os.environ.get('TORCH_HOME')} "
f"HF_HOME={os.environ.get('HF_HOME')}")
+ # The pair index too: this is the other thing a compute node should
+ # never have to build, and a prepost node has the walltime for it.
+ build_pair_index(cfg.data_root, args.index_cache,
+ rebuild=args.rebuild_index)
return
rank, local_rank, world = setup_distributed()
@@ -705,6 +842,11 @@ def main():
log(f"out {cfg.log_dir}")
log(f"backbone {cfg.backbone} | epochs {args.epochs} | "
f"batch {cfg.batch_size}/gpu | lr {cfg.lr:.2e} ({args.lr_scaling} x{world})")
+ log(f"preset {args.preset}"
+ + (f" | trunc<={cfg.geom_scale_max} trans={cfg.geom_trans} "
+ f"occl_p={cfg.occl_p} jpeg_p={cfg.jpeg_p} "
+ f"w_reproj={cfg.w_reproj} cam={cfg.cam_loss}"
+ if args.preset != "baseline" else ""))
log(f"seed {args.seed} | mix {args.mix} | "
f"samples/epoch {args.samples_per_epoch:,}")
log("=" * 66)
diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.py
index 2df05d2..5e71719 100644
--- a/instanthmr_distill_train/train_distill_mhr_only.py
+++ b/instanthmr_distill_train/train_distill_mhr_only.py
@@ -44,6 +44,7 @@ import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import argparse
+import io
import math
import random
import gc
@@ -149,6 +150,22 @@ class DistillConfig:
geom_trans: float = 0.08
geom_flip_p: float = 0.5
+ # --- v2 knobs. Every default below reproduces the pre-v2 behaviour bit for
+ # --- bit, so a baseline run started before these existed is unaffected.
+ geom_scale_max: float | None = None # zoom-in ceiling; None => symmetric
+ # U(1-range, 1+range) as before. 2.0 is
+ # MeTRAbs's quarter-area truncation bound.
+ cliff_follows_aug: bool = False # CLIFF cx/cy/b_scale track the augmented
+ # crop box, as a real detector box would
+ occl_p: float = 0.2 # RandomErasing probability (MeTRAbs: 0.7)
+ occl_scale: tuple = (0.02, 0.15)
+ jpeg_p: float = 0.0 # random JPEG re-encode probability
+ jpeg_quality: tuple = (30, 90)
+ mask_oob_2d: bool = False # mask 2D targets outside kp2d_range
+ # instead of clamping them to the edge
+ reproj_all_samples: bool = False # include flipped samples in loss_reproj
+ cam_loss: str = "mse" # "mse" | "euclid" (unsquared L2 on xyz)
+
kp2d_bins: int = 96
kp2d_range: float = 1.5
@@ -236,6 +253,28 @@ class DistillConfig:
# ============================================================
# Cell 3 — Dataset and Data Augmentation
# ============================================================
+class RandomJPEG:
+ """Re-encode through JPEG at a random quality.
+
+ The one item on NLF's augmentation list that this pipeline lacked, and the
+ most deployment-relevant of them: phone and tablet camera stacks emit JPEG,
+ often aggressively compressed, while training crops are decoded from disk at
+ whatever quality the source dataset happened to store. Blocking artefacts
+ and chroma subsampling are a distribution the model otherwise never sees.
+ """
+
+ def __init__(self, p: float = 0.0, qmin: int = 30, qmax: int = 90):
+ self.p, self.qmin, self.qmax = p, qmin, qmax
+
+ def __call__(self, img):
+ if self.p <= 0.0 or random.random() >= self.p:
+ return img
+ buf = io.BytesIO()
+ img.save(buf, format="JPEG", quality=random.randint(self.qmin, self.qmax))
+ buf.seek(0)
+ return Image.open(buf).convert("RGB")
+
+
class RandomPixelate:
def __init__(self, p=0.2, min_res=32, max_res=112):
self.p = p
@@ -255,9 +294,14 @@ class SAM3DStudentDataset(Dataset):
def __init__(self, data_root: str,
image_size: int = 224, max_images: int | None = None,
augment: bool = True, per_dataset_caps: dict | None = None,
+ pairs: list | None = None,
geom_p: float = 0.8, geom_rot_deg: float = 30.0,
geom_scale_range: float = 0.25, geom_trans: float = 0.08,
- geom_flip_p: float = 0.5):
+ geom_flip_p: float = 0.5,
+ geom_scale_max: float | None = None,
+ cliff_follows_aug: bool = False,
+ occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),
+ jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90)):
super().__init__()
self.image_size = image_size
self.data_root = Path(data_root)
@@ -267,6 +311,8 @@ class SAM3DStudentDataset(Dataset):
self.geom_scale_range = geom_scale_range
self.geom_trans = geom_trans
self.geom_flip_p = geom_flip_p
+ self.geom_scale_max = geom_scale_max
+ self.cliff_follows_aug = cliff_follows_aug
tfm_list = [transforms.Resize((image_size, image_size))]
@@ -275,6 +321,7 @@ class SAM3DStudentDataset(Dataset):
transforms.RandomApply([transforms.ColorJitter(
brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),
RandomPixelate(p=0.2, min_res=48, max_res=112),
+ RandomJPEG(p=jpeg_p, qmin=jpeg_quality[0], qmax=jpeg_quality[1]),
transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),
])
@@ -285,10 +332,23 @@ class SAM3DStudentDataset(Dataset):
])
if augment:
- tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))
+ # MeTRAbs Table 7 puts synthetic occlusion at p=0.7 and credits it
+ # with 52.8 -> 49.3 mm on top of an already-strong colour pipeline.
+ tfm_list.append(transforms.RandomErasing(p=occl_p, scale=tuple(occl_scale)))
self.transform = transforms.Compose(tfm_list)
+ # A prebuilt index short-circuits the filesystem scan below. That scan
+ # is one glob plus a stat() per crop -- ~7.5 M metadata operations on a
+ # 3.7 M-crop corpus -- and on a parallel filesystem it dominates job
+ # startup, so having every DDP rank redo it is the largest fixed cost of
+ # a cluster run. See build_pair_index() in train_distill_jz.py.
+ if pairs is not None:
+ self.pairs = list(pairs)
+ print(f" Prebuilt index: {len(self.pairs)} (image, npz) pairs "
+ f"(augment={augment}).")
+ return
+
self.pairs = []
dataset_dirs = self._find_dataset_dirs(self.data_root)
if not dataset_dirs:
@@ -407,7 +467,16 @@ class SAM3DStudentDataset(Dataset):
M_flip = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
angle = random.uniform(-self.geom_rot_deg, self.geom_rot_deg)
- scale = random.uniform(1.0 - self.geom_scale_range, 1.0 + self.geom_scale_range)
+ # Asymmetric by design. bbox_square is exactly 1.2x the tight box on
+ # every split, so the body fills 83.3% of the crop and the old
+ # symmetric ceiling of 1.25 shaved ~2% off each side -- it
+ # essentially never truncated. A ceiling of 2.0 retains the central
+ # quarter by area, which is MeTRAbs's bound. Zoom-OUT is still
+ # capped at 1-geom_scale_range because there are no pixels outside
+ # bbox_square; warpAffine would fill the margin with black.
+ scale_hi = (1.0 + self.geom_scale_range if self.geom_scale_max is None
+ else self.geom_scale_max)
+ scale = random.uniform(1.0 - self.geom_scale_range, scale_hi)
dx = random.uniform(-self.geom_trans, self.geom_trans)
dy = random.uniform(-self.geom_trans, self.geom_trans)
M_rst = cv2.getRotationMatrix2D((W / 2.0, H / 2.0), angle, scale)
@@ -451,6 +520,19 @@ class SAM3DStudentDataset(Dataset):
dxy = R_v.astype(np.float64) @ dxy
cx, cy = orig_w / 2.0 + dxy[0], orig_h / 2.0 + dxy[1]
+ # CLIFF conditioning describes the box this crop came from. The flip
+ # and rotation above already move it; dx/dy/scale did not, which was
+ # harmless at +/-0.08 and 1.25 but not at truncation strength. A
+ # crop-space shift of dx corresponds to the box moving
+ # dx * crop_w / 2 original pixels, and a zoom of s shrinks the
+ # visible box by s -- which is exactly what an off-centre or badly
+ # sized detector box looks like at deployment.
+ if self.cliff_follows_aug:
+ crop_w = float(sq_bbox[2] - sq_bbox[0])
+ cx -= dx * crop_w / 2.0
+ cy -= dy * crop_w / 2.0
+ b_scale = b_scale / scale
+
image = self.transform(img_pil)
cx_norm = 2.0 * (cx / orig_w) - 1.0
@@ -479,7 +561,11 @@ class SAM3DStudentDataset(Dataset):
def build_dataloaders(cfg):
geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,
geom_scale_range=cfg.geom_scale_range,
- geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)
+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,
+ geom_scale_max=cfg.geom_scale_max,
+ cliff_follows_aug=cfg.cliff_follows_aug,
+ occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)
full_dataset = SAM3DStudentDataset(
cfg.data_root,
augment=cfg.augment,
@@ -811,7 +897,15 @@ class DistillationLoss(nn.Module):
losses['loss_pose'] = self.m_smooth_l1(pred_pose, tgt_pose, m_ident) * self.cfg.w_pose
losses['loss_scale'] = self.m_mse(pred_scale, tgt_scale, m_noflip) * self.cfg.w_scale
losses['loss_shape'] = self.m_mse(preds["shape_params"], targets["shape_params"], m_noflip) * self.cfg.w_shape
- losses['loss_cam'] = self.mse(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam
+ # NLF applies its translation loss as an unsquared Euclidean norm over the
+ # xyz triple rather than elementwise MSE: MSE lets one badly-scaled axis
+ # (in practice depth, which is an order of magnitude larger than x/y)
+ # dominate the gradient of all three.
+ if getattr(self.cfg, "cam_loss", "mse") == "euclid":
+ e_cam = torch.linalg.norm(preds["cam_trans"] - targets["cam_trans"], dim=-1)
+ losses['loss_cam'] = e_cam.mean() * self.cfg.w_cam
+ else:
+ losses['loss_cam'] = self.mse(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam
pred_mhr_joints = self.mhr_module.get_joints(preds["mhr_params"], preds["shape_params"])[..., :3]
with torch.no_grad():
@@ -852,15 +946,31 @@ class DistillationLoss(nn.Module):
if w_kp is None:
w_kp = torch.ones_like(tgt_2d[..., 0])
+ # Truncation augmentation pushes keypoints outside the crop. Both 2D
+ # terms below are BOUNDED -- SimCC spans +/-kp2d_range and pred_2d is a
+ # soft-argmax over those same bins -- so a target at 1.67 cannot be
+ # reached. Clamping it to 1.5 asserts the joint is exactly at the crop
+ # edge, which is false, and leaves a permanent saturating gradient at
+ # w_keypoints2d = 10. Mask instead: the 3D loss still supervises those
+ # joints, which is the whole MeTRo argument for predicting them at all.
+ #
+ # Do NOT widen kp2d_range instead: bin_w = 2*range/(bins-1), so at 96
+ # bins that trades 3.5 px bins for 5.9 px, degrading every in-frame
+ # joint to accommodate the out-of-frame ones.
+ w_kp_2d = w_kp
+ if getattr(self.cfg, "mask_oob_2d", False):
+ in_crop = (tgt_2d.abs() <= self.cfg.kp2d_range).all(-1).to(w_kp.dtype)
+ w_kp_2d = w_kp * in_crop
+
# 2D head — training-only auxiliary task, and the ONLY supervision that
# survives a horizontal flip in full (targets["joints_2d"] is mirrored
# AND FLIP_PERM-permuted by the dataset, so it stays self-consistent —
# and so is joints_2d_vis).
e2d = F.smooth_l1_loss(pred_2d[..., :2], tgt_2d, reduction='none').mean(-1)
- losses['loss_2d_native'] = self._kp_weighted(e2d, w_kp) * self.cfg.w_keypoints2d
+ losses['loss_2d_native'] = self._kp_weighted(e2d, w_kp_2d) * self.cfg.w_keypoints2d
if "joints_2d_logits" in preds:
losses['loss_2d_simcc'] = self._simcc_ce(
- preds["joints_2d_logits"], tgt_2d, w_kp) * self.cfg.w_simcc
+ preds["joints_2d_logits"], tgt_2d, w_kp_2d) * self.cfg.w_simcc
# Main 3D loss, now on the MHR-derived keypoints. Deliberately UNMASKED:
# targets["joints_3d"] is mirrored + permuted under flip, and the MHR rest
@@ -883,14 +993,32 @@ class DistillationLoss(nn.Module):
cam_cr = _counter_rot_cam(preds["cam_trans"])
# Single re-projection (replaces loss_reproj_native + loss_reproj_mhr):
- # one geometry, all 70 keypoints. Still m_noflip — the re-projection
- # path undoes the rotation but not the mirror, so flipped samples would
- # need an extra FLIP_PERM + cx mirror that is not worth the complexity
- # for a 0.01-weight auxiliary term.
- pred_reproj = self._project_3d_to_norm_2d(_counter_rot_pts(pred_kp3d), cam_cr, targets)
+ # one geometry, all 70 keypoints.
+ #
+ # The projection runs in the ORIGINAL image frame (real focal, image
+ # centre, un-augmented bbox_square) and only then applies aug_M, which
+ # carries the mirror. So a flipped sample has to be un-mirrored first,
+ # or it gets mirrored twice. The dataset applies flip THEN rotation, so
+ # the inverse is counter-rotate THEN negate x -- on the keypoints and on
+ # cam_trans alike, both of which the dataset negated. Index order needs
+ # no fixing: pred_kp3d is already in FLIP_PERM order and aug_M maps it to
+ # the same order as tgt_2d.
+ #
+ # Previously this whole term was masked to m_noflip, which was defensible
+ # at w_reproj = 0.01 and is not once the weight matters: half the batch
+ # would contribute nothing to the only loss tying 3D to pixels.
+ pts_cr, cam_cr_r = _counter_rot_pts(pred_kp3d), cam_cr
+ m_reproj = m_noflip
+ if getattr(self.cfg, "reproj_all_samples", False):
+ sgn = (1.0 - 2.0 * aug_flip).view(-1, 1) # +1 / -1
+ pts_cr = torch.stack([pts_cr[..., 0] * sgn, pts_cr[..., 1],
+ pts_cr[..., 2]], dim=-1)
+ cam_cr_r = torch.cat([cam_cr[:, 0:1] * sgn, cam_cr[:, 1:]], dim=1)
+ m_reproj = ones
+ pred_reproj = self._project_3d_to_norm_2d(pts_cr, cam_cr_r, targets)
e_reproj = (pred_reproj - tgt_2d).abs().mean(-1)
losses['loss_reproj'] = self._kp_weighted(
- e_reproj, w_kp, m_noflip) * self.cfg.w_reproj * self.fk_scale
+ e_reproj, w_kp, m_reproj) * self.cfg.w_reproj * self.fk_scale
losses['total_loss'] = sum(losses.values())
return losses