Python
feat: focal-aware CLIFF conditioning (--cliff-focal), mandatory for new runs
mohamdevf6f6b2b66c0b
diff --git a/CLAUDE.md b/CLAUDE.mdindex 888c073..767b58a 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -136,6 +136,24 @@ float64 the unregularised fit reaches landmarks through ±2000 weights onfingers; `RCOND` in `eval_3dpw.fit_adapter` truncates it to a rank-24 map withweights under 1 at no measurable cost.+**The CLIFF conditioning is computed in three places and they must agree.**+`SAM3DStudentDataset.__getitem__`, `val3dpw._preprocess` and+`instanthmr.inference.InstantHMR._preprocess` each build the 3-vector+separately. Feeding a model trained on one variant with the other raises+nothing — it silently mis-places the person in depth. Since 2026-09-06 the+perspective-correct variant (`--cliff-focal`, config `cliff_focal`) is+mandatory for new runs, because the rebuilt corpus carries real per-image+focals spanning `f/diag` 0.6-1.8 where the pixel form is ambiguous; the old+corpus was constant at 1.000. See `instanthmr_distill_train/README.md`.++**PA-MPJPE is blind to half of what a demo shows.** Procrustes removes+translation, rotation and scale, so `cam_trans` drift, body-size error and+temporal jitter are all invisible to it — and to root-relative MPJPE for the+first two. A change can be worth shipping on stability and move the benchmark+by 0.1 mm. Measure jitter directly (frame-to-frame acceleration on 3DPW's+consecutive frames, and the variance of rigid bone lengths within a person+track, which cannot change and so is pure noise).+**Seed spread is wide.** Observed 70-keypoint PA-MPJPE varies 34–49 mm acrossseeds, which is larger than several of the changes on the roadmap. A single-runcomparison proves nothing; use 2–3 seeds.diff --git a/benchmark/README.md b/benchmark/README.mdindex 4161df4..2fbbe62 100644--- a/benchmark/README.md+++ b/benchmark/README.md@@ -177,6 +177,36 @@ Jean Zay, whose corpus is coco/mpii/aic/sa1b/harmony4d with `3dpw_train`annotations-only, so all 24 test sequences are unseen. The local `data/` corpusis **not** clean — see the contamination section.+### What these metrics cannot see++Both headline metrics are alignment-invariant, and that hides exactly the+failure a demo shows first. PA-MPJPE removes translation, rotation **and**+scale; root-relative MPJPE removes translation. So camera-translation drift,+systematic body-size error and frame-to-frame jitter are all invisible here —+a change can be worth shipping on stability and move PA-MPJPE by 0.1 mm.++3DPW is video, so measure stability directly on consecutive frames:++- **rigid bone lengths** within a person track. A femur cannot change length,+ so its variance is pure prediction noise. GT floor: 1.06%.+- **`cam_trans` acceleration**, `|t[i+1] - 2t[i] + t[i-1]|`. GT floor: 7.0+ mm/frame².++Measured on 3DPW test (35,463 person-frames, J14 + teacher adapter):++| checkpoint | PA | MPJPE | bone-len CV | cam accel | (z only) |+|---|---|---|---|---|---|+| `distill_mhr_only_ckpt90` (old corpus, *contaminated*) | 41.14 | 68.85 | 1.65% | 148.1 | 144.3 |+| b2_s0 | 44.84 | 72.69 | 1.96% | 234.4 | 229.9 |+| b3_s1 | 41.83 | 69.03 | 1.85% | 219.6 | 215.9 |+| v3_s1 | 43.23 | 70.59 | 1.79% | 224.6 | 222.1 |+| ground truth | — | — | 1.06% | 7.0 | |++Every model is 21-33x jitterier than ground truth, and the rebuilt corpus made+it ~50% worse — traced to the focal spread and addressed by `--cliff-focal`+(see `instanthmr_distill_train/README.md`). The legacy row is optimistic on+both axes: it trained on all 60 3DPW sequences.+## Protocols, stated preciselyWrite these into the paper — they are the details a reviewer will ask about.diff --git a/benchmark/benchlib/threedpw.py b/benchmark/benchlib/threedpw.pyindex 8ecd1c1..b184e2d 100644--- a/benchmark/benchlib/threedpw.py+++ b/benchmark/benchlib/threedpw.py@@ -149,6 +149,9 @@ def build_samples(sequence_dir: str | Path, image_root: str | Path,uv=uv.astype(np.float32),sequence=name, person=pid, frame=f,gender=str(seq["genders"][pid]),+ # fy, for the focal-aware CLIFF conditioning. 3DPW is+ # calibrated, so this is the real focal, not an estimate.+ focal=float(K[1, 1]),))gts.append(gt_w[f] @ T[:3, :3].T + T[:3, 3])diff --git a/instanthmr/inference.py b/instanthmr/inference.pyindex 0f6fdf2..7b80113 100644--- a/instanthmr/inference.py+++ b/instanthmr/inference.py@@ -119,6 +119,7 @@ class InstantHMR:device: str = "cuda",providers: Optional[list[str]] = None,mhr: object | None = None,+ focal: float | None = None,):import onnxruntime as ort@@ -161,6 +162,10 @@ class InstantHMR:self._out_names = out_namesself.has_joints_3d_head = "joints_3d" in out_names or len(out_names) >= 5self._mhr = mhr+ # None -> the image-normalised CLIFF conditioning the model was trained+ # with by default. Set it to the camera's focal length in pixels for a+ # checkpoint trained with --cliff-focal.+ self.focal = focal@propertydef derives_joints_3d(self) -> bool:@@ -381,8 +386,8 @@ class InstantHMR:# Preprocessing# ------------------------------------------------------------------- @staticmethoddef _preprocess(+ self,image_rgb: np.ndarray,bbox: np.ndarray,h: int,@@ -395,11 +400,20 @@ class InstantHMR:cx = (x1 + x2) / 2.0cy = (y1 + y2) / 2.0- # CLIFF conditioning vector (full-frame coords)- cx_norm = 2.0 * (cx / w) - 1.0- cy_norm = 2.0 * (cy / h) - 1.0- b_scale = max(bw, bh) / max(w, h)- cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)+ # CLIFF conditioning vector (full-frame coords). Must match the variant+ # the checkpoint was TRAINED with -- see `cliff_focal` in+ # instanthmr_distill_train/train_distill_mhr_only.py. A model trained+ # with the focal-aware form and run with the pixel form (or the other+ # way round) does not error; it silently mis-places the person in depth.+ if self.focal is not None:+ cliff_cond = np.array([math.atan((cx - w / 2.0) / self.focal),+ math.atan((cy - h / 2.0) / self.focal),+ max(bw, bh) / self.focal], dtype=np.float32)+ else:+ cx_norm = 2.0 * (cx / w) - 1.0+ cy_norm = 2.0 * (cy / h) - 1.0+ b_scale = max(bw, bh) / max(w, h)+ cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)sq_size = max(bw, bh) * CROP_EXPANDhalf = sq_size / 2.0diff --git a/instanthmr_distill_train/README.md b/instanthmr_distill_train/README.mdindex 2781d64..73002d9 100644--- a/instanthmr_distill_train/README.md+++ b/instanthmr_distill_train/README.md@@ -112,6 +112,55 @@ must keep the metric it started on: `--val-3dpw-gt jointpositions`, or`DPW_GT=jointpositions` for `52_train_ddp.slurm`. `run_config.json` recordswhich was used.+## `--cliff-focal` (added 2026-09-06, mandatory)++**Use it on every new run.** It changes the CLIFF conditioning from+image-normalised pixels to a perspective-correct form:++```+[ atan((cx - W/2) / f), atan((cy - H/2) / f), box_px / f ]+```++— where the box sits in the field of view, and how large it is *angularly*.+`f` is `cam_focal_length[1]` (fy), a single scalar because the rotation+augmentation mixes the two image axes and `fx != fy` on the calibrated splits.++**Why it is not optional any more.** Under a *fixed* focal the two forms are+interchangeable, and the corpus this trainer was written for had exactly that:+every crop from `tools/annotate_dataset.py` carries the synthetic+`f = sqrt(H^2+W^2)`, i.e. `f/diag = 1.000` with **zero** spread. The rebuilt+corpus carries the datasets' own focals instead:++| split | f / diag | sd |+|---|---|---|+| `sam3d_distill_mix` (old corpus) | 1.000 | **0.000** |+| `sam3d_gt_aic` | 1.184 | 0.371 |+| `sam3d_gt_coco` | 1.049 | 0.344 |+| `sam3d_gt_mpii` | 0.961 | 0.266 |+| `sam3d_gt_harmony4d` | 0.285 | 0.010 |++With that spread the pixel form is **ambiguous**: of 4,000 COCO crops, 124 have+a near-twin under it (conditioning distance < 0.005) whose true depths differ by+**1.34 m**. The network can only predict the conditional mean, and the residual+leaves as unstable `cam_trans` and body scale. Measured on 3DPW test, models+trained on the new corpus jitter at 220-234 mm/frame² of translation+acceleration (98% of it in depth) against 148 for one trained on the+constant-focal corpus, and a ground-truth floor of 7.++**PA-MPJPE cannot see any of this** — Procrustes removes translation and scale,+and the metric moves by ~0.1 mm either way. Judge the change on `cam_trans`+stability and bone-length variance, never on the benchmark.++**Three places compute this vector and all three must agree**:+`SAM3DStudentDataset.__getitem__`, `val3dpw._preprocess` (which takes the real+`fy` from 3DPW's `cam_intrinsics`), and `instanthmr.inference.InstantHMR`+(pass `focal=` for a checkpoint trained with the flag). Mismatching them raises+nothing; it silently mis-places the person in depth. `run_config.json` records+`cliff_focal`.++The default is `False` and is bit-identical to the pre-flag trainer, verified+over augmented samples tensor by tensor.+## The pair index`SAM3DStudentDataset` enumerates the corpus with one `glob` plus a `stat()` perdiff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.pyindex 244940f..66555c7 100755--- a/instanthmr_distill_train/train_distill_jz.py+++ b/instanthmr_distill_train/train_distill_jz.py@@ -332,6 +332,7 @@ def build_jz_loaders(cfg, args, rank: int, world: int):geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,geom_scale_max=cfg.geom_scale_max,cliff_follows_aug=cfg.cliff_follows_aug,+ cliff_focal=cfg.cliff_focal,occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)@@ -492,7 +493,7 @@ def train(args, cfg, rank, local_rank, world):from val3dpw import ThreeDPWValSetdpw = ThreeDPWValSet(args.val_3dpw, args.val_3dpw_images,split=args.val_3dpw_split, stride=args.val_3dpw_stride,- gt=args.val_3dpw_gt)+ gt=args.val_3dpw_gt, cliff_focal=cfg.cliff_focal)# Stride-shard across ranks; the metric is all-reduced afterwards.dpw_loader = torch.utils.data.DataLoader(torch.utils.data.Subset(dpw, list(range(rank, len(dpw), world))),@@ -510,7 +511,7 @@ def train(args, cfg, rank, local_rank, world):if args.val_3dpw_test:dpw_t = ThreeDPWValSet(args.val_3dpw, args.val_3dpw_images,split="test", stride=args.val_3dpw_test_stride,- gt=args.val_3dpw_gt)+ gt=args.val_3dpw_gt, cliff_focal=cfg.cliff_focal)dpw_test_loader = torch.utils.data.DataLoader(torch.utils.data.Subset(dpw_t, list(range(rank, len(dpw_t), world))),batch_size=cfg.batch_size, shuffle=False, drop_last=False,@@ -631,6 +632,7 @@ def train(args, cfg, rank, local_rank, world):"pose_split": cfg.pose_split, "pose_beta": cfg.pose_beta,"finger_weight": cfg.finger_weight,"val_3dpw_gt": args.val_3dpw_gt if args.val_3dpw else None,+ "cliff_focal": cfg.cliff_focal,"started": time.strftime("%Y-%m-%d %H:%M")})for epoch in range(start_epoch, args.epochs):@@ -880,6 +882,14 @@ def parse_args():p.add_argument("--val-3dpw-test-stride", type=int, default=5)p.add_argument("--val-3dpw-stride", type=int, default=5,help="3DPW is 30 fps; neighbouring frames are near-duplicates.")+ p.add_argument("--cliff-focal", action="store_true",+ help="Perspective-correct CLIFF conditioning: angles off the "+ "optical axis and angular box size instead of "+ "image-normalised pixels. Required once the corpus "+ "carries real per-image focals (f/diag spans 0.6-1.8), "+ "where the pixel form is ambiguous and cam_trans and "+ "body scale come out unstable. Applies to training AND "+ "to the 3DPW validation crops; the two must agree.")p.add_argument("--val-3dpw-gt", default="h36m",choices=("h36m", "jointpositions"),help="h36m (default) is the published protocol and needs "@@ -963,6 +973,9 @@ def build_cfg(args):# An explicit --w_reproj / --w_keypoints3d still wins over both.if args.w_reproj is not None: cfg.w_reproj = args.w_reprojif args.w_keypoints3d is not None: cfg.w_keypoints3d = args.w_keypoints3d+ # Orthogonal to both presets: it changes what the network is TOLD, not what+ # it is trained on or scored against.+ cfg.cliff_focal = args.cliff_focalreturn cfgdiff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.pyindex 1d568eb..3fff58e 100644--- a/instanthmr_distill_train/train_distill_mhr_only.py+++ b/instanthmr_distill_train/train_distill_mhr_only.py@@ -182,6 +182,12 @@ class DistillConfig:# MeTRAbs's quarter-area truncation bound.cliff_follows_aug: bool = False # CLIFF cx/cy/b_scale track the augmented# crop box, as a real detector box would+ cliff_focal: bool = False # perspective-correct CLIFF conditioning:+ # angles off the optical axis and angular+ # box size, instead of image-normalised+ # pixels. Needs cam_focal_length, which+ # every npz carries. See the note in+ # SAM3DStudentDataset.__getitem__.occl_p: float = 0.2 # RandomErasing probability (MeTRAbs: 0.7)occl_scale: tuple = (0.02, 0.15)jpeg_p: float = 0.0 # random JPEG re-encode probability@@ -395,6 +401,7 @@ class SAM3DStudentDataset(Dataset):geom_flip_p: float = 0.5,geom_scale_max: float | None = None,cliff_follows_aug: bool = False,+ cliff_focal: bool = False,occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90)):super().__init__()@@ -408,6 +415,7 @@ class SAM3DStudentDataset(Dataset):self.geom_flip_p = geom_flip_pself.geom_scale_max = geom_scale_maxself.cliff_follows_aug = cliff_follows_aug+ self.cliff_focal = cliff_focaltfm_list = [transforms.Resize((image_size, image_size))]@@ -630,9 +638,36 @@ class SAM3DStudentDataset(Dataset):image = self.transform(img_pil)- cx_norm = 2.0 * (cx / orig_w) - 1.0- cy_norm = 2.0 * (cy / orig_h) - 1.0- cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)+ if self.cliff_focal:+ # Where the box sits in the FIELD OF VIEW, and how large it is+ # ANGULARLY -- not in image-normalised pixels.+ #+ # Under a fixed focal the two are interchangeable, and the corpus+ # this trainer was written for had exactly that: every crop from+ # tools/annotate_dataset.py carries the synthetic f = sqrt(H^2+W^2),+ # i.e. f/diag = 1.000 with zero spread. The rebuilt corpus carries+ # the datasets' own focals instead, and those span f/diag 0.6-1.8 on+ # coco/aic/mpii (sd ~0.35) with harmony4d off at 0.285. There the+ # pixel version is ambiguous: two crops with identical conditioning+ # correspond to different depths, the network can only predict the+ # conditional mean, and the residual leaves as unstable cam_trans+ # and body scale -- measured on 3DPW test as 220 mm/frame^2 of+ # translation jitter (98% of it in depth) against 148 for a model+ # trained on the constant-focal corpus.+ #+ # One scalar focal (fy) rather than fx/fy, because the rotation+ # augmentation mixes the two axes and fx != fy on the calibrated+ # splits.+ f = float(ann["cam_focal_length"][1])+ b_px = b_scale * max(orig_w, orig_h)+ cliff_cond = torch.tensor(+ [math.atan((cx - orig_w / 2.0) / f),+ math.atan((cy - orig_h / 2.0) / f),+ b_px / f], dtype=torch.float32)+ else:+ cx_norm = 2.0 * (cx / orig_w) - 1.0+ cy_norm = 2.0 * (cy / orig_h) - 1.0+ cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)return {"image": image,@@ -659,6 +694,7 @@ def build_dataloaders(cfg):geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,geom_scale_max=cfg.geom_scale_max,cliff_follows_aug=cfg.cliff_follows_aug,+ cliff_focal=cfg.cliff_focal,occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)full_dataset = SAM3DStudentDataset(diff --git a/instanthmr_distill_train/val3dpw.py b/instanthmr_distill_train/val3dpw.pyindex 08eb332..ebaed1a 100644--- a/instanthmr_distill_train/val3dpw.py+++ b/instanthmr_distill_train/val3dpw.py@@ -61,7 +61,8 @@ IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)MM = 1000.0-def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int):+def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int,+ focal: float | None = None):"""Square 1.2x crop + ImageNet normalise + CLIFF conditioning.A transcription of ``instanthmr.inference.InstantHMR._preprocess``. It is@@ -74,9 +75,17 @@ def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int):bw, bh = x2 - x1, y2 - y1cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0- cliff = np.array([2.0 * (cx / w) - 1.0,- 2.0 * (cy / h) - 1.0,- max(bw, bh) / max(w, h)], dtype=np.float32)+ # Must match SAM3DStudentDataset.__getitem__ exactly, including which+ # variant. Feeding a model trained on one and evaluated on the other is a+ # silent distribution shift, not an error.+ if focal is not None:+ cliff = np.array([math.atan((cx - w / 2.0) / focal),+ math.atan((cy - h / 2.0) / focal),+ max(bw, bh) / focal], dtype=np.float32)+ else:+ cliff = np.array([2.0 * (cx / w) - 1.0,+ 2.0 * (cy / h) - 1.0,+ max(bw, bh) / max(w, h)], dtype=np.float32)sq = max(bw, bh) * CROP_EXPANDhalf = sq / 2.0@@ -106,12 +115,13 @@ class ThreeDPWValSet(Dataset):def __init__(self, sequence_dir: str, image_root: str,split: str = "validation", stride: int = 5,bbox_scale: float = 1.2, gt: str = "h36m",- gt_dir: str | None = None):+ gt_dir: str | None = None, cliff_focal: bool = False):self.samples, self.gt = P.build_samples(sequence_dir, image_root, split=split,stride=stride, bbox_scale=bbox_scale, gt=gt, gt_dir=gt_dir)self.gt = np.asarray(self.gt, dtype=np.float32)self.gt_source = gt+ self.cliff_focal = cliff_focaldef __len__(self) -> int:return len(self.samples)@@ -128,7 +138,8 @@ class ThreeDPWValSet(Dataset):"ok": torch.tensor(0.0)}rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)h, w = rgb.shape[:2]- crop, cliff = _preprocess(rgb, s["bbox"], h, w)+ crop, cliff = _preprocess(rgb, s["bbox"], h, w,+ s["focal"] if self.cliff_focal else None)return {"image": torch.from_numpy(crop),"cliff_cond": torch.from_numpy(cliff),"gt": torch.from_numpy(self.gt[i]),diff --git a/tools/pth_to_onnx.py b/tools/pth_to_onnx.pyindex c4365a8..c4bd018 100644--- a/tools/pth_to_onnx.py+++ b/tools/pth_to_onnx.py@@ -34,9 +34,9 @@ Usagepython tools/pth_to_onnx.py# or point at a specific checkpoint / output- python tools/pth_to_onnx.py \- --ckpt instanthmr_distill_train/runs/distill_mhr_only/distill_mhr_only/best_student_model_ema.pth \- --output models/instanthmr_mhr_only.onnx+ python ./tools/pth_to_onnx.py \+ --ckpt ./instanthmr_distill_train/runs/b2_s0/b2_s0/best_student_model_ema.pth \+ --output ./models/instanthmr_mhr_only.onnx"""import argparseimport importlib