Skip to content
Python

feat: focal-aware CLIFF conditioning (--cliff-focal), mandatory for new runs

mohamdevf6f6b2b66c0b
diff --git a/CLAUDE.md b/CLAUDE.md
index 888c073..767b58a 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -136,6 +136,24 @@ float64 the unregularised fit reaches landmarks through ±2000 weights on
fingers; `RCOND` in `eval_3dpw.fit_adapter` truncates it to a rank-24 map with
weights under 1 at no measurable cost.
+**The CLIFF conditioning is computed in three places and they must agree.**
+`SAM3DStudentDataset.__getitem__`, `val3dpw._preprocess` and
+`instanthmr.inference.InstantHMR._preprocess` each build the 3-vector
+separately. Feeding a model trained on one variant with the other raises
+nothing — it silently mis-places the person in depth. Since 2026-09-06 the
+perspective-correct variant (`--cliff-focal`, config `cliff_focal`) is
+mandatory for new runs, because the rebuilt corpus carries real per-image
+focals spanning `f/diag` 0.6-1.8 where the pixel form is ambiguous; the old
+corpus was constant at 1.000. See `instanthmr_distill_train/README.md`.
+
+**PA-MPJPE is blind to half of what a demo shows.** Procrustes removes
+translation, rotation and scale, so `cam_trans` drift, body-size error and
+temporal jitter are all invisible to it — and to root-relative MPJPE for the
+first two. A change can be worth shipping on stability and move the benchmark
+by 0.1 mm. Measure jitter directly (frame-to-frame acceleration on 3DPW's
+consecutive frames, and the variance of rigid bone lengths within a person
+track, which cannot change and so is pure noise).
+
**Seed spread is wide.** Observed 70-keypoint PA-MPJPE varies 34–49 mm across
seeds, which is larger than several of the changes on the roadmap. A single-run
comparison proves nothing; use 2–3 seeds.
diff --git a/benchmark/README.md b/benchmark/README.md
index 4161df4..2fbbe62 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -177,6 +177,36 @@ Jean Zay, whose corpus is coco/mpii/aic/sa1b/harmony4d with `3dpw_train`
annotations-only, so all 24 test sequences are unseen. The local `data/` corpus
is **not** clean — see the contamination section.
+### What these metrics cannot see
+
+Both headline metrics are alignment-invariant, and that hides exactly the
+failure a demo shows first. PA-MPJPE removes translation, rotation **and**
+scale; root-relative MPJPE removes translation. So camera-translation drift,
+systematic body-size error and frame-to-frame jitter are all invisible here —
+a change can be worth shipping on stability and move PA-MPJPE by 0.1 mm.
+
+3DPW is video, so measure stability directly on consecutive frames:
+
+- **rigid bone lengths** within a person track. A femur cannot change length,
+ so its variance is pure prediction noise. GT floor: 1.06%.
+- **`cam_trans` acceleration**, `|t[i+1] - 2t[i] + t[i-1]|`. GT floor: 7.0
+ mm/frame².
+
+Measured on 3DPW test (35,463 person-frames, J14 + teacher adapter):
+
+| checkpoint | PA | MPJPE | bone-len CV | cam accel | (z only) |
+|---|---|---|---|---|---|
+| `distill_mhr_only_ckpt90` (old corpus, *contaminated*) | 41.14 | 68.85 | 1.65% | 148.1 | 144.3 |
+| b2_s0 | 44.84 | 72.69 | 1.96% | 234.4 | 229.9 |
+| b3_s1 | 41.83 | 69.03 | 1.85% | 219.6 | 215.9 |
+| v3_s1 | 43.23 | 70.59 | 1.79% | 224.6 | 222.1 |
+| ground truth | — | — | 1.06% | 7.0 | |
+
+Every model is 21-33x jitterier than ground truth, and the rebuilt corpus made
+it ~50% worse — traced to the focal spread and addressed by `--cliff-focal`
+(see `instanthmr_distill_train/README.md`). The legacy row is optimistic on
+both axes: it trained on all 60 3DPW sequences.
+
## Protocols, stated precisely
Write these into the paper — they are the details a reviewer will ask about.
diff --git a/benchmark/benchlib/threedpw.py b/benchmark/benchlib/threedpw.py
index 8ecd1c1..b184e2d 100644
--- a/benchmark/benchlib/threedpw.py
+++ b/benchmark/benchlib/threedpw.py
@@ -149,6 +149,9 @@ def build_samples(sequence_dir: str | Path, image_root: str | Path,
uv=uv.astype(np.float32),
sequence=name, person=pid, frame=f,
gender=str(seq["genders"][pid]),
+ # fy, for the focal-aware CLIFF conditioning. 3DPW is
+ # calibrated, so this is the real focal, not an estimate.
+ focal=float(K[1, 1]),
))
gts.append(gt_w[f] @ T[:3, :3].T + T[:3, 3])
diff --git a/instanthmr/inference.py b/instanthmr/inference.py
index 0f6fdf2..7b80113 100644
--- a/instanthmr/inference.py
+++ b/instanthmr/inference.py
@@ -119,6 +119,7 @@ class InstantHMR:
device: str = "cuda",
providers: Optional[list[str]] = None,
mhr: object | None = None,
+ focal: float | None = None,
):
import onnxruntime as ort
@@ -161,6 +162,10 @@ class InstantHMR:
self._out_names = out_names
self.has_joints_3d_head = "joints_3d" in out_names or len(out_names) >= 5
self._mhr = mhr
+ # None -> the image-normalised CLIFF conditioning the model was trained
+ # with by default. Set it to the camera's focal length in pixels for a
+ # checkpoint trained with --cliff-focal.
+ self.focal = focal
@property
def derives_joints_3d(self) -> bool:
@@ -381,8 +386,8 @@ class InstantHMR:
# Preprocessing
# ------------------------------------------------------------------
- @staticmethod
def _preprocess(
+ self,
image_rgb: np.ndarray,
bbox: np.ndarray,
h: int,
@@ -395,11 +400,20 @@ class InstantHMR:
cx = (x1 + x2) / 2.0
cy = (y1 + y2) / 2.0
- # CLIFF conditioning vector (full-frame coords)
- cx_norm = 2.0 * (cx / w) - 1.0
- cy_norm = 2.0 * (cy / h) - 1.0
- b_scale = max(bw, bh) / max(w, h)
- cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)
+ # CLIFF conditioning vector (full-frame coords). Must match the variant
+ # the checkpoint was TRAINED with -- see `cliff_focal` in
+ # instanthmr_distill_train/train_distill_mhr_only.py. A model trained
+ # with the focal-aware form and run with the pixel form (or the other
+ # way round) does not error; it silently mis-places the person in depth.
+ if self.focal is not None:
+ cliff_cond = np.array([math.atan((cx - w / 2.0) / self.focal),
+ math.atan((cy - h / 2.0) / self.focal),
+ max(bw, bh) / self.focal], dtype=np.float32)
+ else:
+ cx_norm = 2.0 * (cx / w) - 1.0
+ cy_norm = 2.0 * (cy / h) - 1.0
+ b_scale = max(bw, bh) / max(w, h)
+ cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)
sq_size = max(bw, bh) * CROP_EXPAND
half = sq_size / 2.0
diff --git a/instanthmr_distill_train/README.md b/instanthmr_distill_train/README.md
index 2781d64..73002d9 100644
--- a/instanthmr_distill_train/README.md
+++ b/instanthmr_distill_train/README.md
@@ -112,6 +112,55 @@ must keep the metric it started on: `--val-3dpw-gt jointpositions`, or
`DPW_GT=jointpositions` for `52_train_ddp.slurm`. `run_config.json` records
which was used.
+## `--cliff-focal` (added 2026-09-06, mandatory)
+
+**Use it on every new run.** It changes the CLIFF conditioning from
+image-normalised pixels to a perspective-correct form:
+
+```
+[ atan((cx - W/2) / f), atan((cy - H/2) / f), box_px / f ]
+```
+
+— where the box sits in the field of view, and how large it is *angularly*.
+`f` is `cam_focal_length[1]` (fy), a single scalar because the rotation
+augmentation mixes the two image axes and `fx != fy` on the calibrated splits.
+
+**Why it is not optional any more.** Under a *fixed* focal the two forms are
+interchangeable, and the corpus this trainer was written for had exactly that:
+every crop from `tools/annotate_dataset.py` carries the synthetic
+`f = sqrt(H^2+W^2)`, i.e. `f/diag = 1.000` with **zero** spread. The rebuilt
+corpus carries the datasets' own focals instead:
+
+| split | f / diag | sd |
+|---|---|---|
+| `sam3d_distill_mix` (old corpus) | 1.000 | **0.000** |
+| `sam3d_gt_aic` | 1.184 | 0.371 |
+| `sam3d_gt_coco` | 1.049 | 0.344 |
+| `sam3d_gt_mpii` | 0.961 | 0.266 |
+| `sam3d_gt_harmony4d` | 0.285 | 0.010 |
+
+With that spread the pixel form is **ambiguous**: of 4,000 COCO crops, 124 have
+a near-twin under it (conditioning distance < 0.005) whose true depths differ by
+**1.34 m**. The network can only predict the conditional mean, and the residual
+leaves as unstable `cam_trans` and body scale. Measured on 3DPW test, models
+trained on the new corpus jitter at 220-234 mm/frame² of translation
+acceleration (98% of it in depth) against 148 for one trained on the
+constant-focal corpus, and a ground-truth floor of 7.
+
+**PA-MPJPE cannot see any of this** — Procrustes removes translation and scale,
+and the metric moves by ~0.1 mm either way. Judge the change on `cam_trans`
+stability and bone-length variance, never on the benchmark.
+
+**Three places compute this vector and all three must agree**:
+`SAM3DStudentDataset.__getitem__`, `val3dpw._preprocess` (which takes the real
+`fy` from 3DPW's `cam_intrinsics`), and `instanthmr.inference.InstantHMR`
+(pass `focal=` for a checkpoint trained with the flag). Mismatching them raises
+nothing; it silently mis-places the person in depth. `run_config.json` records
+`cliff_focal`.
+
+The default is `False` and is bit-identical to the pre-flag trainer, verified
+over augmented samples tensor by tensor.
+
## The pair index
`SAM3DStudentDataset` enumerates the corpus with one `glob` plus a `stat()` per
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.py
index 244940f..66555c7 100755
--- a/instanthmr_distill_train/train_distill_jz.py
+++ b/instanthmr_distill_train/train_distill_jz.py
@@ -332,6 +332,7 @@ def build_jz_loaders(cfg, args, rank: int, world: int):
geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,
geom_scale_max=cfg.geom_scale_max,
cliff_follows_aug=cfg.cliff_follows_aug,
+ cliff_focal=cfg.cliff_focal,
occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)
@@ -492,7 +493,7 @@ def train(args, cfg, rank, local_rank, world):
from val3dpw import ThreeDPWValSet
dpw = ThreeDPWValSet(args.val_3dpw, args.val_3dpw_images,
split=args.val_3dpw_split, stride=args.val_3dpw_stride,
- gt=args.val_3dpw_gt)
+ gt=args.val_3dpw_gt, cliff_focal=cfg.cliff_focal)
# Stride-shard across ranks; the metric is all-reduced afterwards.
dpw_loader = torch.utils.data.DataLoader(
torch.utils.data.Subset(dpw, list(range(rank, len(dpw), world))),
@@ -510,7 +511,7 @@ def train(args, cfg, rank, local_rank, world):
if args.val_3dpw_test:
dpw_t = ThreeDPWValSet(args.val_3dpw, args.val_3dpw_images,
split="test", stride=args.val_3dpw_test_stride,
- gt=args.val_3dpw_gt)
+ gt=args.val_3dpw_gt, cliff_focal=cfg.cliff_focal)
dpw_test_loader = torch.utils.data.DataLoader(
torch.utils.data.Subset(dpw_t, list(range(rank, len(dpw_t), world))),
batch_size=cfg.batch_size, shuffle=False, drop_last=False,
@@ -631,6 +632,7 @@ def train(args, cfg, rank, local_rank, world):
"pose_split": cfg.pose_split, "pose_beta": cfg.pose_beta,
"finger_weight": cfg.finger_weight,
"val_3dpw_gt": args.val_3dpw_gt if args.val_3dpw else None,
+ "cliff_focal": cfg.cliff_focal,
"started": time.strftime("%Y-%m-%d %H:%M")})
for epoch in range(start_epoch, args.epochs):
@@ -880,6 +882,14 @@ def parse_args():
p.add_argument("--val-3dpw-test-stride", type=int, default=5)
p.add_argument("--val-3dpw-stride", type=int, default=5,
help="3DPW is 30 fps; neighbouring frames are near-duplicates.")
+ p.add_argument("--cliff-focal", action="store_true",
+ help="Perspective-correct CLIFF conditioning: angles off the "
+ "optical axis and angular box size instead of "
+ "image-normalised pixels. Required once the corpus "
+ "carries real per-image focals (f/diag spans 0.6-1.8), "
+ "where the pixel form is ambiguous and cam_trans and "
+ "body scale come out unstable. Applies to training AND "
+ "to the 3DPW validation crops; the two must agree.")
p.add_argument("--val-3dpw-gt", default="h36m",
choices=("h36m", "jointpositions"),
help="h36m (default) is the published protocol and needs "
@@ -963,6 +973,9 @@ def build_cfg(args):
# An explicit --w_reproj / --w_keypoints3d still wins over both.
if args.w_reproj is not None: cfg.w_reproj = args.w_reproj
if args.w_keypoints3d is not None: cfg.w_keypoints3d = args.w_keypoints3d
+ # Orthogonal to both presets: it changes what the network is TOLD, not what
+ # it is trained on or scored against.
+ cfg.cliff_focal = args.cliff_focal
return cfg
diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.py
index 1d568eb..3fff58e 100644
--- a/instanthmr_distill_train/train_distill_mhr_only.py
+++ b/instanthmr_distill_train/train_distill_mhr_only.py
@@ -182,6 +182,12 @@ class DistillConfig:
# MeTRAbs's quarter-area truncation bound.
cliff_follows_aug: bool = False # CLIFF cx/cy/b_scale track the augmented
# crop box, as a real detector box would
+ cliff_focal: bool = False # perspective-correct CLIFF conditioning:
+ # angles off the optical axis and angular
+ # box size, instead of image-normalised
+ # pixels. Needs cam_focal_length, which
+ # every npz carries. See the note in
+ # SAM3DStudentDataset.__getitem__.
occl_p: float = 0.2 # RandomErasing probability (MeTRAbs: 0.7)
occl_scale: tuple = (0.02, 0.15)
jpeg_p: float = 0.0 # random JPEG re-encode probability
@@ -395,6 +401,7 @@ class SAM3DStudentDataset(Dataset):
geom_flip_p: float = 0.5,
geom_scale_max: float | None = None,
cliff_follows_aug: bool = False,
+ cliff_focal: bool = False,
occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),
jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90)):
super().__init__()
@@ -408,6 +415,7 @@ class SAM3DStudentDataset(Dataset):
self.geom_flip_p = geom_flip_p
self.geom_scale_max = geom_scale_max
self.cliff_follows_aug = cliff_follows_aug
+ self.cliff_focal = cliff_focal
tfm_list = [transforms.Resize((image_size, image_size))]
@@ -630,9 +638,36 @@ class SAM3DStudentDataset(Dataset):
image = self.transform(img_pil)
- cx_norm = 2.0 * (cx / orig_w) - 1.0
- cy_norm = 2.0 * (cy / orig_h) - 1.0
- cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)
+ if self.cliff_focal:
+ # Where the box sits in the FIELD OF VIEW, and how large it is
+ # ANGULARLY -- not in image-normalised pixels.
+ #
+ # Under a fixed focal the two are interchangeable, and the corpus
+ # this trainer was written for had exactly that: every crop from
+ # tools/annotate_dataset.py carries the synthetic f = sqrt(H^2+W^2),
+ # i.e. f/diag = 1.000 with zero spread. The rebuilt corpus carries
+ # the datasets' own focals instead, and those span f/diag 0.6-1.8 on
+ # coco/aic/mpii (sd ~0.35) with harmony4d off at 0.285. There the
+ # pixel version is ambiguous: two crops with identical conditioning
+ # correspond to different depths, the network can only predict the
+ # conditional mean, and the residual leaves as unstable cam_trans
+ # and body scale -- measured on 3DPW test as 220 mm/frame^2 of
+ # translation jitter (98% of it in depth) against 148 for a model
+ # trained on the constant-focal corpus.
+ #
+ # One scalar focal (fy) rather than fx/fy, because the rotation
+ # augmentation mixes the two axes and fx != fy on the calibrated
+ # splits.
+ f = float(ann["cam_focal_length"][1])
+ b_px = b_scale * max(orig_w, orig_h)
+ cliff_cond = torch.tensor(
+ [math.atan((cx - orig_w / 2.0) / f),
+ math.atan((cy - orig_h / 2.0) / f),
+ b_px / f], dtype=torch.float32)
+ else:
+ cx_norm = 2.0 * (cx / orig_w) - 1.0
+ cy_norm = 2.0 * (cy / orig_h) - 1.0
+ cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)
return {
"image": image,
@@ -659,6 +694,7 @@ def build_dataloaders(cfg):
geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,
geom_scale_max=cfg.geom_scale_max,
cliff_follows_aug=cfg.cliff_follows_aug,
+ cliff_focal=cfg.cliff_focal,
occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality)
full_dataset = SAM3DStudentDataset(
diff --git a/instanthmr_distill_train/val3dpw.py b/instanthmr_distill_train/val3dpw.py
index 08eb332..ebaed1a 100644
--- a/instanthmr_distill_train/val3dpw.py
+++ b/instanthmr_distill_train/val3dpw.py
@@ -61,7 +61,8 @@ IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
MM = 1000.0
-def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int):
+def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int,
+ focal: float | None = None):
"""Square 1.2x crop + ImageNet normalise + CLIFF conditioning.
A transcription of ``instanthmr.inference.InstantHMR._preprocess``. It is
@@ -74,9 +75,17 @@ def _preprocess(image_rgb: np.ndarray, bbox: np.ndarray, h: int, w: int):
bw, bh = x2 - x1, y2 - y1
cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
- cliff = np.array([2.0 * (cx / w) - 1.0,
- 2.0 * (cy / h) - 1.0,
- max(bw, bh) / max(w, h)], dtype=np.float32)
+ # Must match SAM3DStudentDataset.__getitem__ exactly, including which
+ # variant. Feeding a model trained on one and evaluated on the other is a
+ # silent distribution shift, not an error.
+ if focal is not None:
+ cliff = np.array([math.atan((cx - w / 2.0) / focal),
+ math.atan((cy - h / 2.0) / focal),
+ max(bw, bh) / focal], dtype=np.float32)
+ else:
+ cliff = np.array([2.0 * (cx / w) - 1.0,
+ 2.0 * (cy / h) - 1.0,
+ max(bw, bh) / max(w, h)], dtype=np.float32)
sq = max(bw, bh) * CROP_EXPAND
half = sq / 2.0
@@ -106,12 +115,13 @@ class ThreeDPWValSet(Dataset):
def __init__(self, sequence_dir: str, image_root: str,
split: str = "validation", stride: int = 5,
bbox_scale: float = 1.2, gt: str = "h36m",
- gt_dir: str | None = None):
+ gt_dir: str | None = None, cliff_focal: bool = False):
self.samples, self.gt = P.build_samples(
sequence_dir, image_root, split=split,
stride=stride, bbox_scale=bbox_scale, gt=gt, gt_dir=gt_dir)
self.gt = np.asarray(self.gt, dtype=np.float32)
self.gt_source = gt
+ self.cliff_focal = cliff_focal
def __len__(self) -> int:
return len(self.samples)
@@ -128,7 +138,8 @@ class ThreeDPWValSet(Dataset):
"ok": torch.tensor(0.0)}
rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
h, w = rgb.shape[:2]
- crop, cliff = _preprocess(rgb, s["bbox"], h, w)
+ crop, cliff = _preprocess(rgb, s["bbox"], h, w,
+ s["focal"] if self.cliff_focal else None)
return {"image": torch.from_numpy(crop),
"cliff_cond": torch.from_numpy(cliff),
"gt": torch.from_numpy(self.gt[i]),
diff --git a/tools/pth_to_onnx.py b/tools/pth_to_onnx.py
index c4365a8..c4bd018 100644
--- a/tools/pth_to_onnx.py
+++ b/tools/pth_to_onnx.py
@@ -34,9 +34,9 @@ Usage
python tools/pth_to_onnx.py
# or point at a specific checkpoint / output
- python tools/pth_to_onnx.py \
- --ckpt instanthmr_distill_train/runs/distill_mhr_only/distill_mhr_only/best_student_model_ema.pth \
- --output models/instanthmr_mhr_only.onnx
+ python ./tools/pth_to_onnx.py \
+ --ckpt ./instanthmr_distill_train/runs/b2_s0/b2_s0/best_student_model_ema.pth \
+ --output ./models/instanthmr_mhr_only.onnx
"""
import argparse
import importlib