Python
#!/usr/bin/env python3
"""InstantHMR trainer: dataset, augmentation, model, MHR forward pass, losses, and a single-GPU loop.
The network predicts the MHR parameters (pose, skeleton, body shape), the camera translation, and
the 70 keypoints in 2D (SimCC). Every 3D quantity is derived from the MHR forward pass, as in
SAM 3D Body. `train_ddp.py` imports this file for multi-GPU runs; see training/README.md.
python training/train.py --self-test --data_root data
python training/train.py --overfit-test --data_root data
python training/train.py --data_root data --output_dir runs/my_run
"""
import os
# Must be set before the first CUDA allocation (read lazily by the allocator).
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import argparse
import io
import copy
import json
import math
import random
import re
import gc
from dataclasses import dataclass, field
from pathlib import Path
import numpy as np
from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.optim.lr_scheduler as lr_scheduler
from torch.utils.data import Dataset, DataLoader, random_split
from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn
import torchvision.transforms as transforms
try:
import mhr_cont as MC
except ImportError: # imported as a package rather than a script
from training import mhr_cont as MC
import torchvision.transforms.functional as F_t
from torchvision.transforms import InterpolationMode
import timm
from tqdm import tqdm
# --- Notebook Cell 0 setup (headless-safe) ---
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import cv2
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
PROJECT_ROOT = Path(__file__).resolve().parent
DEFAULT_DATA_ROOT = "data"
# MHR70 joint ordering
JOINT_NAMES = [
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
"left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle",
"left_big_toe_tip", "left_small_toe_tip", "left_heel",
"right_big_toe_tip", "right_small_toe_tip", "right_heel",
"right_thumb_tip", "right_thumb_first_joint", "right_thumb_second_joint", "right_thumb_third_joint",
"right_index_tip", "right_index_first_joint", "right_index_second_joint", "right_index_third_joint",
"right_middle_tip", "right_middle_first_joint", "right_middle_second_joint", "right_middle_third_joint",
"right_ring_tip", "right_ring_first_joint", "right_ring_second_joint", "right_ring_third_joint",
"right_pinky_tip", "right_pinky_first_joint", "right_pinky_second_joint", "right_pinky_third_joint",
"right_wrist",
"left_thumb_tip", "left_thumb_first_joint", "left_thumb_second_joint", "left_thumb_third_joint",
"left_index_tip", "left_index_first_joint", "left_index_second_joint", "left_index_third_joint",
"left_middle_tip", "left_middle_first_joint", "left_middle_second_joint", "left_middle_third_joint",
"left_ring_tip", "left_ring_first_joint", "left_ring_second_joint", "left_ring_third_joint",
"left_pinky_tip", "left_pinky_first_joint", "left_pinky_second_joint", "left_pinky_third_joint",
"left_wrist",
"left_olecranon", "right_olecranon", "left_cubital_fossa", "right_cubital_fossa",
"left_acromion", "right_acromion", "neck",
]
def _build_flip_perm(names):
idx = {n: i for i, n in enumerate(names)}
perm = list(range(len(names)))
for i, n in enumerate(names):
if n.startswith("left_"):
perm[i] = idx.get("right_" + n[5:], i)
elif n.startswith("right_"):
perm[i] = idx.get("left_" + n[6:], i)
perm = np.array(perm, dtype=np.int64)
assert (perm[perm] == np.arange(len(perm))).all(), "flip permutation is not an involution"
return perm
FLIP_PERM = _build_flip_perm(JOINT_NAMES)
# 40 of the 70 keypoints are finger joints. Measured on a trained checkpoint,
# they carry 61.0% of loss_3d_native's L1 mass and 60% of Mesh_PA_MPJPE, while
# the published metrics (J12 / J14) contain none of them and SA-1B — 55% of the
# batch under `--mix sqrt` — records hands as 8% observed. `finger_weight`
# down-weights them in the FK-derived geometric losses; see DistillConfig.
FINGER_KP = np.array(
[i for i, n in enumerate(JOINT_NAMES)
if any(t in n for t in ("thumb", "index", "middle", "ring", "pinky"))],
dtype=np.int64)
BODY_KP = np.array([i for i in range(len(JOINT_NAMES)) if i not in set(FINGER_KP.tolist())],
dtype=np.int64)
assert len(FINGER_KP) == 40 and len(BODY_KP) == 30
# MHR model-parameter layout, read off character_torch.parameter_transform:
# 0:3 root_tx/ty/tz (always 0 in the corpus)
# 3:6 root_rx/ry/rz -> joint-parameter slots 10/11/12 and nothing else
# 6:136 130 local joint angles
# 136:204 68 bone scales
# Because the root rotation is exclusive to slots 10/11/12, the 130 local angles
# are EXACTLY invariant to the in-plane rotation the dataset applies, which is
# what makes `pose_split` valid. Verified: recomposing the root as
# R_aug @ R_root in extrinsic-xyz Euler reproduces the rotated skeleton up to a
# pure translation (the pelvis-vs-origin pivot offset), not a pose change.
POSE_ROOT_DIM = 6
# ============================================================
# Cell 1 — Configuration
# ============================================================
@dataclass
class DistillConfig:
"""All hyperparameters in one place."""
data_root: str = str(DEFAULT_DATA_ROOT)
mhr_model_path: str = str(PROJECT_ROOT / "checkpoints/mhr_model.pt")
# (70, 127) affine map from the MHR skeleton joints to the 70 native
# keypoints. Regenerate with --fit-regressor if the MHR model changes.
# Lives in assets/ rather than checkpoints/ because checkpoints/ is gitignored
# and this 35 KB matrix must ship with the code (otherwise every fresh clone
# silently refits it at job start).
kp_regressor_path: str = str(PROJECT_ROOT / "assets/mhr_j127_to_kp70.npy")
log_dir: str = str(PROJECT_ROOT / "runs/distill_mhr_only")
image_size: int = 224
# --preset v3: sample each crop from a context crop (build_split.py
# --context) over a wide range of framings. The box multiple is drawn
# log-uniformly in [v3_box_min, v3_box_max] x the tight box (1.2 = today's
# crop), the centre shifted by up to +/- v3_shift of the window side.
context_crops: bool = False
v3_box_min: float = 0.6
v3_box_max: float = 2.0
v3_shift: float = 0.15
# Lanczos instead of bilinear for the v3 window cut. Bilinear after the
# stored crop's own resize left v3 training crops at 0.73 (224) / 0.67 (288)
# of the sharpness of a crop cut straight from the photo (val3dpw and the
# phone do the latter); Lanczos gives 1.00 / 0.94. Measured 2026-10-20 on 300
# AIC people. Bilinear-trained g9 models lost 3-5 mm (224) and up to 17 mm
# (288) on sharp tight-box crops. Off by default so in-flight g9 runs resume
# unchanged.
v3_lanczos: bool = False
# --preset v4: each image is a tile of K windows that build_split.py
# --windows cut from the full-resolution photo with ONE resize, the same
# operation as the runtime crop; one window is drawn per sample (window 0,
# the standard 1.2x crop, when not augmenting).
crop_windows: bool = False
# "random": one of the K windows per augmented sample (v4). "standard":
# always window 0, i.e. the runtime crop of a perfect detector box -- the
# baseline framing, read from a window corpus at any --image-size.
window_mode: str = "random"
# window_mode "random" only: probability of window 0 (the standard crop);
# the other windows share the rest. None = all K equally likely (v4, g10f).
window0_p: float | None = None
max_images: int | None = None
per_dataset_caps: dict = field(default_factory=lambda: {"sam3d_gt_harmony4d": 300_000})
val_split: float = 0.1
num_workers: int = 4
augment: bool = True
geom_p: float = 0.8
geom_rot_deg: float = 30.0
geom_scale_range: float = 0.25
geom_trans: float = 0.08
geom_flip_p: float = 0.5
# --- v2 knobs. Every default below reproduces the pre-v2 behaviour bit for
# --- bit, so a baseline run started before these existed is unaffected.
geom_scale_max: float | None = None # zoom-in ceiling; None => symmetric
# U(1-range, 1+range) as before. 2.0 is
# MeTRAbs's quarter-area truncation bound.
# Divide the CLIFF crop-centre correction by the zoom (see the derivation at
# the call site). Default off so gen-5 and `--preset baseline` reproduce.
crop_centre_fix: bool = False
cliff_follows_aug: bool = False # CLIFF cx/cy/b_scale track the augmented
# crop box, as a real detector box would
cliff_focal: bool = False # perspective-correct CLIFF conditioning:
# angles off the optical axis and angular
# box size, instead of image-normalised
# pixels. Needs cam_focal_length, which
# every npz carries. See the note in
# SAM3DStudentDataset.__getitem__.
occl_p: float = 0.2 # RandomErasing probability (MeTRAbs: 0.7)
occl_scale: tuple = (0.02, 0.15)
jpeg_p: float = 0.0 # random JPEG re-encode probability
jpeg_quality: tuple = (30, 90)
mask_oob_2d: bool = False # mask 2D targets outside kp2d_range
# instead of clamping them to the edge
reproj_all_samples: bool = False # include flipped samples in loss_reproj
cam_loss: str = "mse" # "mse" | "euclid" (unsquared L2 on xyz)
# --- loss-rebalance knobs (`--losses rebalanced` in train_ddp.py).
# --- Every default below reproduces the pre-rebalance behaviour bit for bit,
# --- so `--preset baseline` and `--preset v2` are unaffected. All four come
# --- from the measured error/gradient budget:
# * an oracle ablation attributes 34.3 of the 37.8 mm of removable
# PA-MPJPE to pose[6:136] and 0.00 mm to shape_params, the bone scales
# and the root, yet loss_shape owns 27% of the gradient and loss_pose
# 4.7% on 21% of the batch;
# * SmoothL1's default beta=1.0 is a metre, so on metre-scale coordinates
# every sample sits in the quadratic branch -- 16x less gradient than L1
# and 31x less than NLF's unsquared Euclidean, at the same weight.
pose_split: bool = False # supervise pose[6:136] on the 60% of the batch
# that is not flipped instead of the 21% that is
# un-augmented. Root stays on m_ident. The two
# halves keep their per-parameter weight (6/136
# and 130/136 of w_pose), so this changes the
# MASK, not the root-vs-local balance.
pose_beta: float = 1.0 # SmoothL1 beta for the pose terms, in radians.
# 1.0 rad is ~10x the p95 error, i.e. pure MSE.
kp3d_loss: str = "smooth_l1" # "smooth_l1" | "euclid" (NLF 2407.07532 §3.2:
# "the Euclidean loss without squaring, for
# better outlier-robustness"). euclid has a
# FLAT gradient, so it is *gentler* than the
# current form at init (measured 0.62x on a
# fresh model, 515 mm error) and stronger at
# convergence (6.4x at 60 mm) -- the opposite
# of the shape that caused the LR divergences.
finger_weight: float = 1.0 # per-keypoint weight on FINGER_KP inside
# loss_3d_native and loss_reproj
max_consecutive_skips: int = 200 # abort rather than burn the allocation
# on a run that has already diverged
kp2d_bins: int = 96
kp2d_range: float = 1.5
backbone: str = "repvit_m2_3"
backbone_feat_dim: int = 640
d_model: int = 512
n_heads: int = 8
n_decoder_layers: int = 4
dropout: float = 0.1
pose_dim: int = 136
scale_dim: int = 68
shape_dim: int = 45
cam_dim: int = 3
cliff_dim: int = 3
num_joints: int = 70
@property
def model_params_dim(self) -> int:
return self.pose_dim + self.scale_dim
batch_size: int = 64
lr: float = 3e-4
weight_decay: float = 1e-4
# Exclude depthwise-conv kernels, norm params and biases from weight decay
# (see build_param_groups). Off by default so existing runs are unchanged;
# enable with --split-wd. Applies to every backbone, not just MobileNet.
split_weight_decay: bool = False
epochs: int = 400
warmup_epochs: int = 3
grad_clip: float = 1.0
anomaly_loss_threshold: float = 100.0
# A skipped step updates NOTHING — not the weights, not the EMA, not the LR
# schedule (they all live behind the same `continue`). So a model that is
# anomalous on every batch is frozen for good: the loss can never come back
# under the threshold and the run idles until early stopping, which took
# ~139 wasted epochs the one time it happened. After this many CONSECUTIVE
# skips, roll the raw weights back to the EMA copy, scale the LR down and
# clear the optimiser moments; give up after `max_ema_rollbacks` attempts.
anomaly_skip_patience: int = 50
anomaly_rollback_lr_decay: float = 0.5
max_ema_rollbacks: int = 3
# On an anomalous batch, back-propagate the terms that do NOT go through the
# MHR forward kinematics instead of discarding the batch. A skipped step
# throws away `loss_scale` / `loss_pose` -- the ONLY force pulling the bone
# scales back toward the teacher -- so the guard removes exactly the gradient
# that would fix the anomaly and the sample is quarantined for good. Those
# parameter-space terms are bounded by the target magnitudes, so stepping on
# them is safe even when the FK terms are 1e13. Default off: `--preset
# baseline` and `--losses legacy` must stay bit-identical.
anomaly_safe_fallback: bool = False
# Squash the 68 bone scales into the rig's OWN parameter_limits (widened by
# scale_bound_margin). mhr_params[136:204] are raw linear outputs today and
# they enter the forward kinematics multiplicatively along the kinematic
# chain: measured on checkpoints/mhr_model.pt, a uniform +10 turns a 1.5 m
# skeleton into 336 m and a +25 into 28,300 km, which is where the 4e7 and
# 8.8e13 losses in the diverged runs come from. The rig bounds every one of
# them to [-1.1, 1.1]; measured over 9,000 teacher annotations, 0.0000% of
# targets fall outside those limits at margin 0.5 and the worst target sits
# at |atanh| = 0.97, so the bound is free. Default off for bit-identity.
bound_scales: bool = False
scale_bound_margin: float = 0.5
scale_bounds_path: str = str(PROJECT_ROOT / "assets/mhr_size_bounds.npz")
# --- the teacher's continuous regression pathway (--cont-head) ----------
# Off by default: with it off every tensor in this file is bit-identical to
# the previous commit, which generations 6 and 7 depend on as controls.
#
# On, the single `nn.Linear` that emits the rig-native 204-vector is
# replaced by the teacher's own space -- 6D root rotation, a 260-dim
# continuous body block, 28 bone-scale PCA coefficients and two 54-dim hand
# blocks -- converted to the 204-vector inside the model, so nothing
# downstream (losses, FK, export, deployment) sees a different contract.
# See training/mhr_cont.py.
cont_head: bool = False
cont_head_path: str = str(PROJECT_ROOT / "assets/mhr_cont_head.npz")
# Rotation-matrix loss on the root instead of SmoothL1 on its Euler triple.
# Set together with cont_head: a SmoothL1 in radians scores a prediction at
# -pi+eps against a target at pi-eps as a ~2pi error when it is a tiny
# rotation, and 13.59% of corpus root angles sit beyond 2.8 rad.
root_rot_loss: bool = False
w_root_rot: float = 1.0
# --- the teacher's exact 70-landmark readout (--exact-landmarks) --------
# Off by default. On, loss_3d_native and loss_reproj read the 70 keypoints
# out of the skinned mesh with the teacher's own fixed mapping
# (K = Wj @ J + Wv @ V) instead of the fitted (70, 127) skeleton-only
# matrix. 21 of the 70 are pure mesh-surface points with no joint
# contribution at all, so the fitted matrix cannot represent them; measured
# 2.93-3.75 mm mean disagreement over the 30 non-finger landmarks on
# identical GT geometry. Training-only: the exported graph never emitted
# these keypoints, so deployment still uses the approximate readout.
exact_landmarks: bool = False
landmark_assets_path: str = str(PROJECT_ROOT / "assets/mhr_landmarks70.npz")
use_amp: bool = True
ema_decay: float = 0.9998
early_stop_patience: int = 150
resume: bool = True
w_pose: float = 1.0
w_scale: float = 0.1
w_shape: float = 1.0
w_cam: float = 0.1
# 2.0, NOT the 10.0 of the earlier trainer. There, w=10
# fed a free 3D coordinate head that had no other supervision. Here the same
# parameters are ALREADY supervised directly by loss_pose / loss_scale, and
# this term back-propagates through the MHR forward kinematics. Measured
# gradient norms on a fresh model (see the report): at w=10 this loss carries
# |grad| = 50.4 against loss_pose's 1.13, so after grad-clipping to 1.0 it
# owns ~95% of the update direction and nothing keeps the parameters in a
# valid MHR region -> 2 of 3 overfit runs diverged. At w=2 the term leads
# without dominating (|grad| ~ 10 vs loss_cam's 12) and 6/6 seeded runs are
# clean with 0 anomalous steps.
w_keypoints3d: float = 2.0 # on the MHR-derived 70 keypoints
w_keypoints2d: float = 10.0
w_3d_joints: float = 1e-3 # on the raw 127 skeleton joints (see below)
w_reproj: float = 0.01 # single re-projection of the MHR keypoints
w_simcc: float = 1.0
# Surface (PVE) term. `shape_params` move the 127-joint skeleton by exactly
# 0.000e+00 cm -- measured -- so every other geometric loss here is blind to
# body shape and the mesh is supervised only by w_shape in parameter space.
# This term skins the rig and compares vertices, which is the only way a
# geometric gradient reaches the 45 identity coefficients.
#
# n_verts is a farthest-point subset of the rig's OWN 18,439-vertex mesh,
# not a separate LOD asset: same topology and same vertex ids as the
# teacher, so the correspondence is exact rather than approximate, and the
# rig's own skinning op is driven on the subset (verified bit-exact against
# the full mesh to 4.6e-05 cm). Measured at batch 64 on an RTX 4070:
# skeleton-only FK 4.85 ms, +595 verts 5.44 ms, +all 18,439 verts 21.6 ms --
# i.e. the subset costs 0.6 ms (0.3% of a 207 ms step) where the full mesh
# costs 16.7 ms (8.1%). 595 is lod6's vertex count.
# w_verts = 0.0 disables the term and every code path it touches, so
# `--preset baseline` / `--losses legacy` stay bit-identical by default.
w_verts: float = 0.0
n_verts: int = 595
# Linear ramp (in optimiser steps) for the two losses that back-propagate
# through the MHR forward kinematics, so that loss_pose can pull the
# parameters into a valid region before the geometric terms engage.
# NOTE: on its own the ramp did NOT cure the divergence — w_keypoints3d=2.0
# is the actual fix. It is kept as cheap insurance for the first epoch of a
# real run, where the trunk is still random.
kp3d_warmup_steps: int = 2000
# --- Foot lock (--prev-token): the decoder gets one extra query carrying
# --- the previous frame's 70 keypoints + cam_trans, or a learned "no
# --- previous pose" embedding. Keypoints, not MHR parameters, because the
# --- flip augmentation mirrors the labels and the parameters cannot be
# --- mirrored (the reason loss_pose is masked to m_noflip). All off by
# --- default, so every existing preset stays bit-identical.
prev_token: bool = False
prev_mask_p: float = 0.5 # P(previous pose withheld) per sample
# Withhold the previous pose on every sample without a real earlier frame
# (stills, a missing pair). Off, a still's "previous" pose is its own GT +
# noise, ~87% of the mix, and the model learns prev ~= now: fed its own
# prediction 5 frames back on 3DPW test, g11C goes 39.85 -> 43.58 J14 PA.
prev_pairs_only: bool = False
# Noise on the previous pose fed to the token, standing in for the model's
# own error at inference: per-joint metres, cam x/z and y/z, log-depth.
# Set to g10m75_b4's per-frame error on 512 held-in Harmony4D / AIC crops
# (2026-10-30): joints 2.3-5.3 cm per axis, x/z 0.016-0.021, y/z
# 0.020-0.030, log depth 0.032-0.037.
prev_noise_joint: float = 0.035
prev_noise_xy: float = 0.02
prev_noise_logz: float = 0.035
# Splits whose crops are named <...>_<frame>_p<id> with consecutive
# annotations `pair_frame_step` frames apart, and the largest gap (in
# annotation steps) drawn for each. Stills use their own GT as "previous".
pair_gaps: dict = field(default_factory=lambda: {"harmony4d": 2})
pair_frame_step: int = 5
# Contact from GT foot motion where no measured contact exists (Harmony4D):
# the mean displacement of a foot's 4 keypoints between the pair, under
# contact_thr is contact, over noncontact_thr is not, in between masked.
contact_thr: float = 0.020
noncontact_thr: float = 0.050
# That rule fails on Harmony4D: a foot within 1 cm of the floor still moves
# a median 3.1 cm over 5 frames there, so 42% of its pairs come out "both
# feet in the air" against ~4% really off the floor. With a floor table
# (tools/build_contact_floor.py), a track that has a fitted floor is
# labelled by height instead: lowest foot point < floor_contact_h above it
# and moved < floor_contact_d is contact, > floor_air_h is not (pair or
# not), else masked; 97.3% agreement with insole-measured contact. "" = off.
contact_floor: str = ""
floor_contact_h: float = 0.020
floor_contact_d: float = 0.050
floor_air_h: float = 0.050
# Contact BCE multiplier on samples with BOTH feet validly off the ground
# (flight), the rarest label: ~3 in 10,000 samples.
contact_flight_weight: float = 1.0
contact_head: bool = False
w_contact: float = 0.0 # BCE on the per-foot contact logits
w_zero_vel: float = 0.0 # foot displacement vs the previous-pose input, in contact
w_foot_mocap: float = 0.0 # mocap foot keypoints (camera frame) where a split has them, replacing
# the teacher's in loss_3d_native
# REMOVED w.r.t. the earlier trainer:
# w_structural / w_feet — they glued the 3D head to the mesh; no head, no glue.
# w_bone_length — MHR enforces bone lengths through its scale params.
# w_reproj_3d/w_reproj_mhr — merged into the single w_reproj above.
# Kept only for the legacy 52-anchor Mesh52_* metrics, so that runs of this
# script stay comparable with the earlier trainer.
# No loss uses them any more.
native_mapping_ids = [ 1, 2, 5, 6, 9, 10, 11, 12, 13, 14, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62 ]
mhr_mapping_ids = [ 125, 123, 75, 39, 2, 18, 3, 19, 5, 21, 64, 63, 62, 61, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 41, 100, 99, 98, 97, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 77 ]
def apply_rebalanced_losses(cfg) -> None:
"""The four loss-budget fixes, as one switch. Lives here, not in
train_ddp.py, so the single-GPU trainer and the DDP driver cannot
drift apart.
Every number is chosen from a measured unit-weight gradient norm on real
augmented batches, not from taste. The resulting split of the update
direction (batch 12, epoch-78 checkpoint, COCO):
term before after
loss_3d_native 8.5% 30.3%
loss_2d_simcc 29.6% 26.4%
loss_pose 5.6% 20.2%
loss_cam 13.8% 12.3%
loss_mhr_joints 8.5% 7.6%
loss_shape 14.4% 0.4%
`w_keypoints3d` drops 2.0 -> 0.35 because the loss form under it changed:
at equal weight the unsquared Euclidean carries 31x the gradient of
SmoothL1(beta=1) at the operating point, so keeping 2.0 would put the
FK path at ~20x its current strength -- the failure mode the w=10
experiments in this file's header already found. 0.35 lands it at ~3x,
leading without dominating.
"""
cfg.w_shape = 0.03 # 45 identity blendshapes. Verified: driving MHR
# params 204..248 to +/-2 sigma moves the 127-joint
# skeleton by 0.00e+00 cm, and get_joints() zeroes
# them before the forward pass anyway. Non-zero so
# the deployed mesh identity stays supervised.
cfg.pose_split = True # pose[6:136] on m_noflip (60%) not m_ident (21%)
cfg.pose_beta = 0.05 # ~= the p50 pose-parameter error, so the term is
# linear over the bulk of the distribution
cfg.kp3d_loss = "euclid" # NLF 2407.07532 §3.2
cfg.w_keypoints3d = 0.35 # re-scaled for the new form; see docstring
cfg.finger_weight = 0.2 # 40 of 70 keypoints, none of them in J12/J14
def config_from_checkpoint(state, ckpt_path=None, **overrides):
"""Rebuild the DistillConfig a checkpoint was actually trained with.
Two sources, because the checkpoint dict carries neither. (``image_size``
sits in both: the positional-embedding grid in the weights and the value
in run_config.json, which must agree.)
* The ARCHITECTURE flags are readable from the weights. ``--bound-scales``
registers ``scale_lo`` / ``scale_hi`` buffers, so their presence in the
state dict IS the flag. Building the default config instead drops the
tanh remap while the head still emits pre-tanh values -- measured on 64
COCO teacher targets, that is 21.0 mm MPJPE / 14.4 mm PA-MPJPE of silent
corruption on a model whose benchmark is 41.8 mm.
* The INPUT-PIPELINE flags are not. ``--cliff-focal`` changes what the
conditioning vector MEANS, not the graph, so nothing in the weights
records it; it is read from the ``run_config.json`` that
train_ddp.py writes beside the checkpoint. Feeding a focal-aware
model the pixel form (or the reverse) raises nothing -- it silently
mis-places the person in depth.
Returns ``(cfg, provenance)``; *provenance* maps each restored field to
where it came from, so a caller can print what it inferred rather than
leaving the user to trust it.
"""
cfg = DistillConfig()
prov = {}
cfg.bound_scales = "scale_lo" in state
prov["bound_scales"] = "checkpoint weights"
# --cont-head replaces head_global with a ContMHRHead, so the buffers it
# registers ARE the flag -- more reliable than run_config.json, which a
# hand-copied checkpoint may not have beside it. Rebuilding the default
# head for these weights would fail loudly on the missing keys rather than
# silently, but only because the shapes differ; do not rely on that.
cfg.cont_head = any(k.startswith("cont_head.") for k in state)
cfg.root_rot_loss = cfg.cont_head
prov["cont_head"] = "checkpoint weights"
# --prev-token / the contact head are parameters, so the weights say so.
cfg.prev_token = any(k.startswith("prev_proj.") for k in state)
cfg.contact_head = any(k.startswith("contact_head.") for k in state)
prov["prev_token"] = prov["contact_head"] = "checkpoint weights"
# --exact-landmarks leaves NO trace in the weights: it changes which
# operator two training losses use, not the graph. So it can only come from
# run_config.json, and its absence there is indistinguishable from "off".
# That is acceptable because nothing downstream of a checkpoint depends on
# it -- export and inference use the fitted readout either way.
run_cfg = {}
if ckpt_path is not None:
rc = Path(ckpt_path).resolve().parent / "run_config.json"
if rc.is_file():
run_cfg = json.loads(rc.read_text())
for k in ("cliff_focal", "backbone", "scale_bound_margin"):
if k in run_cfg:
setattr(cfg, k, run_cfg[k])
prov[k] = "run_config.json"
# The input size IS in the weights: the decoder's positional embedding is a
# saved buffer with one row per 32-px patch, so its grid fixes the size up
# to the stride. run_config.json carries the exact value; a contradiction
# between the two means the wrong run_config.json sits beside the weights.
if "mem_pos_embed" in state:
grid = math.isqrt(state["mem_pos_embed"].shape[1])
cfg.image_size = grid * 32
prov["image_size"] = "checkpoint weights"
if "image_size" in run_cfg and int(run_cfg["image_size"]) != grid * 32:
raise ValueError(
f"run_config.json says image_size={run_cfg['image_size']} but "
f"the weights have a {grid}x{grid} patch grid ({grid * 32} px)")
elif "image_size" in run_cfg:
cfg.image_size = int(run_cfg["image_size"])
prov["image_size"] = "run_config.json"
for k, v in overrides.items():
if v is not None:
setattr(cfg, k, v)
prov[k] = "command line"
if "cliff_focal" not in prov:
# Either no run_config.json beside the checkpoint, or one written
# before --cliff-focal existed. Both mean "assume the old pixel form",
# which is right for those runs but must be visible, not assumed.
prov["cliff_focal"] = ("DEFAULT -- run_config.json is missing or "
"predates the flag; pass --cliff-focal / "
"--no-cliff-focal if that is wrong")
return cfg, prov
# ============================================================
# Cell 3 — Dataset and Data Augmentation
# ============================================================
class RandomJPEG:
"""Re-encode through JPEG at a random quality.
The one item on NLF's augmentation list that this pipeline lacked, and the
most deployment-relevant of them: phone and tablet camera stacks emit JPEG,
often aggressively compressed, while training crops are decoded from disk at
whatever quality the source dataset happened to store. Blocking artefacts
and chroma subsampling are a distribution the model otherwise never sees.
"""
def __init__(self, p: float = 0.0, qmin: int = 30, qmax: int = 90):
self.p, self.qmin, self.qmax = p, qmin, qmax
def __call__(self, img):
if self.p <= 0.0 or random.random() >= self.p:
return img
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=random.randint(self.qmin, self.qmax))
buf.seek(0)
return Image.open(buf).convert("RGB")
class RandomPixelate:
def __init__(self, p=0.2, min_res=32, max_res=112):
self.p = p
self.min_res = min_res
self.max_res = max_res
def __call__(self, img):
if random.random() < self.p:
orig_w, orig_h = img.size
degraded_h = random.randint(self.min_res, self.max_res)
degraded_w = int(orig_w * (degraded_h / orig_h))
img = F_t.resize(img, [degraded_h, degraded_w], interpolation=InterpolationMode.NEAREST)
img = F_t.resize(img, [orig_h, orig_w], interpolation=InterpolationMode.NEAREST)
return img
def _apply_label_geometry(M_total, aug_flip, angle, joints_2d, joints_2d_vis,
joints_3d, cam_trans, cx, cy, orig_w, orig_h, W, H):
"""Move every label with an augmentation: joints_2d through M_total (crop
px -> augmented px), the left/right swap for a flip, the in-plane rotation
of joints_3d / cam_trans, and the CLIFF box centre with both. Shared by the
standard and the v3 path so the two can never disagree."""
px = (joints_2d[:, 0] + 1.0) * 0.5 * W
py = (joints_2d[:, 1] + 1.0) * 0.5 * H
px2 = M_total[0, 0] * px + M_total[0, 1] * py + M_total[0, 2]
py2 = M_total[1, 0] * px + M_total[1, 1] * py + M_total[1, 2]
joints_2d[:, 0] = (px2 / (0.5 * W) - 1.0).astype(np.float32)
joints_2d[:, 1] = (py2 / (0.5 * H) - 1.0).astype(np.float32)
if aug_flip > 0.5:
joints_3d[:, 0] = -joints_3d[:, 0]
joints_3d = joints_3d[FLIP_PERM]
# joints_2d was already mirrored geometrically by M_total, but a
# mirrored image LOOKS like the opposite-handed person, so the
# left/right labels must be swapped too - exactly as for joints_3d.
# Without this the 2D head gets contradictory left/right supervision
# and collapses symmetric joints toward the body midline.
joints_2d = joints_2d[FLIP_PERM]
joints_2d_vis = joints_2d_vis[FLIP_PERM]
cam_trans[0] = -cam_trans[0]
cx = orig_w - cx
rad = math.radians(angle)
cos_t, sin_t = math.cos(rad), math.sin(rad)
R_v = np.array([[cos_t, sin_t], [-sin_t, cos_t]], dtype=np.float32)
joints_3d[:, :2] = joints_3d[:, :2] @ R_v.T
cam_trans[:2] = R_v @ cam_trans[:2]
dxy = np.array([cx - orig_w / 2.0, cy - orig_h / 2.0], dtype=np.float64)
dxy = R_v.astype(np.float64) @ dxy
cx, cy = orig_w / 2.0 + dxy[0], orig_h / 2.0 + dxy[1]
return joints_2d, joints_2d_vis, joints_3d, cam_trans, cx, cy, cos_t, sin_t
# Left then right: ankle, big toe tip, small toe tip, heel. FLIP_PERM swaps the
# two halves, so a flipped sample keeps this order with its sides exchanged.
FOOT_KP = np.array([13, 15, 16, 17, 14, 18, 19, 20])
PREV_DIM = 70 * 3 + 3 # keypoints + (x/z, y/z, log z) of cam_trans
PAIR_NAME = re.compile(r"^(.*)_(\d+)_p(\d+)$")
def _geom_points(pts, aug_flip, angle):
"""joints_3d's half of _apply_label_geometry, for other (K, 3) point sets."""
pts = pts.copy()
if aug_flip > 0.5:
pts[:, 0] = -pts[:, 0]
pts = pts[FLIP_PERM]
rad = math.radians(angle)
R_v = np.array([[math.cos(rad), math.sin(rad)], [-math.sin(rad), math.cos(rad)]], dtype=np.float32)
pts[:, :2] = pts[:, :2] @ R_v.T
return pts
def _geom_cam(cam, aug_flip, angle):
"""cam_trans's half of _apply_label_geometry."""
cam = cam.copy()
if aug_flip > 0.5:
cam[0] = -cam[0]
rad = math.radians(angle)
R_v = np.array([[math.cos(rad), math.sin(rad)], [-math.sin(rad), math.cos(rad)]], dtype=np.float32)
cam[:2] = R_v @ cam[:2]
return cam
def temporal_kwargs(cfg):
"""The dataset's `temporal` argument: None unless --prev-token, so the
sample dict and the random stream of every existing preset are unchanged."""
if not getattr(cfg, "prev_token", False):
return None
return dict(mask_p=cfg.prev_mask_p, noise_joint=cfg.prev_noise_joint,
noise_xy=cfg.prev_noise_xy, noise_logz=cfg.prev_noise_logz,
pair_gaps=dict(cfg.pair_gaps), frame_step=cfg.pair_frame_step,
contact_thr=cfg.contact_thr, noncontact_thr=cfg.noncontact_thr,
pairs_only=getattr(cfg, "prev_pairs_only", False),
floor=_load_floors(getattr(cfg, "contact_floor", "")),
floor_contact_h=getattr(cfg, "floor_contact_h", 0.02),
floor_contact_d=getattr(cfg, "floor_contact_d", 0.05),
floor_air_h=getattr(cfg, "floor_air_h", 0.05))
def _load_floors(path):
"""tools/build_contact_floor.py output -> {camera track: (normal, point)}."""
if not path:
return None
z = np.load(path)
return {str(k): (n.astype(np.float32), a.astype(np.float32))
for k, n, a in zip(z["keys"], z["normal"], z["point"])}
class SAM3DStudentDataset(Dataset):
def __init__(self, data_root: str,
image_size: int = 224, max_images: int | None = None,
augment: bool = True, per_dataset_caps: dict | None = None,
pairs: list | None = None,
geom_p: float = 0.8, geom_rot_deg: float = 30.0,
geom_scale_range: float = 0.25, geom_trans: float = 0.08,
geom_flip_p: float = 0.5,
geom_scale_max: float | None = None,
cliff_follows_aug: bool = False,
crop_centre_fix: bool = False,
cliff_focal: bool = False,
occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),
jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90),
context_crops: bool = False, v3_box_min: float = 0.6,
v3_box_max: float = 2.0, v3_shift: float = 0.15,
v3_lanczos: bool = False, crop_windows: bool = False,
window_mode: str = "random", window0_p: float | None = None,
temporal: dict | None = None):
super().__init__()
self.temporal = temporal
self.crop_windows = crop_windows
self.window_mode = window_mode
self.window0_p = window0_p
self.v3_interp = cv2.INTER_LANCZOS4 if v3_lanczos else cv2.INTER_LINEAR
self.image_size = image_size
self.context_crops = context_crops
self.v3_box_min, self.v3_box_max, self.v3_shift = v3_box_min, v3_box_max, v3_shift
self.data_root = Path(data_root)
self.augment = augment
self.geom_p = geom_p
self.geom_rot_deg = geom_rot_deg
self.geom_scale_range = geom_scale_range
self.geom_trans = geom_trans
self.geom_flip_p = geom_flip_p
self.geom_scale_max = geom_scale_max
self.cliff_follows_aug = cliff_follows_aug
self.crop_centre_fix = crop_centre_fix
self.cliff_focal = cliff_focal
tfm_list = [transforms.Resize((image_size, image_size))]
if augment:
tfm_list.extend([
transforms.RandomApply([transforms.ColorJitter(
brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),
RandomPixelate(p=0.2, min_res=48, max_res=112),
RandomJPEG(p=jpeg_p, qmin=jpeg_quality[0], qmax=jpeg_quality[1]),
transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),
])
tfm_list.extend([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
if augment:
# MeTRAbs Table 7 puts synthetic occlusion at p=0.7 and credits it
# with 52.8 -> 49.3 mm on top of an already-strong colour pipeline.
tfm_list.append(transforms.RandomErasing(p=occl_p, scale=tuple(occl_scale)))
self.transform = transforms.Compose(tfm_list)
# A prebuilt index short-circuits the filesystem scan below. That scan
# is one glob plus a stat() per crop -- ~7.5 M metadata operations on a
# 3.7 M-crop corpus -- and on a parallel filesystem it dominates job
# startup, so having every DDP rank redo it is the largest fixed cost of
# a cluster run. See build_pair_index() in train_ddp.py.
if pairs is not None:
# Keep a compact index (train_ddp.PairIndex) AS IS. Calling
# list() on it would materialise 4.2 M Path pairs -- ~2 GB of Python
# objects that 36 forked dataloader workers then copy page by page
# through refcount writes, which OOM-killed four 4-GPU
# jobs at 74-76 GiB per node. Only copy a real list, which is the local /
# single-process path and small.
self.pairs = list(pairs) if isinstance(pairs, (list, tuple)) else pairs
print(f" Prebuilt index: {len(self.pairs)} (image, npz) pairs "
f"(augment={augment}).")
self._check_context_split()
return
self.pairs = []
dataset_dirs = self._find_dataset_dirs(self.data_root)
if not dataset_dirs:
raise FileNotFoundError(
f"No annotations/images sub-folders found under '{self.data_root}'."
)
cap_rng = random.Random(42)
caps = per_dataset_caps or {}
for ann_dir, img_dir in dataset_dirs:
sub_name = ann_dir.parent.name
sub_pairs = []
for npz_path in sorted(ann_dir.glob("*.npz")):
stem = npz_path.stem
img_path = img_dir / f"{stem}.jpg"
if not img_path.exists():
img_path = img_dir / f"{stem}.png"
if img_path.exists():
sub_pairs.append((img_path, npz_path))
# "*" is a default cap applied to every sub-folder that has no
# explicit entry — used to build a *balanced* subset for small-scale
# ablations. (max_images can't: it slices a path-sorted list, so it
# would return a single dataset.)
cap = caps.get(sub_name, caps.get("*"))
if cap is not None and len(sub_pairs) > cap:
n_before = len(sub_pairs)
sub_pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
sub_pairs = cap_rng.sample(sub_pairs, cap)
print(f" [cap] {sub_name}: sampled {cap:,} of {n_before:,} crops")
self.pairs.extend(sub_pairs)
self.pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
if max_images is not None:
self.pairs = self.pairs[:max_images]
print(f" Found {len(dataset_dirs)} sub-folder(s) under {self.data_root}; "
f"{len(self.pairs)} valid (image, npz) pairs (augment={augment}).")
self._check_context_split()
def _check_context_split(self):
"""v3 samples from context crops. A standard 224 split would still
'work' -- with the 1.2x crop read as if it were the context, i.e. every
box silently 1.67x too large -- so refuse it up front."""
if not (self.context_crops or self.crop_windows) or len(self.pairs) == 0:
return
# One probe per sub-folder, so a --data_root mixing context and
# standard splits fails here and not hours in, inside a worker.
firsts = getattr(self.pairs, "first_per_subfolder", None)
if firsts is not None:
probes = firsts()
else:
seen = {}
for k, (_, npz) in enumerate(self.pairs):
seen.setdefault(Path(npz).parent, k)
probes = list(seen.values())
for k in probes:
npz = Path(self.pairs[k][1])
with np.load(npz) as z:
if self.crop_windows:
if "win_boxes" not in z.files:
raise ValueError(
f"{npz.parent.parent.name}: {npz.name} has no win_boxes. "
f"--preset v4 needs a window split, built with "
f"build_split.py --windows 4 --window-sizes 224,288 ...")
tw, th = Image.open(self.pairs[k][0]).size
if th != self.image_size or tw != th * len(z["win_boxes"]):
raise ValueError(
f"{npz.parent.parent.name}: tile {tw}x{th} does not hold "
f"{len(z['win_boxes'])} windows of {self.image_size} px; "
f"use the data_win{self.image_size} corpus")
continue
if "bbox_context" not in z.files:
raise ValueError(
f"{npz.parent.parent.name}: {npz.name} has no bbox_context. "
f"--preset v3 needs a context-crop split, built with "
f"build_split.py --context 2.0 --body-size 448 --output-dir ...")
@staticmethod
def _find_dataset_dirs(root: Path):
dirs = []
if (root / "annotations").is_dir() and (root / "images").is_dir():
dirs.append((root / "annotations", root / "images"))
if root.is_dir():
for sub in sorted(root.iterdir()):
if not sub.is_dir():
continue
ann, img = sub / "annotations", sub / "images"
if ann.is_dir() and img.is_dir():
dirs.append((ann, img))
return dirs
def _context_window(self, img_pil, ann, W, H, b_size, cx, cy):
"""v3: cut the network input out of a context crop in ONE resampling.
The window is drawn in full-frame px: side = m x the tight box, m
log-uniform in [v3_box_min, v3_box_max]; centre = box centre + a shift of
up to +/- v3_shift x side; plus v2's flip and rotation. It is clamped so
the ROTATED window stays inside the stored context square, shrinking
the side first if it cannot fit -- so no synthetic black margin ever
appears, only the real frame edge the builder padded.
The window is expressed as an ordinary aug_M relative to the standard
1.2x crop (bbox_square at W px), so labels, the reprojection loss and
everything downstream are unchanged. The image is warped from the
context crop with aug_M composed with the context -> standard-crop map.
Un-augmented samples use the identity window, i.e. today's crop.
Returns (image, M_total, aug_active, flip, angle, window centre (frame
px, before flip/rotation), window side / 1.2 in frame px).
"""
sq = ann["bbox_square"].astype(np.float64)
ctx = ann["bbox_context"].astype(np.float64)
kx, ky = W / max(sq[2] - sq[0], 1.0), H / max(sq[3] - sq[1], 1.0)
M_total = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=np.float64)
aug_active, aug_flip, angle, centre, bpx = 0.0, 0.0, 0.0, None, None
if self.augment and random.random() < self.geom_p:
aug_active = 1.0
aug_flip = 1.0 if random.random() < self.geom_flip_p else 0.0
angle = random.uniform(-self.geom_rot_deg, self.geom_rot_deg)
m = math.exp(random.uniform(math.log(self.v3_box_min), math.log(self.v3_box_max)))
side = m * b_size
wx = cx + random.uniform(-self.v3_shift, self.v3_shift) * side
wy = cy + random.uniform(-self.v3_shift, self.v3_shift) * side
# Keep the rotated window inside the stored square -- but only on
# sides where that square ends INSIDE the photo. Where it already
# runs past the photo's edge, anything further out is outside the
# photo too, so the black it reads IS the real frame edge (what a
# deployment crop shows when its box leaves the frame); clamping
# there would only thin out the loose, off-centre framings v3 is for.
rad = math.radians(angle)
reach = abs(math.cos(rad)) + abs(math.sin(rad)) # rotated square's extent / side
oh, ow = (float(v) for v in ann["orig_shape"][:2])
lo_x = ctx[0] if ctx[0] >= 0 else -math.inf
hi_x = ctx[2] if ctx[2] <= ow else math.inf
lo_y = ctx[1] if ctx[1] >= 0 else -math.inf
hi_y = ctx[3] if ctx[3] <= oh else math.inf
side = min(side, min(hi_x - lo_x, hi_y - lo_y) / reach)
e = side * reach / 2.0
wx = min(max(wx, lo_x + e), hi_x - e)
wy = min(max(wy, lo_y + e), hi_y - e)
centre, bpx = (wx, wy), side / 1.2
# standard-crop px: flip, then zoom and rotate about the window
# centre and move it to the output centre -- v2's M_total form. The
# zoom is per axis: bbox_square's int() edges can make it 198x200
# px, and only per-axis zoom keeps the window SQUARE in the frame,
# which the clamp above and the 3D label rotation both assume.
c = np.array([(wx - sq[0]) * kx, (wy - sq[1]) * ky])
M_flip = (np.array([[-1.0, 0.0, float(W)], [0.0, 1.0, 0.0]]) if aug_flip
else np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]))
fc = M_flip[:, :2] @ c + M_flip[:, 2]
rot = cv2.getRotationMatrix2D((0.0, 0.0), angle, 1.0)[:, :2]
lin = rot @ np.diag([(sq[2] - sq[0]) / side, (sq[3] - sq[1]) / side])
M_rst = np.hstack([lin, (np.array([W / 2.0, H / 2.0]) - lin @ fc)[:, None]])
A2, t2 = M_rst[:, :2], M_rst[:, 2]
A1, t1 = M_flip[:, :2], M_flip[:, 2]
M_total = np.hstack([A2 @ A1, (A2 @ t1 + t2)[:, None]])
# context-crop pixel index -> standard-crop pixel index (pixel centres,
# as cv2.resize maps them on both sides; verified against standard crops)
ctx_np = np.asarray(img_pil)
sc_y, sc_x = ctx_np.shape[:2]
ax, ay = (ctx[2] - ctx[0]) / sc_x, (ctx[3] - ctx[1]) / sc_y
C = np.array([[ax * kx, 0.0, (0.5 * ax + ctx[0] - sq[0]) * kx - 0.5],
[0.0, ay * ky, (0.5 * ay + ctx[1] - sq[1]) * ky - 0.5]])
A, t = M_total[:, :2], M_total[:, 2]
M_img = np.hstack([A @ C[:, :2], (A @ C[:, 2] + t)[:, None]])
img_np = cv2.warpAffine(ctx_np, M_img, (W, H), flags=self.v3_interp,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
return Image.fromarray(img_np), M_total, aug_active, aug_flip, angle, centre, bpx
def _temporal_sample(self, npz_path, ann, aug_flip, angle):
"""Previous pose, foot contact and mocap feet for --prev-token.
All of it is built in the original camera frame and then moved with
this sample's own flip and rotation, so it lives in the same frame as
joints_3d / cam_trans. "Previous" is the annotation `frame_step * g`
frames earlier on a video split, else this sample's own GT; either way
it is noised like a prediction and withheld with probability mask_p.
Validation (augment=False) always withholds it."""
tc = self.temporal
j_cur = ann["joints_3d"].astype(np.float32)
t_cur = ann["cam_trans"].astype(np.float32)
j_prev, t_prev, has_pair = j_cur, t_cur, 0.0
fm_prev = None
split = npz_path.parent.parent.name
gap_max = next((g for k, g in tc["pair_gaps"].items() if k in split), 0)
m = PAIR_NAME.match(npz_path.stem) if (self.augment and gap_max) else None
if m is not None:
f = int(m.group(2)) - tc["frame_step"] * random.randint(1, gap_max)
p = npz_path.with_name(f"{m.group(1)}_{f:0{len(m.group(2))}d}_p{m.group(3)}.npz")
if f >= 0 and p.exists():
with np.load(p) as z:
jp, tp = z["joints_3d"].astype(np.float32), z["cam_trans"].astype(np.float32)
if "foot_mocap" in z:
fm_prev = (z["foot_mocap"].astype(np.float32) + tp,
z["foot_mocap_valid"].astype(np.float32))
# A person-id swap between the two frames shows up as a jump.
if np.linalg.norm(tp - t_cur) < 0.5:
j_prev, t_prev, has_pair = jp, tp, 1.0
else:
fm_prev = None
contact, c_valid = np.zeros(2, np.float32), np.zeros(2, np.float32)
fl = tc.get("floor")
fl = fl.get(PAIR_NAME.match(npz_path.stem).group(1)) if (
fl and PAIR_NAME.match(npz_path.stem)) else None
if "contact" in ann:
contact = ann["contact"].astype(np.float32)
c_valid = ann["contact_valid"].astype(np.float32)
elif fl is not None:
h = (((j_cur + t_cur)[FOOT_KP] - fl[1]) @ fl[0]).reshape(2, 4).min(1)
air = h > tc["floor_air_h"]
on = h < tc["floor_contact_h"]
if has_pair:
d = np.linalg.norm((j_cur + t_cur)[FOOT_KP] - (j_prev + t_prev)[FOOT_KP], axis=1)
on &= d.reshape(2, 4).mean(1) < tc["floor_contact_d"]
else:
on[:] = False # no motion to check: only "off" is certain
contact = on.astype(np.float32)
c_valid = (on | air).astype(np.float32)
elif has_pair:
d = np.linalg.norm((j_cur + t_cur)[FOOT_KP] - (j_prev + t_prev)[FOOT_KP], axis=1)
d = d.reshape(2, 4).mean(1)
contact = (d < tc["contact_thr"]).astype(np.float32)
c_valid = ((d < tc["contact_thr"]) | (d > tc["noncontact_thr"])).astype(np.float32)
fm, fm_valid = np.zeros((70, 3), np.float32), np.zeros(70, np.float32)
if "foot_mocap" in ann:
fm = np.nan_to_num(ann["foot_mocap"].astype(np.float32))
fm_valid = ann["foot_mocap_valid"].astype(np.float32)
has_prev = 1.0 if (self.augment and random.random() >= tc["mask_p"]) else 0.0
if tc.get("pairs_only") and not has_pair:
has_prev = 0.0
rng = np.random.default_rng(random.getrandbits(32))
jn = j_prev + rng.normal(0.0, tc["noise_joint"], j_prev.shape).astype(np.float32)
tn = t_prev.copy()
tn[:2] += rng.normal(0.0, tc["noise_xy"], 2).astype(np.float32) * tn[2]
tn[2] *= math.exp(rng.normal(0.0, tc["noise_logz"]))
jn, tn = _geom_points(jn, aug_flip, angle), _geom_cam(tn, aug_flip, angle)
disp = (j_cur + t_cur) - (j_prev + t_prev)
if fm_prev is not None:
# The markers' motion where both frames have them: the teacher's
# feet jitter, and a rolling foot in contact is not motionless.
both = (fm_valid * fm_prev[1]) > 0
disp[both] = ((fm + t_cur) - fm_prev[0])[both]
disp = _geom_points(disp, aug_flip, angle)[FOOT_KP]
fm = _geom_points(fm, aug_flip, angle)
if aug_flip > 0.5:
fm_valid = fm_valid[FLIP_PERM]
contact, c_valid = contact[::-1].copy(), c_valid[::-1].copy()
z = max(float(tn[2]), 0.1)
prev_pose = np.concatenate([jn.reshape(-1), [tn[0] / z, tn[1] / z, math.log(z)]])
return {
"prev_pose": torch.from_numpy(prev_pose.astype(np.float32) * has_prev),
"has_prev": torch.tensor(has_prev, dtype=torch.float32),
"has_pair": torch.tensor(has_pair, dtype=torch.float32),
"prev_feet": torch.from_numpy((jn + tn)[FOOT_KP].astype(np.float32)),
"feet_disp": torch.from_numpy(disp.astype(np.float32)),
"contact": torch.from_numpy(contact),
"contact_valid": torch.from_numpy(c_valid),
"foot_mocap": torch.from_numpy(fm.astype(np.float32)),
"foot_mocap_valid": torch.from_numpy(fm_valid.astype(np.float32)),
}
def __len__(self):
return len(self.pairs)
MAX_BAD_SAMPLES = 1000
def __getitem__(self, idx):
"""Load sample `idx`; if its files are unreadable, log the path and
substitute a random other sample instead of killing the run.
One unreadable file out of millions (an npz left empty by a builder
killed mid-write) stopped all four generation-9 runs twice, each time
at the same step because the seed fixes the sample order. On healthy
data the except branch never runs, so samples are bit-identical to
before. More than MAX_BAD_SAMPLES failures in one worker still raise:
that is a broken corpus, not a stray file."""
try:
return self._load_sample(idx)
except Exception as e: # noqa: BLE001
self._n_bad = getattr(self, "_n_bad", 0) + 1
bad = self.pairs[idx]
print(f"[dataset] unreadable sample skipped ({self._n_bad} in this worker): "
f"{bad[1]} / {bad[0]}: {type(e).__name__}: {e}", flush=True)
if self._n_bad > self.MAX_BAD_SAMPLES:
raise
return self[random.randrange(len(self))]
def _load_sample(self, idx):
img_path, npz_path = self.pairs[idx]
W = H = self.image_size
img_pil = Image.open(img_path).convert("RGB")
ann = np.load(npz_path)
orig_h, orig_w = ann["orig_shape"]
sq_bbox = ann["bbox_square"]
win = 0
if self.crop_windows:
# The tile is K windows side by side, each already at image_size;
# the chosen one's box replaces bbox_square everywhere below.
win = 0
if self.augment and self.window_mode == "random":
k = len(ann["win_boxes"])
if self.window0_p is None:
win = random.randrange(k)
elif random.random() >= self.window0_p:
win = random.randrange(1, k)
img_pil = img_pil.crop((win * W, 0, (win + 1) * W, H))
sq_bbox = ann["win_boxes"][win]
sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]
orig_crop_size = sq_bbox[2] - sq_bbox[0]
true_scale = self.image_size / max(orig_crop_size, 1.0)
joints_2d = ann["joints_2d"].copy().astype(np.float32)
joints_2d[:, 0] = (joints_2d[:, 0] - sq_x1) * true_scale
joints_2d[:, 1] = (joints_2d[:, 1] - sq_y1) * true_scale
joints_2d[:, 0] = (joints_2d[:, 0] / self.image_size) * 2.0 - 1.0
joints_2d[:, 1] = (joints_2d[:, 1] / self.image_size) * 2.0 - 1.0
# Per-keypoint 2D visibility. Present only on folders built by
# datasets_pipeline/build_split.py; older folders (and every split
# except SA-1B) are fully observed, so the all-ones default reproduces
# the previous behaviour bit for bit.
#
# SA-1B is the one split that needs this: its keypoints_2d stores
# unobserved points as literal (0, 0) with a graded confidence in the
# third column — body 100% observed, feet 97%, hands 8%. Training on it
# without the mask drags every predicted hand keypoint to the top-left
# of the frame. The 3D keypoints and the MHR params are complete, so
# only the 2D losses need it.
if "joints_2d_vis" in ann:
joints_2d_vis = ann["joints_2d_vis"].astype(np.float32).copy()
else:
joints_2d_vis = np.ones(joints_2d.shape[0], dtype=np.float32)
joints_3d = ann["joints_3d"].astype(np.float32).copy()
cam_trans = ann["cam_trans"].astype(np.float32).copy()
tight_bbox = ann["bbox"]
cx = (tight_bbox[0] + tight_bbox[2]) / 2.0
cy = (tight_bbox[1] + tight_bbox[3]) / 2.0
b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])
b_scale = b_size / max(orig_w, orig_h)
if win:
# CLIFF describes the window: its centre, and the tight box that
# gives it at the standard 1.2x expansion -- what a detector box
# framing the person this way would produce at runtime.
cx = (sq_bbox[0] + sq_bbox[2]) / 2.0
cy = (sq_bbox[1] + sq_bbox[3]) / 2.0
b_size = (sq_bbox[2] - sq_bbox[0]) / 1.2
b_scale = b_size / max(orig_w, orig_h)
aug_flip, cos_t, sin_t = 0.0, 1.0, 0.0
angle = 0.0
M_total = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=np.float64)
aug_active = 0.0
if self.context_crops:
(img_pil, M_total, aug_active, aug_flip, angle,
v3_centre, v3_bpx) = self._context_window(img_pil, ann, W, H, b_size, cx, cy)
if aug_active:
# CLIFF describes the window actually cut: its centre, and the
# tight box that would give it at the standard 1.2x expansion.
# Both are exact (no crop-space approximation), then flipped and
# rotated with the labels below.
cx, cy = v3_centre
b_scale = v3_bpx / max(orig_w, orig_h)
(joints_2d, joints_2d_vis, joints_3d, cam_trans, cx, cy,
cos_t, sin_t) = _apply_label_geometry(
M_total, aug_flip, angle, joints_2d, joints_2d_vis, joints_3d,
cam_trans, cx, cy, orig_w, orig_h, W, H)
elif self.augment and random.random() < self.geom_p:
aug_active = 1.0
img_np = np.asarray(img_pil.resize((W, H)))
if random.random() < self.geom_flip_p:
aug_flip = 1.0
M_flip = np.array([[-1.0, 0.0, float(W)], [0.0, 1.0, 0.0]])
else:
M_flip = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
angle = random.uniform(-self.geom_rot_deg, self.geom_rot_deg)
# Asymmetric by design. bbox_square is exactly 1.2x the tight box on
# every split, so the body fills 83.3% of the crop and the old
# symmetric ceiling of 1.25 shaved ~2% off each side -- it
# essentially never truncated. A ceiling of 2.0 retains the central
# quarter by area, which is MeTRAbs's bound. Zoom-OUT is still
# capped at 1-geom_scale_range because there are no pixels outside
# bbox_square; warpAffine would fill the margin with black.
scale_hi = (1.0 + self.geom_scale_range if self.geom_scale_max is None
else self.geom_scale_max)
scale = random.uniform(1.0 - self.geom_scale_range, scale_hi)
dx = random.uniform(-self.geom_trans, self.geom_trans)
dy = random.uniform(-self.geom_trans, self.geom_trans)
M_rst = cv2.getRotationMatrix2D((W / 2.0, H / 2.0), angle, scale)
M_rst[0, 2] += dx * (W / 2.0)
M_rst[1, 2] += dy * (H / 2.0)
A2, t2 = M_rst[:, :2], M_rst[:, 2]
A1, t1 = M_flip[:, :2], M_flip[:, 2]
M_total = np.hstack([A2 @ A1, (A2 @ t1 + t2)[:, None]])
img_np = cv2.warpAffine(img_np, M_total, (W, H), flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
img_pil = Image.fromarray(img_np)
(joints_2d, joints_2d_vis, joints_3d, cam_trans, cx, cy,
cos_t, sin_t) = _apply_label_geometry(
M_total, aug_flip, angle, joints_2d, joints_2d_vis, joints_3d,
cam_trans, cx, cy, orig_w, orig_h, W, H)
# CLIFF conditioning describes the box this crop came from. The flip
# and rotation above already move it; dx/dy/scale did not, which was
# harmless at +/-0.08 and 1.25 but not at truncation strength. A
# crop-space shift of dx corresponds to the box moving
# dx * crop_w / 2 original pixels, and a zoom of s shrinks the
# visible box by s -- which is exactly what an off-centre or badly
# sized detector box looks like at deployment.
if self.cliff_follows_aug:
crop_w = float(sq_bbox[2] - sq_bbox[0])
# The warp is u' = s*R*(u - c) + c + t, so inverting it puts the
# visible window's centre at -t/s, not at -t: the shift has to be
# divided by the zoom. Without the /s the correction is too big
# when zooming in and too small when zooming out -- measured over
# 972 augmented samples at v2 strength (geom_trans 0.20,
# geom_scale_max 2.0): 11.7 px mean / 33.6 px p95 of error in the
# box centre, i.e. 0.61 deg mean / 1.73 deg p95 in the bearing the
# network is conditioned on. It is a wrong INPUT, not a wrong
# label -- the reprojection chain is exact to 0.000 px -- and it
# only affects lateral placement, which PA-MPJPE cannot see.
#
# Default off so `--preset baseline` and every gen-5 run stay
# bit-identical; `--crop-centre-fix` turns it on.
z = scale if getattr(self, "crop_centre_fix", False) else 1.0
cx -= dx * crop_w / (2.0 * z)
cy -= dy * crop_w / (2.0 * z)
b_scale = b_scale / scale
image = self.transform(img_pil)
if self.cliff_focal:
# Where the box sits in the FIELD OF VIEW, and how large it is
# ANGULARLY -- not in image-normalised pixels.
#
# Under a fixed focal the two are interchangeable, and the first
# corpus had exactly that: every crop carried the synthetic
# f = sqrt(H^2+W^2), i.e. f/diag = 1.000 with zero spread. The corpus
# the datasets' own focals instead, and those span f/diag 0.6-1.8 on
# coco/aic/mpii (sd ~0.35) with harmony4d off at 0.285. There the
# pixel version is ambiguous: two crops with identical conditioning
# correspond to different depths, the network can only predict the
# conditional mean, and the residual leaves as unstable cam_trans
# and body scale -- measured on 3DPW test as 220 mm/frame^2 of
# translation jitter (98% of it in depth) against 148 for a model
# trained on the constant-focal corpus.
#
# One scalar focal (fy) rather than fx/fy, because the rotation
# augmentation mixes the two axes and fx != fy on the calibrated
# splits.
f = float(ann["cam_focal_length"][1])
b_px = b_scale * max(orig_w, orig_h)
cliff_cond = torch.tensor(
[math.atan((cx - orig_w / 2.0) / f),
math.atan((cy - orig_h / 2.0) / f),
b_px / f], dtype=torch.float32)
else:
cx_norm = 2.0 * (cx / orig_w) - 1.0
cy_norm = 2.0 * (cy / orig_h) - 1.0
cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)
extra = ({} if self.temporal is None
else self._temporal_sample(Path(npz_path), ann, aug_flip, angle))
return {
**extra,
"image": image,
"cliff_cond": cliff_cond,
"mhr_model_params": torch.from_numpy(ann["mhr_model_params"]).float(),
"shape_params": torch.from_numpy(ann["shape_params"]).float(),
"cam_trans": torch.from_numpy(cam_trans).float(),
"cam_focal": torch.from_numpy(ann["cam_focal_length"]).float(),
"joints_2d": torch.from_numpy(joints_2d).float(),
"joints_2d_vis": torch.from_numpy(joints_2d_vis).float(),
"joints_3d": torch.from_numpy(joints_3d).float(),
"orig_shape": torch.tensor([orig_h, orig_w], dtype=torch.float32),
"bbox_square": sq_bbox,
"aug_active": torch.tensor(aug_active, dtype=torch.float32),
"aug_flip": torch.tensor(aug_flip, dtype=torch.float32),
"aug_cos": torch.tensor(cos_t, dtype=torch.float32),
"aug_sin": torch.tensor(sin_t, dtype=torch.float32),
"aug_M": torch.from_numpy(M_total.astype(np.float32).reshape(-1)),
}
def build_dataloaders(cfg):
geom = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,
geom_scale_range=cfg.geom_scale_range,
geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p,
geom_scale_max=cfg.geom_scale_max,
cliff_follows_aug=cfg.cliff_follows_aug,
crop_centre_fix=cfg.crop_centre_fix,
cliff_focal=cfg.cliff_focal,
occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality,
image_size=cfg.image_size,
context_crops=cfg.context_crops, v3_box_min=cfg.v3_box_min,
v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift,
v3_lanczos=cfg.v3_lanczos, crop_windows=cfg.crop_windows,
window_mode=cfg.window_mode, window0_p=cfg.window0_p,
temporal=temporal_kwargs(cfg))
full_dataset = SAM3DStudentDataset(
cfg.data_root,
augment=cfg.augment,
max_images=cfg.max_images,
per_dataset_caps=cfg.per_dataset_caps,
**geom,
)
val_dataset_clean = SAM3DStudentDataset(
cfg.data_root,
augment=False,
max_images=cfg.max_images,
per_dataset_caps=cfg.per_dataset_caps,
**geom,
)
n_val = int(len(full_dataset) * cfg.val_split)
n_train = len(full_dataset) - n_val
generator = torch.Generator().manual_seed(42)
train_dataset, _ = random_split(full_dataset, [n_train, n_val], generator=generator)
generator = torch.Generator().manual_seed(42)
_, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=generator)
loader_kwargs = dict(num_workers=cfg.num_workers, pin_memory=True)
if cfg.num_workers > 0:
loader_kwargs["persistent_workers"] = True
loader_kwargs["prefetch_factor"] = 4
train_loader = DataLoader(
train_dataset,
batch_size=cfg.batch_size,
shuffle=True,
drop_last=True,
**loader_kwargs,
)
val_loader = DataLoader(
val_dataset,
batch_size=cfg.batch_size,
shuffle=False,
**loader_kwargs,
)
print(f"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}")
return train_loader, val_loader, full_dataset
# ============================================================
# Cell 7 — Student Architecture (Mini-SAM3D with PE)
# ============================================================
def get_2d_sincos_pos_embed(embed_dim, grid_size):
grid_h, grid_w = grid_size, grid_size
grid_y, grid_x = torch.meshgrid(torch.arange(grid_h), torch.arange(grid_w), indexing='ij')
omega = torch.arange(embed_dim // 4).float() / (embed_dim // 4)
omega = 1.0 / (10000 ** omega)
out_y = grid_y.flatten().unsqueeze(1) * omega.unsqueeze(0)
out_x = grid_x.flatten().unsqueeze(1) * omega.unsqueeze(0)
pe_y = torch.cat([torch.sin(out_y), torch.cos(out_y)], dim=1)
pe_x = torch.cat([torch.sin(out_x), torch.cos(out_x)], dim=1)
pos_embed = torch.cat([pe_y, pe_x], dim=1)
return pos_embed.unsqueeze(0)
class ContMHRHead(nn.Module):
"""SAM 3D Body's continuous pose head, emitting the rig-native 204-vector.
447 numbers come out of a two-layer MLP and are converted here, so the rest
of the model, every loss, the ONNX graph and `instanthmr.inference` all keep
the contract they already have: 204 MHR parameters and 45 identity
coefficients. (The teacher emits 519; the extra 72 are a face block its own
`forward()` multiplies by zero.)
6 root rotation, 6D -> 3 extrinsic-XYZ Euler angles
260 body pose, continuous -> 130 angles
45 identity -> passed through
28 bone-scale PCA coefficients -> 68 bone scales
108 two 54-dim hand blocks -> overwrite the 54 finger channels
Three things are deliberate and each is measured in the file that owns it:
* **Root translation is emitted as exact zero**, not regressed. It is
`0.000e+00` in all 24,000 sampled annotations -- the person is placed by
`cam_trans` -- and the teacher's head does the same (`global_trans =
torch.zeros_like(...)`).
* **The bone-scale bound is on the 28 coefficients, not the 68 scales.**
Clamping after the PCA expansion is what `--bound-scales` does and it is
wrong here: an arbitrary point of the 68-dim box is generally not in the
24-dimensional column space `scale_comps` spans, so a coordinatewise tanh
silently leaves the subspace. The coefficient limits come from the corpus
(`tools/build_cont_head_assets.py`; GT projection residual 2.1e-08).
* **The six `*_length/_width_flexible` channels at `130:136` are bounded
separately.** The continuous body space carries them raw -- they are
joint TRANSLATION channels, not rotations -- so they are exactly as
unbounded here as they were before, and they reach 375 m on their own.
Assembly is one `index_select` over a fixed 136-long gather table rather
than three masked writes: `x[..., idx] = v` traces to a scatter, a gather
exports everywhere and is its own documentation.
"""
def __init__(self, cfg, d_model: int):
super().__init__()
a = np.load(cfg.cont_head_path)
self.register_buffer("scale_mean", torch.from_numpy(a["scale_mean"]).float())
self.register_buffer("scale_comps", torch.from_numpy(a["scale_comps"]).float())
self.register_buffer("coeff_lo", torch.from_numpy(a["coeff_lo"]).float())
self.register_buffer("coeff_hi", torch.from_numpy(a["coeff_hi"]).float())
self.register_buffer("hand_mean", torch.from_numpy(a["hand_pose_mean"]).float())
self.register_buffer("hand_comps", torch.from_numpy(a["hand_pose_comps"]).float())
# One gather that builds the 136-vector [trans(3), root(3), body(130)]
# with the finger channels already replaced by the hand blocks. Source
# layout: [zeros(3), root_euler(3), body(130), left(27), right(27)].
g = torch.arange(136)
for j, t in enumerate(a["hand_joint_idxs_left"].tolist()):
g[t] = 136 + j
for j, t in enumerate(a["hand_joint_idxs_right"].tolist()):
g[t] = 163 + j
assert len(set(g.tolist())) == 136, "hand index sets overlap"
self.register_buffer("full_gather", g)
# Two-layer MLP at the student's own width, on normalised pose-token
# features: `nn.TransformerDecoder` is built here with norm_first=True
# layers and no final norm, so its output is unnormalised.
self.norm = nn.LayerNorm(d_model)
self.mlp = nn.Sequential(
nn.Linear(d_model, d_model), nn.GELU(), nn.Linear(d_model, MC.N_HEAD_OUT))
# Learned initial estimate, added to the MLP output. Initialised to the
# neutral encoding: all-zeros is not a valid point of this space -- a
# zero 6D vector has no rotation and a zero (sin, cos) pair puts atan2
# at a 0/0 gradient.
self.init_estimate = nn.Parameter(MC.neutral_head_output())
nn.init.normal_(self.mlp[-1].weight, mean=0.0, std=1e-4)
nn.init.zeros_(self.mlp[-1].bias)
self.bound_scales = bool(getattr(cfg, "bound_scales", False))
if self.bound_scales:
b = np.load(cfg.scale_bounds_path)
# mhr_size_bounds.npz stores 0:3 then 130:204; take the six
# flexible channels, which are the first six of the second slice.
lo = torch.from_numpy(b["lo"]).float()
hi = torch.from_numpy(b["hi"]).float()
m = torch.clamp(0.25 * (hi - lo), min=float(cfg.scale_bound_margin))
lo, hi = lo - m, hi + m
starts = b["starts"].tolist()
off = 0
for st, sp in zip(starts, b["stops"].tolist()):
if st == 130:
break
off += sp - st
self.register_buffer("flex_lo", lo[off:off + 6])
self.register_buffer("flex_hi", hi[off:off + 6])
@staticmethod
def _to_range(x, lo, hi):
return lo + (hi - lo) * 0.5 * (torch.tanh(x) + 1.0)
def forward(self, feat):
"""(B, d_model) pose token -> (204) MHR params, (45) identity, (3,3) root R."""
return self.decode(self.mlp(self.norm(feat)) + self.init_estimate)
def decode(self, pred):
"""(B, 447) continuous -> (204) MHR params, (45) identity, (3,3) root R.
Split out of `forward` so `tools/verify_cont_head.py` can drive the
conversion with hand-built vectors -- a second copy of this arithmetic
would be free to drift away from the one that trains.
"""
B = pred.shape[0]
root_6d = pred[:, :6]
body_cont = pred[:, 6:6 + MC.N_BODY_CONT]
o = 6 + MC.N_BODY_CONT
shape = pred[:, o:o + MC.N_SHAPE]; o += MC.N_SHAPE
coeff = pred[:, o:o + MC.N_SCALE_COEFF]; o += MC.N_SCALE_COEFF
hand_l = pred[:, o:o + MC.N_HAND_CONT]
hand_r = pred[:, o + MC.N_HAND_CONT:o + 2 * MC.N_HAND_CONT]
R_root = MC.rot6d_to_rotmat(root_6d)
root_euler = MC.rotmat_to_euler_xyz(R_root)
body = MC.cont_to_body_params(body_cont)[:, :130]
if self.bound_scales:
# 130:136 are size, not pose, and the continuous space leaves them raw.
body = torch.cat(
[body[:, :130 - 6],
self._to_range(body[:, 130 - 6:], self.flex_lo, self.flex_hi)], dim=1)
left = MC.cont_to_hand_params(self.hand_mean + hand_l @ self.hand_comps)
right = MC.cont_to_hand_params(self.hand_mean + hand_r @ self.hand_comps)
zeros3 = torch.zeros(B, 3, dtype=pred.dtype, device=pred.device)
src = torch.cat([zeros3, root_euler, body, left, right], dim=1)
full136 = src.index_select(1, self.full_gather)
if self.bound_scales:
coeff = self._to_range(coeff, self.coeff_lo, self.coeff_hi)
scales = self.scale_mean + coeff @ self.scale_comps
# R_root is returned so the root loss can be taken on the rotation the
# head actually built. Going back through the Euler triple would put
# atan2 in the gradient path, and 2.25% of corpus roots sit within
# 0.1 rad of the +-pi/2 gimbal singularity where that is unstable.
return torch.cat([full136, scales], dim=1), shape, R_root
class InstantHMRStudent(nn.Module):
def __init__(self, cfg, pretrained=True):
super().__init__()
self.cfg = cfg
self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)
embed_dim = self.backbone.num_features
# forward_features() stops before the classifier head, so any parameter
# left in that head never receives a gradient and DDP (which runs with
# find_unused_parameters=False) aborts on the second step with
# "Expected to have finished reduction in the prior iteration".
# num_classes=0 empties the head for repvit_m2_3 but NOT for hgnetv2_b4,
# which keeps a 2048x2048 last_conv -- 4,194,304 parameters, DDP index
# 241. That killed all four generation-7 hgnet jobs at step 2.
# No-op for a head that is already parameterless, so the parameter list,
# its order and the state_dict of every existing run are untouched.
head = getattr(self.backbone, "head", None)
if head is not None and any(p.numel() for p in head.parameters()):
self.backbone.head = nn.Identity()
self.feat_proj = nn.Linear(embed_dim, cfg.d_model)
self.grid_size = cfg.image_size // 32
pos_embed = get_2d_sincos_pos_embed(cfg.d_model, self.grid_size)
self.register_buffer("mem_pos_embed", pos_embed)
self.num_global = 1
self.num_2d = cfg.num_joints
# No 3D queries: the 3D keypoints are derived from the MHR forward pass,
# exactly like the teacher does. 141 -> 71 decoder tokens.
self.total_queries = self.num_global + self.num_2d
self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model) * 0.02)
self.cond_proj = nn.Sequential(
nn.Linear(cfg.cliff_dim, 128),
nn.GELU(),
nn.Linear(128, cfg.d_model)
)
decoder_layer = nn.TransformerDecoderLayer(
d_model=cfg.d_model,
nhead=cfg.n_heads,
dim_feedforward=cfg.d_model * 4,
dropout=cfg.dropout,
activation="gelu",
batch_first=True,
norm_first=True
)
self.transformer = nn.TransformerDecoder(decoder_layer, num_layers=cfg.n_decoder_layers)
# The continuous pathway replaces head_global outright rather than
# leaving it unused: an orphaned parameter receives no gradient and
# aborts DDP on step 2 (see tools/ddp_smoke.py).
self.cont_head = ContMHRHead(cfg, cfg.d_model) \
if getattr(cfg, "cont_head", False) else None
if self.cont_head is not None:
self.head_cam = nn.Linear(cfg.d_model, cfg.cam_dim)
else:
self.global_out_dim = cfg.model_params_dim + cfg.shape_dim + cfg.cam_dim
self.head_global = nn.Linear(cfg.d_model, self.global_out_dim)
self.kp2d_bins = getattr(cfg, 'kp2d_bins', 96)
self.kp2d_range = getattr(cfg, 'kp2d_range', 1.5)
self.head_2d_feat = nn.Sequential(
nn.Linear(cfg.d_model, 256),
nn.GELU()
)
self.head_2d_logits = nn.Linear(256, 2 * self.kp2d_bins)
self.register_buffer(
"kp2d_bin_centers",
torch.linspace(-self.kp2d_range, self.kp2d_range, self.kp2d_bins))
if self.cont_head is not None:
# Same camera start as head_global's: 2.0 on depth, 0 on x/y.
nn.init.normal_(self.head_cam.weight, mean=0.0, std=1e-4)
nn.init.zeros_(self.head_cam.bias)
nn.init.constant_(self.head_cam.bias[-1], 2.0)
else:
nn.init.constant_(self.head_global.bias[-1], 2.0)
nn.init.constant_(self.head_global.bias[-2], 0.0)
nn.init.constant_(self.head_global.bias[-3], 0.0)
nn.init.normal_(self.head_global.weight, mean=0.0, std=1e-4)
nn.init.constant_(self.head_global.bias[:-3], 0.0)
nn.init.normal_(self.head_2d_logits.weight, mean=0.0, std=1e-4)
nn.init.constant_(self.head_2d_logits.bias, 0.0)
# --prev-token: one query appended AFTER the 71, so indices 0..70 keep
# their meaning and a base checkpoint loads into them unchanged. The
# projection's last layer starts at zero, so a withheld and a supplied
# previous pose start out identical and the fine-tune departs from the
# base model's function only as far as one extra query moves it.
self.prev_token = bool(getattr(cfg, "prev_token", False))
if self.prev_token:
self.prev_proj = nn.Sequential(
nn.Linear(PREV_DIM, 256), nn.GELU(), nn.Linear(256, cfg.d_model))
nn.init.zeros_(self.prev_proj[2].weight)
nn.init.zeros_(self.prev_proj[2].bias)
self.prev_query = nn.Parameter(torch.randn(1, 1, cfg.d_model) * 0.02)
self.prev_mask_embed = nn.Parameter(torch.zeros(1, cfg.d_model))
self.contact_head = (nn.Linear(cfg.d_model, 2)
if getattr(cfg, "contact_head", False) else None)
if self.contact_head is not None:
# Default init gives logits of +-10 on feat_global's scale, a BCE of
# 6.3 on a base checkpoint; this starts it at log 2 = 0.69.
nn.init.normal_(self.contact_head.weight, mean=0.0, std=1e-4)
nn.init.zeros_(self.contact_head.bias)
# Bone-scale bound. Buffers, not constants, so they survive
# state_dict round-trips and bake into the ONNX graph; tanh exports.
self.bound_scales = bool(getattr(cfg, "bound_scales", False))
if self.bound_scales:
b = np.load(cfg.scale_bounds_path)
# 130, not pose_dim=136: params 130:136 are the six
# *_length/*_width_flexible parameters. They sit at the end of the
# pose block but they drive joint TRANSLATION channels with weights
# up to 10.0, not rotations, so they are body-size parameters and
# are exactly as unbounded as the 68 scales. Leaving them free
# still let a blown-up head reach a 59 m skeleton with the scales
# already clamped -- measured.
# Two slices, not one: 0:3 is root translation (no rig limit, but
# exactly 0.000e+00 in all 24,000 sampled annotations across all six
# splits -- the person is placed by cam_trans, not by these) and
# 130:204 is the size block. Measured: with the size block clamped
# but root translation free, a blown-up head still reached a 375 m
# skeleton.
self.bound_slices = list(zip(b["starts"].tolist(), b["stops"].tolist()))
lo = torch.from_numpy(b["lo"]).float()
hi = torch.from_numpy(b["hi"]).float()
m = torch.clamp(0.25 * (hi - lo), min=float(cfg.scale_bound_margin))
self.register_buffer("scale_lo", lo - m)
self.register_buffer("scale_hi", hi + m)
def forward(self, images, cliff_cond, prev_pose=None, has_prev=None):
B = images.shape[0]
img_feats = self.backbone.forward_features(images)
if img_feats.dim() == 4:
img_feats = img_feats.flatten(2).transpose(1, 2)
memory = self.feat_proj(img_feats)
memory = memory + self.mem_pos_embed
queries = self.query_embed.expand(B, -1, -1)
cond = self.cond_proj(cliff_cond).unsqueeze(1)
queries = queries + cond
if self.prev_token:
# No previous pose given (validation, a first frame) = withheld.
if prev_pose is None:
prev_pose = cliff_cond.new_zeros(B, PREV_DIM)
has_prev = cliff_cond.new_zeros(B)
h = has_prev.view(B, 1).to(prev_pose.dtype)
tok = h * self.prev_proj(prev_pose) + (1.0 - h) * self.prev_mask_embed
queries = torch.cat([queries, self.prev_query + tok.unsqueeze(1) + cond], dim=1)
shared_feats = self.transformer(tgt=queries, memory=memory)
feat_global = shared_feats[:, 0, :]
feat_2d = shared_feats[:, 1 : 1 + self.num_2d, :]
if self.cont_head is not None:
# Bounds live inside ContMHRHead: the bone scales are clamped as PCA
# coefficients so the expansion stays in the teacher's subspace, and
# root translation comes out as exact zero rather than regressed.
pred_mhr_params, pred_shape_params, pred_root_rotmat = \
self.cont_head(feat_global)
pred_cam_trans = self.head_cam(feat_global)
else:
global_preds = self.head_global(feat_global)
idx_mhr = self.cfg.model_params_dim
idx_shape = idx_mhr + self.cfg.shape_dim
pred_mhr_params = global_preds[:, :idx_mhr]
pred_shape_params = global_preds[:, idx_mhr : idx_shape]
pred_cam_trans = global_preds[:, idx_shape :]
if self.bound_scales and self.cont_head is None:
# tanh(0) = 0 maps a zero-init head to the middle of each interval,
# which is ~0 because they are near-symmetric, so this does not move
# the starting point of a fresh run. Static python loop over two
# slices: it unrolls under tracing, so it exports to ONNX.
parts, prev, off = [], 0, 0
for a, b_ in self.bound_slices:
if a > prev:
parts.append(pred_mhr_params[:, prev:a])
n = b_ - a
t = torch.tanh(pred_mhr_params[:, a:b_])
lo = self.scale_lo[off:off + n]
hi = self.scale_hi[off:off + n]
parts.append(lo + (hi - lo) * 0.5 * (t + 1.0))
prev, off = b_, off + n
if prev < pred_mhr_params.shape[1]:
parts.append(pred_mhr_params[:, prev:])
pred_mhr_params = torch.cat(parts, dim=1)
feat_2d_processed = self.head_2d_feat(feat_2d)
logits_2d = self.head_2d_logits(feat_2d_processed)
logits_2d = logits_2d.unflatten(-1, (2, self.kp2d_bins))
# softmax in fp32 (autocast already promotes it; explicit for no-autocast/bf16/export paths)
prob_2d = torch.softmax(logits_2d.float(), dim=-1)
pred_joints_2d = (prob_2d * self.kp2d_bin_centers.float()).sum(dim=-1)
out = {
"mhr_params": pred_mhr_params,
"shape_params": pred_shape_params,
"cam_trans": pred_cam_trans,
"joints_2d": pred_joints_2d,
"joints_2d_logits": logits_2d,
}
if self.contact_head is not None:
out["contact_logits"] = self.contact_head(feat_global)
if self.cont_head is not None:
# Training-only. The deploy wrapper names its four outputs
# explicitly, so this never reaches the ONNX graph.
out["root_rotmat"] = pred_root_rotmat
return out
# ============================================================
# Cell 8 — The Distillation Loss Module (Foolproof Edition)
# ============================================================
class MHRForwardPass:
"""MHR skeleton forward + the linear map onto the 70 native keypoints.
Only the SKELETON branch is evaluated (model_parameters_to_joint_parameters
followed by joint_parameters_to_skeleton_state). No skinning, no vertices:
the 70 keypoints come from an affine regressor over the 127 skeleton joints
instead of the teacher's (70, 18566) vertex mapping. That is ~11x cheaper
per training step and makes the mesh LOD irrelevant to training cost.
"""
def __init__(self, mhr_path, device, kp_regressor=None, n_verts: int = 0,
landmark_assets: str | None = None):
self.device = device
self.mhr = torch.jit.load(mhr_path, map_location=device).eval()
for p in self.mhr.parameters():
p.requires_grad_(False)
self.kp_regressor = None
if kp_regressor is not None:
W = torch.as_tensor(kp_regressor, dtype=torch.float32, device=device)
assert W.shape == (70, 127), f"expected a (70, 127) regressor, got {tuple(W.shape)}"
self.kp_regressor = W
# The teacher's exact landmark readout, when asked for: K = Wj @ J + Wv @ V
# over the 468 mesh vertices its first 70 mapping rows actually touch.
self.lm_W_vert = self.lm_W_joint = None
lm_verts = None
if landmark_assets is not None:
a = np.load(landmark_assets)
lm_verts = torch.as_tensor(a["vert_idx"], dtype=torch.int64, device=device)
self.lm_W_vert = torch.as_tensor(a["W_vert"], dtype=torch.float32, device=device)
self.lm_W_joint = torch.as_tensor(a["W_joint"], dtype=torch.float32, device=device)
self.vert_idx = None
self.verts_loss_pos = None # where the w_verts samples sit in the subset
self.lm_vert_pos = None # where the landmark vertices sit, in W_vert order
if n_verts or lm_verts is not None:
self._build_vertex_subset(int(n_verts), lm_verts)
def _build_vertex_subset(self, n: int, extra: torch.Tensor | None = None):
"""A farthest-point subset of the rig mesh, plus a skinning op for it.
`extra` is an additional index set to include -- the 468 vertices the
teacher's landmark mapping references. The subset becomes the UNION, so
the two consumers are independent: `loss_verts` still averages over
exactly the `n` farthest-point samples it always did, with the same
uniform weighting, via `verts_loss_pos`. The 468 are not a subset of the
595 and a larger `n` does not contain them -- they have to be named.
The rig's `linear_blend_skinning` asserts it is handed all 18,439
vertices, so the subset is installed by rewriting its three flattened
influence tables -- (vertex, bone, weight) triples -- to reference only
the chosen vertices under a compacted index. The op itself is
untouched, which is why the result is the full mesh's answer restricted
to those vertices rather than an approximation of it.
Deterministic: the seed vertex is fixed and the input mesh is fixed, so
every DDP rank builds the identical subset without communicating.
"""
ct = self.mhr.character_torch
rest = ct.mesh.rest_vertices.to(self.device)
assert n <= rest.shape[0], f"n_verts={n} exceeds the {rest.shape[0]}-vertex mesh"
# Farthest-point sampling in float64: the mesh spans ~180 cm and
# neighbouring vertices are sub-millimetre apart, so float32 ties would
# make the traversal order depend on accumulation noise.
fps = None
if n:
P = rest.double()
idx = [0]
d = (P - P[0]).norm(dim=-1)
for _ in range(n - 1):
i = int(d.argmax())
idx.append(i)
d = torch.minimum(d, (P - P[i]).norm(dim=-1))
fps = torch.tensor(sorted(idx), device=self.device)
parts = [t for t in (fps, extra.to(self.device) if extra is not None else None)
if t is not None]
assert parts, "_build_vertex_subset called with neither n_verts nor extra"
sub = torch.unique(torch.cat(parts))
n_sub = int(sub.numel())
# Positions, not vertex ids: both consumers index the skinned output.
# searchsorted is exact here because `sub` is sorted and contains both.
self.verts_loss_pos = torch.searchsorted(sub, fps) if fps is not None else None
self.lm_vert_pos = (torch.searchsorted(sub, extra.to(self.device))
if extra is not None else None)
lbs = copy.deepcopy(ct.linear_blend_skinning)
vi = ct.linear_blend_skinning.vert_indices_flattened
keep = torch.isin(vi, sub)
remap = torch.full((int(vi.max()) + 1,), -1, dtype=torch.int64, device=self.device)
remap[sub] = torch.arange(n_sub, device=self.device)
lbs.vert_indices_flattened = remap[vi[keep]]
lbs.skin_indices_flattened = ct.linear_blend_skinning.skin_indices_flattened[keep]
lbs.skin_weights_flattened = ct.linear_blend_skinning.skin_weights_flattened[keep]
lbs.num_vertices = n_sub
self.vert_idx, self.vert_lbs = sub, lbs
def get_joints(self, model_params, shape_params):
"""(B, 127, 8) skeleton state in raw MHR units (cm, Y-up)."""
B = model_params.shape[0]
dummy_identity = torch.zeros(B, 45, device=model_params.device, dtype=model_params.dtype)
concat_params = torch.cat([model_params, dummy_identity], dim=1)
joint_params = self.mhr.character_torch.model_parameters_to_joint_parameters(concat_params.to(self.device))
skel_state = self.mhr.character_torch.joint_parameters_to_skeleton_state(joint_params)
return skel_state.to(model_params.device)
def get_joints_and_vertices(self, model_params, shape_params):
"""(B, 127, 3) joints and (B, n_verts, 3) skin, both raw MHR cm.
One forward kinematics pass feeds both. Unlike `get_joints` this passes
the REAL identity coefficients rather than zeros -- which changes the
skeleton by exactly nothing (verified bit-identical, `torch.equal`),
because the 45 identity coefficients are mesh-only -- while the mesh
needs them, since they are the only thing this term exists to supervise.
"""
assert self.vert_idx is not None, "MHRForwardPass was built with n_verts=0"
ct = self.mhr.character_torch
cat = torch.cat([model_params, shape_params], dim=1).to(self.device)
skel = ct.joint_parameters_to_skeleton_state(
ct.model_parameters_to_joint_parameters(cat))
rest = ct.blend_shape(shape_params.to(self.device))[:, self.vert_idx]
verts = self.vert_lbs(skel, rest)
return skel[..., :3].to(model_params.device), verts.to(model_params.device)
@staticmethod
def to_vision(joints):
"""Raw MHR (cm, Y-up/Z-back) -> annotation frame (metres, Y-down/Z-fwd)."""
j = joints / 100.0
return torch.stack([j[..., 0], -j[..., 1], -j[..., 2]], dim=-1)
def regress_keypoints(self, joints_vision):
"""(B, 127, 3) skeleton joints -> (B, 70, 3) native keypoints."""
assert self.kp_regressor is not None, (
"no keypoint regressor loaded — run --fit-regressor or pass kp_regressor_path")
return torch.einsum('kj,bjc->bkc', self.kp_regressor, joints_vision)
def regress_keypoints_exact(self, joints_vision, verts_vision):
"""The teacher's own landmark readout: `Wj @ J + Wv @ V`.
`joints_vision` is (B, 127, 3) and `verts_vision` the (B, n_sub, 3)
skinned subset, both already through `to_vision`. That is legitimate
rather than a shortcut: every row of the mapping sums to exactly 1.0, so
the readout is translation-equivariant, and `to_vision` is a diagonal
scale-and-flip, so the two commute. Verified against the full
(70, 18566) mapping in `tools/verify_exact_landmarks.py`.
21 of the 70 landmarks have NO joint contribution -- nose, elbows, toe
tips, acromion and other surface points -- which is why a fitted
skeleton-only matrix cannot reproduce them and why these carry a
gradient into `shape_params`, unlike `regress_keypoints`.
"""
assert self.lm_W_vert is not None, (
"MHRForwardPass was built without landmark_assets")
v = verts_vision.index_select(1, self.lm_vert_pos)
return (torch.einsum('kj,bjc->bkc', self.lm_W_joint, joints_vision)
+ torch.einsum('kv,bvc->bkc', self.lm_W_vert, v))
def get_native_keypoints(self, model_params, shape_params, exact: bool = False):
"""(B, 70, 3) native keypoints in the annotation frame.
`exact=True` switches from the fitted (70, 127) skeleton matrix to the
teacher's own `Wj @ J + Wv @ V` readout -- the operator the stored
annotations came out of, and therefore the one the MHR70 -> J14 / SMPL24
adapters were fitted against. It needs a module built with
`landmark_assets=`. Default off: the fitted readout is what the exported
graph's consumer uses, so it is what the published metric measures.
"""
if not exact:
j = self.get_joints(model_params, shape_params)[..., :3]
return self.regress_keypoints(self.to_vision(j))
j, v = self.get_joints_and_vertices(model_params, shape_params)
return self.regress_keypoints_exact(self.to_vision(j), self.to_vision(v))
class DistillationLoss(nn.Module):
def __init__(self, cfg, mhr_module):
super().__init__()
self.cfg = cfg
self.mhr_module = mhr_module
self.mse = nn.MSELoss()
self.l1 = nn.SmoothL1Loss()
bin_w = 2.0 * cfg.kp2d_range / (cfg.kp2d_bins - 1)
self.simcc_sigma = 2.0 * bin_w
self._global_step = 0
self.warmup_steps = int(getattr(cfg, "kp3d_warmup_steps", 0))
# Per-keypoint weight for the FK-derived geometric losses. All ones
# unless finger_weight != 1, so the unweighted path stays bit-identical.
#
# Built on the MHR module's device, NOT the CPU default: every call site
# in this repo constructs DistillationLoss without .to(device)
# (run_overfit_test, run_self_tests, train_instant_hmr and
# train_ddp.train all do), so a CPU buffer would raise on the
# first step. register_buffer still lets an explicit .to() move it.
w = torch.ones(cfg.num_joints, device=getattr(mhr_module, "device", "cpu"))
w[torch.from_numpy(FINGER_KP).to(w.device)] = float(getattr(cfg, "finger_weight", 1.0))
self.register_buffer("kp3d_weight", w, persistent=False)
def set_step(self, step: int):
"""Drives the FK-loss warm-up ramp. Call once per optimiser step."""
self._global_step = int(step)
@property
def fk_scale(self) -> float:
"""0 -> 1 linear ramp on the losses that back-propagate through the FK."""
if self.warmup_steps <= 0:
return 1.0
return min(1.0, self._global_step / float(self.warmup_steps))
@staticmethod
def _mmean(per_sample, mask):
return (per_sample * mask).sum() / mask.sum().clamp_min(1.0)
def m_smooth_l1(self, pred, tgt, mask):
e = F.smooth_l1_loss(pred, tgt, reduction='none').flatten(1).mean(1)
return self._mmean(e, mask)
def _m_beta(self, pred, tgt, mask, beta):
"""m_smooth_l1 with an explicit beta. beta=1.0 is F.smooth_l1_loss's
default, so this reproduces m_smooth_l1 exactly at the default."""
e = F.smooth_l1_loss(pred, tgt, reduction='none', beta=beta).flatten(1).mean(1)
return self._mmean(e, mask)
def root_chordal(self, preds, pred_pose, tgt_pose):
"""Per-sample squared chordal distance between the two root rotations.
`||Rp - Rt||_F^2 / 4 == 1 - cos(theta)`, which is `theta^2 / 2` for
small errors -- the same curvature SmoothL1 at beta=1.0 has on one
angle, so swapping the metric does not silently rescale the term -- and
saturates at 2 instead of growing, so a half-turn cannot dominate a
batch. Chosen over an `acos` geodesic, whose derivative is unbounded at
theta = pi; this one is smooth everywhere (its gradient vanishes at pi,
a measure-zero stationary point every chordal rotation loss has).
Uses the rotation the head actually built when there is one. Rebuilding
it from the predicted Euler triple would put `atan2` in the backward
path, and the corpus sits near the +-pi/2 gimbal singularity often
enough (2.25% within 0.1 rad) for that to matter.
"""
Rp = preds.get("root_rotmat")
if Rp is None:
Rp = MC.euler_xyz_to_rotmat(pred_pose[:, 3:6])
Rt = MC.euler_xyz_to_rotmat(tgt_pose[:, 3:6])
# Divided by the 6 root parameters the old term averaged over, so this
# swaps the METRIC without also rescaling the loss budget. Measured
# without it the chordal term is 6x larger at every angle (0.00015 vs
# 0.00003 at 1 deg, 1.99985 vs 0.43736 at 179 deg), which is exactly
# that dilution factor -- three of those six parameters are the root
# translation, which is identically zero. `w_root_rot` scales from here.
return (Rp - Rt.detach()).pow(2).flatten(1).sum(1) * (0.25 / POSE_ROOT_DIM)
def m_mse(self, pred, tgt, mask):
e = F.mse_loss(pred, tgt, reduction='none').flatten(1).mean(1)
return self._mmean(e, mask)
def m_l1(self, pred, tgt, mask):
e = (pred - tgt).abs().flatten(1).mean(1)
return self._mmean(e, mask)
@staticmethod
def _kp_weighted(per_kp, w_kp, m_sample=None):
"""Mean of a (B, K) error under per-keypoint weights and a sample mask.
With w_kp all ones and no sample mask this is exactly `.mean()`, so the
masked 2D losses below are numerically identical to the unmasked ones
on every split that is fully observed.
"""
w = w_kp if m_sample is None else w_kp * m_sample.view(-1, 1)
return (per_kp * w).sum() / w.sum().clamp_min(1e-6)
def _simcc_ce(self, logits, tgt_2d, w_kp=None):
bins = torch.linspace(-self.cfg.kp2d_range, self.cfg.kp2d_range,
self.cfg.kp2d_bins, device=logits.device)
tgt = tgt_2d.clamp(-self.cfg.kp2d_range, self.cfg.kp2d_range)
d = bins.view(1, 1, 1, -1) - tgt.unsqueeze(-1)
label = torch.exp(-(d ** 2) / (2.0 * self.simcc_sigma ** 2))
label = label / label.sum(dim=-1, keepdim=True).clamp_min(1e-8)
logp = F.log_softmax(logits, dim=-1)
per_kp = -(label * logp).sum(dim=-1).mean(dim=-1) # (B, K), mean over x/y
if w_kp is None:
return per_kp.mean()
return self._kp_weighted(per_kp, w_kp)
def _project_3d_to_norm_2d(self, j3d, pred_cam_trans, targets):
j3d_cam = j3d + pred_cam_trans.unsqueeze(1)
z_cam = torch.clamp(j3d_cam[..., 2], min=0.4)
fx = targets["cam_focal"][:, 0:1]
fy = targets["cam_focal"][:, 1:2]
cx = targets["orig_shape"][:, 1:2] / 2.0
cy = targets["orig_shape"][:, 0:1] / 2.0
u_full = (fx * j3d_cam[..., 0] / z_cam) + cx
v_full = (fy * j3d_cam[..., 1] / z_cam) + cy
bbox = targets["bbox_square"]
sq_x1 = bbox[:, 0:1]
sq_y1 = bbox[:, 1:2]
crop_w = torch.clamp(bbox[:, 2:3] - bbox[:, 0:1], min=1.0)
true_scale = self.cfg.image_size / crop_w
u_crop = (u_full - sq_x1) * true_scale
v_crop = (v_full - sq_y1) * true_scale
M = targets["aug_M"]
u_aug = M[:, 0:1] * u_crop + M[:, 1:2] * v_crop + M[:, 2:3]
v_aug = M[:, 3:4] * u_crop + M[:, 4:5] * v_crop + M[:, 5:6]
u_norm = (u_aug / self.cfg.image_size) * 2.0 - 1.0
v_norm = (v_aug / self.cfg.image_size) * 2.0 - 1.0
return torch.stack([u_norm, v_norm], dim=-1)
def forward(self, preds, targets):
losses = {}
idx_pose = self.cfg.pose_dim
B = preds["mhr_params"].shape[0]
ones = torch.ones(B, device=preds["mhr_params"].device)
aug_active = targets.get("aug_active", 1.0 - ones)
aug_flip = targets.get("aug_flip", 0.0 * ones)
m_ident = 1.0 - aug_active
m_noflip = 1.0 - aug_flip
pred_pose = preds["mhr_params"][:, :idx_pose]
pred_scale = preds["mhr_params"][:, idx_pose:]
tgt_pose = targets["mhr_model_params"][:, :idx_pose]
tgt_scale = targets["mhr_model_params"][:, idx_pose:]
# loss_pose. Masked to m_ident (~21% of the batch at geom_p 0.8) because
# the dataset rotates the LABELS and the target parameters are not
# rotated with them. That is only true of the 6 root parameters:
# model params 3/4/5 drive joint-parameter slots 10/11/12 exclusively,
# so pose[6:136] -- the 130 local angles, which the oracle ablation
# attributes 100% of the PA-MPJPE error to -- is invariant to the
# in-plane rotation and only needs the flip mask. See POSE_ROOT_DIM.
beta = float(getattr(self.cfg, "pose_beta", 1.0))
if getattr(self.cfg, "pose_split", False):
r = POSE_ROOT_DIM
n = pred_pose.shape[1]
# Proportional weights: the split changes the MASK, not the
# root-vs-local balance a single mean over n parameters implied.
if getattr(self.cfg, "root_rot_loss", False):
# Rotation-matrix metric. A SmoothL1 in radians scores a
# prediction at -pi+eps against a target at pi-eps as a ~2pi
# error when the rotation between them is tiny, and 13.59% of
# corpus root angles lie beyond 2.8 rad. Root translation
# (0:3) is excluded because it is exactly 0.000e+00 on both
# sides under --cont-head and contributes nothing.
losses['loss_pose_root'] = (
self._mmean(self.root_chordal(preds, pred_pose, tgt_pose), m_ident)
* self.cfg.w_pose * (r / n) * float(getattr(self.cfg, "w_root_rot", 1.0)))
else:
losses['loss_pose_root'] = self._m_beta(
pred_pose[:, :r], tgt_pose[:, :r], m_ident, 1.0) * self.cfg.w_pose * (r / n)
losses['loss_pose'] = self._m_beta(
pred_pose[:, r:], tgt_pose[:, r:], m_noflip, beta) * self.cfg.w_pose * ((n - r) / n)
else:
losses['loss_pose'] = self._m_beta(
pred_pose, tgt_pose, m_ident, beta) * self.cfg.w_pose
losses['loss_scale'] = self.m_mse(pred_scale, tgt_scale, m_noflip) * self.cfg.w_scale
losses['loss_shape'] = self.m_mse(preds["shape_params"], targets["shape_params"], m_noflip) * self.cfg.w_shape
# NLF applies its translation loss as an unsquared Euclidean norm over the
# xyz triple rather than elementwise MSE: MSE lets one badly-scaled axis
# (in practice depth, which is an order of magnitude larger than x/y)
# dominate the gradient of all three.
if getattr(self.cfg, "cam_loss", "mse") == "euclid":
e_cam = torch.linalg.norm(preds["cam_trans"] - targets["cam_trans"], dim=-1)
losses['loss_cam'] = e_cam.mean() * self.cfg.w_cam
else:
losses['loss_cam'] = self.mse(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam
want_verts = float(getattr(self.cfg, "w_verts", 0.0)) > 0.0
exact_lm = bool(getattr(self.cfg, "exact_landmarks", False))
# The prediction needs skinned vertices for either consumer. The TARGET
# needs them only for loss_verts: the landmark losses compare against
# the stored annotation, which the teacher already produced with this
# mapping, so --exact-landmarks adds no target-side cost at all.
if want_verts or exact_lm:
pred_mhr_joints, pred_verts = self.mhr_module.get_joints_and_vertices(
preds["mhr_params"], preds["shape_params"])
else:
pred_mhr_joints = self.mhr_module.get_joints(preds["mhr_params"], preds["shape_params"])[..., :3]
with torch.no_grad():
if want_verts:
tgt_mhr_joints, tgt_verts = self.mhr_module.get_joints_and_vertices(
targets["mhr_model_params"], targets["shape_params"])
else:
tgt_mhr_joints = self.mhr_module.get_joints(targets["mhr_model_params"], targets["shape_params"])[..., :3]
c = targets["aug_cos"].view(-1, 1) if "aug_cos" in targets else torch.ones(B, 1, device=ones.device)
s = targets["aug_sin"].view(-1, 1) if "aug_sin" in targets else torch.zeros(B, 1, device=ones.device)
xm, ym = tgt_mhr_joints[..., 0], tgt_mhr_joints[..., 1]
tgt_mhr_joints = torch.stack(
[c * xm - s * ym, s * xm + c * ym, tgt_mhr_joints[..., 2]], dim=-1)
if want_verts:
# The same in-plane rotation the joints get: the dataset rotates
# the LABELS, not the target parameters, so the FK'd target has
# to be brought into the augmented frame.
xv, yv = tgt_verts[..., 0], tgt_verts[..., 1]
tgt_verts = torch.stack(
[c * xv - s * yv, s * xv + c * yv, tgt_verts[..., 2]], dim=-1)
# KEPT (w=1e-3) even though loss_3d_native now covers the same parameters.
# It is not redundant: the best linear reconstruction of the 127 skeleton
# joints FROM the 70 keypoints still leaves 48/127 joints with >1 mm and
# 6 with >5 mm of unexplained motion (worst 19 mm), i.e. the keypoint set
# is blind to part of the skeleton. Cost is nil — pred_mhr_joints is
# already computed and the target runs under no_grad.
losses['loss_mhr_joints'] = self.m_smooth_l1(
pred_mhr_joints, tgt_mhr_joints.detach(), m_noflip) * self.cfg.w_3d_joints
if want_verts:
# Per-vertex unsquared Euclidean distance in METRES, so the weight is
# on the same scale as w_keypoints3d under kp3d_loss="euclid" and the
# logged value reads as mean PVE / 1000. Masked to m_noflip for the
# same reason loss_mhr_joints is: under a horizontal flip the target
# PARAMETERS are not mirrored, only the 2D and 3D labels are.
# Restricted to the farthest-point samples this term has always
# used. --exact-landmarks widens the skinned subset to the union
# with the mapping's 468 vertices; averaging over the union instead
# would silently reweight the term and break comparability with
# generations 6-8.
vp = self.mhr_module.verts_loss_pos
pv = pred_verts if vp is None else pred_verts.index_select(1, vp)
tv = tgt_verts if vp is None else tgt_verts.index_select(1, vp)
e_verts = (pv - tv.detach()).norm(dim=-1).mean(1) / 100.0
losses['loss_verts'] = (self._mmean(e_verts, m_noflip)
* self.cfg.w_verts * self.fk_scale)
# The 70 native keypoints ARE the mesh now: an affine function of the
# 127 skeleton joints, so they are differentiable w.r.t. mhr_params and
# carry no extra forward pass (pred_mhr_joints is already computed).
pred_mhr_vision = self.mhr_module.to_vision(pred_mhr_joints)
if exact_lm:
# The teacher's readout, off the skinned mesh, instead of the fitted
# skeleton-only matrix. Feeds loss_3d_native and loss_reproj below,
# which is the whole change: `tgt_3d` already came out of this same
# mapping, so this removes an operator mismatch between the
# prediction and its own target.
pred_kp3d = self.mhr_module.regress_keypoints_exact(
pred_mhr_vision, self.mhr_module.to_vision(pred_verts))
else:
pred_kp3d = self.mhr_module.regress_keypoints(pred_mhr_vision)
pred_2d = preds["joints_2d"]
tgt_2d = targets["joints_2d"]
tgt_3d = targets["joints_3d"]
# Per-keypoint 2D observation weight. All-ones unless the dataset
# supplies `joints_2d_vis` (only SA-1B does), in which case the three
# 2D terms below skip the keypoints whose target is a (0, 0)
# placeholder. Measured on SA-1B: vis > 0 <=> the coordinate is real,
# with zero exceptions in 183k keypoints, so this gate is exact; the
# graded values (0.25 / 0.5 / 1.0) additionally down-weight the
# uncertain observations.
w_kp = targets.get("joints_2d_vis")
if w_kp is None:
w_kp = torch.ones_like(tgt_2d[..., 0])
# Truncation augmentation pushes keypoints outside the crop. Both 2D
# terms below are BOUNDED -- SimCC spans +/-kp2d_range and pred_2d is a
# soft-argmax over those same bins -- so a target at 1.67 cannot be
# reached. Clamping it to 1.5 asserts the joint is exactly at the crop
# edge, which is false, and leaves a permanent saturating gradient at
# w_keypoints2d = 10. Mask instead: the 3D loss still supervises those
# joints, which is the whole MeTRo argument for predicting them at all.
#
# Do NOT widen kp2d_range instead: bin_w = 2*range/(bins-1), so at 96
# bins that trades 3.5 px bins for 5.9 px, degrading every in-frame
# joint to accommodate the out-of-frame ones.
w_kp_2d = w_kp
if getattr(self.cfg, "mask_oob_2d", False):
in_crop = (tgt_2d.abs() <= self.cfg.kp2d_range).all(-1).to(w_kp.dtype)
w_kp_2d = w_kp * in_crop
# 2D head — training-only auxiliary task, and the ONLY supervision that
# survives a horizontal flip in full (targets["joints_2d"] is mirrored
# AND FLIP_PERM-permuted by the dataset, so it stays self-consistent —
# and so is joints_2d_vis).
e2d = F.smooth_l1_loss(pred_2d[..., :2], tgt_2d, reduction='none').mean(-1)
losses['loss_2d_native'] = self._kp_weighted(e2d, w_kp_2d) * self.cfg.w_keypoints2d
if "joints_2d_logits" in preds:
losses['loss_2d_simcc'] = self._simcc_ce(
preds["joints_2d_logits"], tgt_2d, w_kp_2d) * self.cfg.w_simcc
# Main 3D loss, now on the MHR-derived keypoints. Deliberately UNMASKED:
# targets["joints_3d"] is mirrored + permuted under flip, and the MHR rest
# skeleton is mirror-symmetric to 0.7 mm, so a mirrored pose is a perfectly
# representable MHR pose. (loss_pose / loss_scale / loss_shape stay masked
# because the TARGET PARAMETERS are not mirrored — only the geometry is.)
#
# SmoothL1's beta defaults to 1.0 -- one METRE here -- so every sample of
# a 37.7 mm mean / 119 mm p95 error distribution sits in the quadratic
# branch and the term behaves as a scaled MSE: measured 16x less gradient
# than L1 and 31x less than the unsquared Euclidean at the same weight,
# which is why w_keypoints3d=2.0 delivered 7.7% of the update direction.
# "euclid" also has a FLAT gradient (d||x||/dx is a unit vector), so it is
# gentler than the current form where the error is large -- 0.62x on a
# freshly initialised model -- and only stronger once the model is close.
# Frames with mocap feet: the teacher's foot keypoints leave this
# term and loss_foot_mocap supervises them instead (the teacher's feet
# slide from frame to frame; the markers do not).
mocap_feet = (float(getattr(self.cfg, "w_foot_mocap", 0.0)) > 0.0
and "foot_mocap_valid" in targets)
if getattr(self.cfg, "kp3d_loss", "smooth_l1") == "euclid":
e_kp3d = (pred_kp3d - tgt_3d).norm(dim=-1) # (B, 70), metres
w3 = self.kp3d_weight.expand_as(e_kp3d)
if mocap_feet:
w3 = w3 * (1.0 - targets["foot_mocap_valid"])
losses['loss_3d_native'] = (
self._kp_weighted(e_kp3d, w3)
* self.cfg.w_keypoints3d * self.fk_scale)
elif mocap_feet:
e_kp3d = F.smooth_l1_loss(pred_kp3d, tgt_3d, reduction='none').mean(-1)
w3 = 1.0 - targets["foot_mocap_valid"]
losses['loss_3d_native'] = (self._kp_weighted(e_kp3d, w3)
* self.cfg.w_keypoints3d * self.fk_scale)
else:
losses['loss_3d_native'] = self.l1(pred_kp3d, tgt_3d) * self.cfg.w_keypoints3d * self.fk_scale
cc = targets["aug_cos"].view(-1, 1)
ss = targets["aug_sin"].view(-1, 1)
def _counter_rot_pts(pts):
x, y = pts[..., 0], pts[..., 1]
return torch.stack([cc * x - ss * y, ss * x + cc * y, pts[..., 2]], dim=-1)
def _counter_rot_cam(cam):
x, y = cam[:, 0:1], cam[:, 1:2]
return torch.cat([cc * x - ss * y, ss * x + cc * y, cam[:, 2:3]], dim=1)
cam_cr = _counter_rot_cam(preds["cam_trans"])
# Single re-projection (replaces loss_reproj_native + loss_reproj_mhr):
# one geometry, all 70 keypoints.
#
# The projection runs in the ORIGINAL image frame (real focal, image
# centre, un-augmented bbox_square) and only then applies aug_M, which
# carries the mirror. So a flipped sample has to be un-mirrored first,
# or it gets mirrored twice. The dataset applies flip THEN rotation, so
# the inverse is counter-rotate THEN negate x -- on the keypoints and on
# cam_trans alike, both of which the dataset negated. Index order needs
# no fixing: pred_kp3d is already in FLIP_PERM order and aug_M maps it to
# the same order as tgt_2d.
#
# Previously this whole term was masked to m_noflip, which was defensible
# at w_reproj = 0.01 and is not once the weight matters: half the batch
# would contribute nothing to the only loss tying 3D to pixels.
pts_cr, cam_cr_r = _counter_rot_pts(pred_kp3d), cam_cr
m_reproj = m_noflip
if getattr(self.cfg, "reproj_all_samples", False):
sgn = (1.0 - 2.0 * aug_flip).view(-1, 1) # +1 / -1
pts_cr = torch.stack([pts_cr[..., 0] * sgn, pts_cr[..., 1],
pts_cr[..., 2]], dim=-1)
cam_cr_r = torch.cat([cam_cr[:, 0:1] * sgn, cam_cr[:, 1:]], dim=1)
m_reproj = ones
pred_reproj = self._project_3d_to_norm_2d(pts_cr, cam_cr_r, targets)
e_reproj = (pred_reproj - tgt_2d).abs().mean(-1)
# Same finger down-weight as loss_3d_native: this is the other geometric
# term over all 70 keypoints, and at --preset v2's w_reproj = 0.5 it would
# otherwise also be ~60% fingers. The 2D head's own terms are left alone —
# joints_2d is a deployed output and SA-1B's visibility mask already
# handles its unobserved hands.
losses['loss_reproj'] = self._kp_weighted(
e_reproj, w_kp * self.kp3d_weight, m_reproj) * self.cfg.w_reproj * self.fk_scale
if "contact" in targets:
self._foot_lock_losses(losses, preds, targets, pred_kp3d)
losses['total_loss'] = sum(losses.values())
return losses
def _foot_lock_losses(self, losses, preds, targets, pred_kp3d):
"""--prev-token terms. Everything is in the augmented camera frame
(keypoints + cam_trans), the frame the dataset moved its labels into.
loss_zero_vel: on a real pair with the previous pose supplied and the
foot in contact, the predicted foot must sit where the previous-pose
INPUT put it, plus the foot's true motion between the two frames. At
inference that input is the model's own last estimate, so this is what
keeps a planted foot planted."""
B = pred_kp3d.shape[0]
cfg = self.cfg
pred_abs = pred_kp3d + preds["cam_trans"].unsqueeze(1) # (B, 70, 3)
cv = targets["contact_valid"]
if "contact_logits" in preds and cfg.w_contact > 0:
bce = F.binary_cross_entropy_with_logits(
preds["contact_logits"].float(), targets["contact"], reduction="none")
w = cv
if getattr(cfg, "contact_flight_weight", 1.0) != 1.0:
flight = ((cv > 0).all(1) & (targets["contact"] == 0).all(1)).float()
w = cv * (1.0 + (cfg.contact_flight_weight - 1.0) * flight).unsqueeze(1)
losses['loss_contact'] = (bce * w).sum() / cv.sum().clamp_min(1.0) * cfg.w_contact
if cfg.w_zero_vel > 0:
feet = torch.as_tensor(FOOT_KP, device=pred_kp3d.device)
e = ((pred_abs[:, feet] - targets["prev_feet"]) - targets["feet_disp"]).norm(dim=-1)
e = e.view(B, 2, 4).mean(-1) # (B, 2) metres
m = targets["contact"] * cv * (targets["has_prev"] * targets["has_pair"]).view(B, 1)
losses['loss_zero_vel'] = ((e * m).sum() / m.sum().clamp_min(1.0)
* cfg.w_zero_vel * self.fk_scale)
if cfg.w_foot_mocap > 0:
fv = targets["foot_mocap_valid"]
tgt_abs = targets["foot_mocap"] + targets["cam_trans"].unsqueeze(1)
e = (pred_abs - tgt_abs).norm(dim=-1)
losses['loss_foot_mocap'] = ((e * fv).sum() / fv.sum().clamp_min(1.0)
* cfg.w_foot_mocap * self.fk_scale)
# The three terms that back-propagate through the MHR forward kinematics. They
# are the only unbounded ones: the parameter regressions are bounded by the
# target magnitudes and the 2D terms are bounded by construction (SimCC spans
# +/-kp2d_range and its CE by log(kp2d_bins)).
FK_LOSS_KEYS = ("loss_3d_native", "loss_mhr_joints", "loss_reproj", "loss_verts",
"loss_zero_vel", "loss_foot_mocap")
def safe_loss_subset(losses):
"""Total over the terms that do NOT pass through the forward kinematics.
This is what an anomalous batch should still be allowed to contribute. A
blown-up bone scale makes `loss_3d_native` / `loss_reproj` enormous while
`loss_scale` and `loss_pose` stay small and point straight back at the
teacher value, so dropping only the FK terms keeps the restoring gradient
and discards the exploding one.
"""
return sum(v for k, v in losses.items()
if k != "total_loss" and k not in FK_LOSS_KEYS)
def format_loss_terms(losses, top_k: int = 5) -> str:
"""The largest individual loss terms, for diagnosing an anomalous batch.
The anomaly guard only ever printed the total, which says nothing about
which head blew up. The 2D terms are bounded (`joints_2d` is a SimCC
expectation over bins in [-kp2d_range, kp2d_range], and the SimCC CE by
log(kp2d_bins)); the parameter regressions and the FK-derived keypoint loss
are not, so this line is what tells the two cases apart in the job log.
"""
terms = [(k.replace("loss_", ""), float(v))
for k, v in losses.items() if k != "total_loss"]
terms.sort(key=lambda kv: kv[1], reverse=True)
return " ".join(f"{k}={v:.1f}" for k, v in terms[:top_k])
def scale_lr(optimizer, factor: float, scheduler=None) -> None:
"""Multiply every learning-rate anchor in place.
OneCycleLR does not read a stored LR — it interpolates between `initial_lr`,
`max_lr` and `min_lr` on the param groups, and those all scale linearly with
the peak. Scaling the anchors therefore rescales the whole remaining
schedule while leaving its *position* (``last_epoch``) untouched, which is
what both the resume path and the rollback want.
"""
for group in optimizer.param_groups:
for key in ("lr", "max_lr", "initial_lr", "min_lr"):
if key in group:
group[key] *= factor
if scheduler is not None:
# LRScheduler stashes these at construction and restores them from a
# checkpoint, so they have to move with the param groups.
for attr in ("base_lrs", "_last_lr"):
if hasattr(scheduler, attr):
setattr(scheduler, attr, [lr * factor for lr in getattr(scheduler, attr)])
# ============================================================
# Cell 10 — HMR Evaluation Metrics (MPJPE & PA-MPJPE)
# ============================================================
def batched_procrustes_alignment(pred_pts, gt_pts):
B, N, D = pred_pts.shape
mu_pred = pred_pts.mean(dim=1, keepdim=True)
mu_gt = gt_pts.mean(dim=1, keepdim=True)
pred_centered = pred_pts - mu_pred
gt_centered = gt_pts - mu_gt
norm_pred = torch.linalg.norm(pred_centered, dim=(1, 2), keepdim=True)
norm_gt = torch.linalg.norm(gt_centered, dim=(1, 2), keepdim=True)
pred_normalized = pred_centered / torch.clamp(norm_pred, min=1e-8)
gt_normalized = gt_centered / torch.clamp(norm_gt, min=1e-8)
H = torch.bmm(pred_normalized.transpose(1, 2), gt_normalized)
U, S, Vh = torch.linalg.svd(H)
R = torch.bmm(U, Vh)
det = torch.linalg.det(R)
det_sign = torch.where(det < 0, torch.tensor(-1.0, device=pred_pts.device), torch.tensor(1.0, device=pred_pts.device)).unsqueeze(-1)
U_fixed = U.clone()
U_fixed[:, :, 2] *= det_sign
R_fixed = torch.bmm(U_fixed, Vh)
S_fixed = S.clone()
S_fixed[:, 2] *= det_sign.squeeze(-1)
scale = S_fixed.sum(dim=-1, keepdim=True).unsqueeze(-1) * (norm_gt / torch.clamp(norm_pred, min=1e-8))
pred_aligned = scale * torch.bmm(pred_centered, R_fixed) + mu_gt
return pred_aligned
@torch.no_grad()
def evaluate_hmr_batch(preds, targets, cfg, mhr_module):
"""Mesh_* on the full 70 MHR-derived keypoints are the primary metrics.
The Head_3D_* metrics are gone with the 3D head. Mesh52_* replicates the
old 52-anchor formula verbatim so that these runs remain comparable with
the earlier trainer.
"""
metrics = {}
gt_3d_native = targets["joints_3d"][..., :3]
pred_mhr_vision = mhr_module.to_vision(
mhr_module.get_joints(preds["mhr_params"], preds["shape_params"])[..., :3])
pred_kp3d = mhr_module.regress_keypoints(pred_mhr_vision)
pred_c = pred_kp3d - pred_kp3d.mean(dim=1, keepdim=True)
gt_c = gt_3d_native - gt_3d_native.mean(dim=1, keepdim=True)
metrics['Mesh_MPJPE'] = torch.linalg.norm(pred_c - gt_c, dim=-1).mean().item() * 1000.0
pred_aligned = batched_procrustes_alignment(pred_kp3d, gt_3d_native)
metrics['Mesh_PA_MPJPE'] = torch.linalg.norm(
pred_aligned - gt_3d_native, dim=-1).mean().item() * 1000.0
# Body-only variant over the 30 non-finger keypoints. Mesh_PA_MPJPE is 60%
# fingers by keypoint count, so on the single-GPU path it selects largely on
# hands; the published J12/J14 sets contain none of them. Reported alongside,
# never substituted — selection behaviour is unchanged.
b = torch.from_numpy(BODY_KP).to(pred_kp3d.device)
pb, gb = pred_kp3d[:, b], gt_3d_native[:, b]
metrics['Mesh_MPJPE_body'] = torch.linalg.norm(
(pb - pb.mean(1, keepdim=True)) - (gb - gb.mean(1, keepdim=True)),
dim=-1).mean().item() * 1000.0
metrics['Mesh_PA_MPJPE_body'] = torch.linalg.norm(
batched_procrustes_alignment(pb, gb) - gb, dim=-1).mean().item() * 1000.0
if hasattr(cfg, 'native_mapping_ids') and hasattr(cfg, 'mhr_mapping_ids'):
pred_mesh_anchors = pred_mhr_vision[:, cfg.mhr_mapping_ids, :]
gt_mesh_anchors = gt_3d_native[:, cfg.native_mapping_ids, :]
pred_mesh_centered = pred_mesh_anchors - pred_mesh_anchors.mean(dim=1, keepdim=True)
gt_mesh_centered = gt_mesh_anchors - gt_mesh_anchors.mean(dim=1, keepdim=True)
err_mesh = torch.linalg.norm(pred_mesh_centered - gt_mesh_centered, dim=-1)
metrics['Mesh52_MPJPE'] = err_mesh.mean().item() * 1000.0
pred_mesh_aligned = batched_procrustes_alignment(pred_mesh_anchors, gt_mesh_anchors)
err_mesh_pa = torch.linalg.norm(pred_mesh_aligned - gt_mesh_anchors, dim=-1)
metrics['Mesh52_PA_MPJPE'] = err_mesh_pa.mean().item() * 1000.0
return metrics
# ============================================================
# Cell 11 — Full Distillation Training Loop
# ============================================================
def build_param_groups(model, weight_decay, split_wd=True):
"""AdamW param groups with weight decay removed from the params that should
never see it.
Applying a single weight_decay to `model.parameters()` penalises every
tensor equally, which is wrong for three families:
* **norm parameters** (BatchNorm/LayerNorm weight and bias) — decaying a
BN gamma toward 0 shrinks the layer's output scale, which the following
layer has to keep re-learning.
* **biases** — a bias is an offset, not a capacity knob; decay just
re-centres it toward 0 for no regularisation benefit.
* **depthwise convolution kernels** — a depthwise filter has only
k*k weights per channel (9 for 3x3) instead of C_in*k*k, so an L2
penalty of the same magnitude is a far larger *relative* pull on it.
MobileNet-family backbones are mostly depthwise, so uniform wd hits
them much harder than it hits a conventional/hybrid backbone.
This is a general recipe fix, not a MobileNet-specific hack -- the same
exclusions are standard for RepViT too (both are BN-heavy and RepViT's
token mixer is itself depthwise). Use it for every backbone or none, or
the comparison just moves the bias to the other side.
Returns the two-group list AdamW expects.
"""
if not split_wd:
return [{"params": list(model.parameters()), "weight_decay": weight_decay}]
# Names of depthwise conv weights: groups == in_channels (and > 1).
depthwise = set()
for mod_name, mod in model.named_modules():
if isinstance(mod, nn.modules.conv._ConvNd) and mod.groups > 1 \
and mod.groups == mod.in_channels:
depthwise.add(f"{mod_name}.weight" if mod_name else "weight")
decay, no_decay = [], []
for name, param in model.named_parameters():
if not param.requires_grad:
continue
if name in depthwise or name.endswith(".bias") or param.ndim <= 1:
# ndim <= 1 catches every BN/LayerNorm weight and every bias,
# including the ones inside nn.TransformerDecoderLayer.
no_decay.append(param)
else:
decay.append(param)
return [
{"params": decay, "weight_decay": weight_decay},
{"params": no_decay, "weight_decay": 0.0},
]
def make_ema_avg_fn(max_decay):
@torch.no_grad()
def avg_fn(ema_params, model_params, num_averaged):
if not (torch.is_floating_point(ema_params[0]) or torch.is_complex(ema_params[0])):
for e, m in zip(ema_params, model_params):
e.copy_(m)
return
n = num_averaged.item() if torch.is_tensor(num_averaged) else float(num_averaged)
decay = min(max_decay, (1.0 + n) / (10.0 + n))
torch._foreach_lerp_(ema_params, model_params, 1.0 - decay)
return avg_fn
def train_instant_hmr():
EMA_DECAY = getattr(cfg, "ema_decay", 0.9998)
EARLY_STOP_PATIENCE = getattr(cfg, "early_stop_patience", 50)
print(f"--- Training {cfg.backbone} | EMA decay={EMA_DECAY} | early-stop patience={EARLY_STOP_PATIENCE} ---")
os.makedirs(cfg.log_dir, exist_ok=True)
RESUME_FROM_CHECKPOINT = getattr(cfg, "resume", True)
model = InstantHMRStudent(cfg, pretrained=True).to(device)
criterion = DistillationLoss(cfg, mhr_module)
split_wd = getattr(cfg, "split_weight_decay", False)
param_groups = build_param_groups(model, cfg.weight_decay, split_wd=split_wd)
optimizer = optim.AdamW(param_groups, lr=cfg.lr, weight_decay=cfg.weight_decay)
if split_wd:
n_dec = sum(p.numel() for p in param_groups[0]["params"])
n_nod = sum(p.numel() for p in param_groups[1]["params"])
print(f" param groups: wd={cfg.weight_decay} on {n_dec/1e6:.2f} M params | "
f"wd=0 on {n_nod/1e6:.2f} M (depthwise kernels, norms, biases)")
scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
steps_per_epoch = len(train_loader)
scheduler = lr_scheduler.OneCycleLR(
optimizer, max_lr=cfg.lr, epochs=cfg.epochs,
steps_per_epoch=steps_per_epoch, pct_start=0.1,
)
best_raw_path = os.path.join(cfg.log_dir, "best_student_model_raw.pth")
best_ema_path = os.path.join(cfg.log_dir, "best_student_model_ema.pth")
best_ckpt_path = os.path.join(cfg.log_dir, "best_student_model_v3.pth")
start_epoch = 0
best_raw_pa = float('inf')
best_ema_pa = float('inf')
best_overall_pa = float('inf')
epochs_no_improve = 0
if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):
print(f"🔄 Resuming from checkpoint: {best_ckpt_path}")
ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=False)
model.load_state_dict(ckpt["model_state_dict"], strict=False)
if "optimizer_state_dict" in ckpt:
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
if "scaler_state_dict" in ckpt:
scaler.load_state_dict(ckpt["scaler_state_dict"])
if "scheduler_state_dict" in ckpt:
scheduler.load_state_dict(ckpt["scheduler_state_dict"])
# Both loads above put the checkpoint's LR state back: `param_groups`
# (max_lr / initial_lr / min_lr) come from the optimizer state_dict and
# `base_lrs` from the scheduler's, which overwrites everything the
# freshly-built OneCycleLR just installed. Without this, --lr is
# SILENTLY IGNORED on every resume. Re-apply the requested peak by
# rescaling the anchors, keeping the schedule's shape and position.
ckpt_max_lr = float(optimizer.param_groups[0].get("max_lr", cfg.lr))
if ckpt_max_lr > 0.0 and abs(ckpt_max_lr - cfg.lr) > 1e-12:
scale_lr(optimizer, cfg.lr / ckpt_max_lr, scheduler)
print(f" LR rescaled: checkpoint peak {ckpt_max_lr:.2e} -> requested "
f"{cfg.lr:.2e} (current {optimizer.param_groups[0]['lr']:.2e})")
start_epoch = ckpt.get("epoch", -1) + 1
best_overall_pa = ckpt.get("val_pa_mpjpe", float('inf'))
best_raw_pa = best_overall_pa
best_ema_pa = best_overall_pa
print(f"✅ Resumed successfully from Epoch {start_epoch}. Best PA-MPJPE was {best_overall_pa:.1f}mm.")
else:
print("🚀 Starting training from scratch with fresh weights.")
ema_model = AveragedModel(model, multi_avg_fn=make_ema_avg_fn(EMA_DECAY), use_buffers=True)
@torch.no_grad()
def run_validation(eval_model, tag, epoch):
eval_model.eval()
vloss = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'reproj': 0.0}
hmr = {'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0, 'Mesh52_MPJPE': 0.0, 'Mesh52_PA_MPJPE': 0.0}
n = 0
vbar = tqdm(val_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]", leave=False)
for batch in vbar:
batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
preds_fp16 = eval_model(batch["image"], batch["cliff_cond"])
preds = {k: v.float() for k, v in preds_fp16.items()}
losses = criterion(preds, batch)
if math.isnan(losses['total_loss'].item()) or math.isinf(losses['total_loss'].item()):
continue
vloss['total'] += losses['total_loss'].item()
vloss['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()
vloss['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()
vloss['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()
vloss['reproj'] += losses.get('loss_reproj', torch.tensor(0)).item()
bm = evaluate_hmr_batch(preds, batch, cfg, mhr_module)
for k in hmr:
hmr[k] += bm.get(k, 0.0)
n += 1
vbar.set_postfix({'PA-MPJPE': f"{hmr['Mesh_PA_MPJPE']/max(n,1):.1f}mm"})
vbar.close()
vloss = {k: v / max(n, 1) for k, v in vloss.items()}
hmr = {k: v / max(n, 1) for k, v in hmr.items()}
return vloss, hmr, n
global_step = start_epoch * steps_per_epoch
criterion.set_step(global_step)
# Skips are counted ACROSS epochs — a stalled run skips whole epochs at a
# time, so a per-epoch counter would never trip.
consecutive_skips = 0
ema_rollbacks = 0
training_aborted = False
for epoch in range(start_epoch, cfg.epochs):
model.train()
tm = {'total': 0.0, '2d': 0.0, 'simcc': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'reproj': 0.0}
nb = 0
pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Train]")
for batch in pbar:
batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
optimizer.zero_grad(set_to_none=True)
with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
preds_fp16 = model(batch["image"], batch["cliff_cond"])
preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}
losses = criterion(preds_fp32, batch)
loss = losses['total_loss']
loss_val = loss.item()
skip_reason = None
if math.isnan(loss_val) or math.isinf(loss_val):
skip_reason = "NaN/Inf loss detected!"
elif loss_val > cfg.anomaly_loss_threshold:
skip_reason = f"anomalous loss {loss_val:.1f} > {cfg.anomaly_loss_threshold}"
if skip_reason is not None:
# Same policy as train_ddp: an anomalous batch still
# carries the restoring gradient in its non-FK terms, so step
# on those rather than discarding it. Kept in both loops on
# purpose -- the DDP path having a recovery the single-GPU path
# lacked (and vice versa) is how this file and its fork drifted.
if cfg.anomaly_safe_fallback:
safe = safe_loss_subset(losses)
sv = safe.item()
if math.isfinite(sv) and sv <= cfg.anomaly_loss_threshold:
scaler.scale(safe).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(),
cfg.grad_clip)
scaler.step(optimizer)
scaler.update()
scheduler.step()
ema_model.update_parameters(model)
global_step += 1
criterion.set_step(global_step)
consecutive_skips = 0
continue
consecutive_skips += 1
print(f"⚠️ WARNING: {skip_reason} — skipping step. "
f"[{consecutive_skips} in a row] {format_loss_terms(losses)}")
if consecutive_skips >= cfg.anomaly_skip_patience:
if ema_rollbacks >= cfg.max_ema_rollbacks:
print(f"\n🛑 Aborting: still stuck after {cfg.max_ema_rollbacks} EMA "
f"rollback(s). The best RAW / EMA checkpoints on disk are "
f"unaffected, and the ONNX export still runs off "
f"best_student_model_v3.pth — inspect the loss terms above "
f"before relaunching.")
training_aborted = True
break
ema_rollbacks += 1
print(f"\n🚑 {consecutive_skips} consecutive skipped steps: the raw weights "
f"are outside the valid MHR region and CANNOT recover on their own, "
f"because a skipped step updates neither the weights nor the LR.\n"
f" Rolling back to the EMA weights "
f"(attempt {ema_rollbacks}/{cfg.max_ema_rollbacks}), scaling the LR by "
f"{cfg.anomaly_rollback_lr_decay} and clearing the optimiser moments.")
model.load_state_dict(ema_model.module.state_dict())
# The Adam moments are what drove the model out of the valid
# region; carrying them over would walk straight back out.
optimizer.state.clear()
scale_lr(optimizer, cfg.anomaly_rollback_lr_decay, scheduler)
print(f" LR is now {optimizer.param_groups[0]['lr']:.2e}", flush=True)
consecutive_skips = 0
continue
consecutive_skips = 0
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
scale_before = scaler.get_scale()
scaler.step(optimizer)
scaler.update()
scale_after = scaler.get_scale()
if scale_before <= scale_after:
scheduler.step()
ema_model.update_parameters(model)
global_step += 1
criterion.set_step(global_step)
tm['total'] += loss.item()
tm['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()
tm['simcc'] += losses.get('loss_2d_simcc', torch.tensor(0)).item()
tm['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()
tm['pose'] += (losses.get('loss_pose', torch.tensor(0)).item()
+ losses.get('loss_pose_root', torch.tensor(0)).item())
tm['scale'] += losses.get('loss_scale', torch.tensor(0)).item()
tm['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()
tm['reproj'] += losses.get('loss_reproj', torch.tensor(0)).item()
nb += 1
pbar.set_postfix({'Tot': f"{loss.item():.3f}",
'MHR': f"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}"})
pbar.close()
if training_aborted:
break
avg_train = {k: v / max(nb, 1) for k, v in tm.items()}
raw_val, raw_hmr, n_raw = run_validation(model, "raw", epoch)
ema_val, ema_hmr, n_ema = run_validation(ema_model, "ema", epoch)
print(f"\n📈 Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}")
print(f" [Train] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | 2D: {avg_train['2d']:.4f} "
f"| 3D: {avg_train['3d']:.4f} | reproj: {avg_train['reproj']:.4f} | simcc: {avg_train['simcc']:.3f}")
print(f" [Val RAW] Tot: {raw_val['total']:.4f} | Mesh PA-MPJPE: {raw_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {raw_hmr['Mesh_MPJPE']:.1f} | Mesh52 PA: {raw_hmr['Mesh52_PA_MPJPE']:.1f} | Mesh52 MPJPE: {raw_hmr['Mesh52_MPJPE']:.1f} mm")
print(f" [Val EMA] Tot: {ema_val['total']:.4f} | Mesh PA-MPJPE: {ema_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {ema_hmr['Mesh_MPJPE']:.1f} | Mesh52 PA: {ema_hmr['Mesh52_PA_MPJPE']:.1f} | Mesh52 MPJPE: {ema_hmr['Mesh52_MPJPE']:.1f} mm")
delta = raw_hmr['Mesh_PA_MPJPE'] - ema_hmr['Mesh_PA_MPJPE']
print(f" 🔬 EMA vs RAW (Mesh PA-MPJPE): {'EMA better' if delta > 0 else 'RAW better'} by {abs(delta):.1f} mm")
def get_save_dict(model_to_save, val_metric, source_name):
return {
'epoch': epoch,
'model_state_dict': model_to_save.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scaler_state_dict': scaler.state_dict(),
'scheduler_state_dict': scheduler.state_dict(),
'val_pa_mpjpe': val_metric,
'source': source_name
}
improved_raw = n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa
improved_ema = n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa
if improved_raw:
best_raw_pa = raw_hmr['Mesh_PA_MPJPE']
torch.save(get_save_dict(model, best_raw_pa, 'raw'), best_raw_path)
if improved_ema:
best_ema_pa = ema_hmr['Mesh_PA_MPJPE']
torch.save(get_save_dict(ema_model.module, best_ema_pa, 'ema'), best_ema_path)
current = min(raw_hmr['Mesh_PA_MPJPE'], ema_hmr['Mesh_PA_MPJPE'])
if current < best_overall_pa - 1e-4:
best_overall_pa = current
use_ema = ema_hmr['Mesh_PA_MPJPE'] <= raw_hmr['Mesh_PA_MPJPE']
better = ema_model.module if use_ema else model
torch.save(get_save_dict(better, best_overall_pa, 'ema' if use_ema else 'raw'), best_ckpt_path)
print(f"💾 New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}")
if improved_raw or improved_ema:
epochs_no_improve = 0
else:
epochs_no_improve += 1
print(f" ⏳ Neither RAW nor EMA improved: {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best raw {best_raw_pa:.1f} / ema {best_ema_pa:.1f} mm)")
if epochs_no_improve >= EARLY_STOP_PATIENCE:
print(f"\n🛑 Early stopping at epoch {epoch+1}: neither RAW nor EMA improved for {EARLY_STOP_PATIENCE} epochs.")
break
print(f"\n🎉 Training complete! Best overall Mesh PA-MPJPE: {best_overall_pa:.1f} mm")
print(f" best RAW: {best_raw_pa:.1f} mm -> {best_raw_path}")
print(f" best EMA: {best_ema_pa:.1f} mm -> {best_ema_path}")
# ============================================================
# Self-tests
# ============================================================
def _mock_preds_from(batch):
return {
"mhr_params": batch["mhr_model_params"].clone(),
"shape_params": batch["shape_params"].clone(),
"cam_trans": batch["cam_trans"].clone(),
"joints_2d": batch["joints_2d"].clone(),
}
def run_self_tests():
print("--- Architecture Output Shapes ---")
test_model = InstantHMRStudent(cfg, pretrained=False)
out = test_model(torch.randn(2, 3, cfg.image_size, cfg.image_size), torch.randn(2, 3))
for k, v in out.items():
print(f" {k}: {tuple(v.shape)}")
n_par = sum(p.numel() for p in test_model.parameters())
print(f" decoder queries: {test_model.total_queries} | params: {n_par/1e6:.2f} M")
assert "joints_3d" not in out, "the 3D head should be gone"
criterion = DistillationLoss(cfg, mhr_module)
criterion.warmup_steps = 0 # exercise the FK losses at full weight
print("\n--- Keypoint regressor fidelity (127 skeleton joints -> 70 keypoints) ---")
W = mhr_module.kp_regressor
rs = W.sum(dim=1)
print(f" W {tuple(W.shape)} | row sums {rs.min().item():.6f}..{rs.max().item():.6f} "
f"(must be 1 for rigid-equivariance) | |W|max {W.abs().max().item():.2f}")
ok_affine = bool(((rs - 1.0).abs() < 1e-4).all())
print("\n--- Perfect Student (identity batch) ---")
clean_ds = SAM3DStudentDataset(cfg.data_root, augment=False,
max_images=256, per_dataset_caps=cfg.per_dataset_caps,
image_size=cfg.image_size)
clean_loader = DataLoader(clean_ds, batch_size=32, shuffle=False, num_workers=0)
cb = {k: v.to(device) for k, v in next(iter(clean_loader)).items()}
l_clean = criterion(_mock_preds_from(cb), cb)
for k, v in l_clean.items():
print(f" {k:<20}: {v.item():.6f}")
with torch.no_grad():
kp_c = mhr_module.get_native_keypoints(cb["mhr_model_params"], cb["shape_params"])
d_clean = (kp_c - cb["joints_3d"]).norm(dim=-1).mean().item() * 1000.0
print(f"\n regressed-vs-label residual (clean): {d_clean:.3f} mm")
ok_reg = d_clean < 5.0
print("\n--- MHR consistency under geometric augmentation (rotation only) ---")
aug_ds = SAM3DStudentDataset(cfg.data_root, augment=True,
max_images=256, per_dataset_caps=cfg.per_dataset_caps,
geom_p=1.0, geom_rot_deg=cfg.geom_rot_deg,
geom_scale_range=cfg.geom_scale_range,
geom_trans=cfg.geom_trans, geom_flip_p=0.0,
image_size=cfg.image_size)
aug_loader = DataLoader(aug_ds, batch_size=32, shuffle=False, num_workers=0)
ab = {k: v.to(device) for k, v in next(iter(aug_loader)).items()}
with torch.no_grad():
# target MHR params are NOT augmented -> rotate their geometry forward
tgt_mhr = mhr_module.get_joints(ab["mhr_model_params"], ab["shape_params"])[..., :3]
cth = ab["aug_cos"].view(-1, 1)
sth = ab["aug_sin"].view(-1, 1)
xm, ym = tgt_mhr[..., 0], tgt_mhr[..., 1]
tgt_mhr_rot = torch.stack([cth * xm - sth * ym, sth * xm + cth * ym, tgt_mhr[..., 2]], dim=-1)
kp_a = mhr_module.regress_keypoints(mhr_module.to_vision(tgt_mhr_rot))
d_aug = (kp_a - ab["joints_3d"]).norm(dim=-1).mean().item() * 1000.0
print(f" regressed-vs-label residual: clean {d_clean:.3f} mm | augmented {d_aug:.3f} mm")
ok_struct = d_aug < d_clean * 1.5 + 1.0
cc, ss = cth, sth
def counter(pts):
x, y = pts[..., 0], pts[..., 1]
return torch.stack([cc * x - ss * y, ss * x + cc * y, pts[..., 2]], dim=-1)
cam_cr = torch.cat([cc * ab["cam_trans"][:, 0:1] - ss * ab["cam_trans"][:, 1:2],
ss * ab["cam_trans"][:, 0:1] + cc * ab["cam_trans"][:, 1:2],
ab["cam_trans"][:, 2:3]], dim=1)
reproj_aug = criterion._project_3d_to_norm_2d(counter(kp_a), cam_cr, ab)
r_aug = (reproj_aug - ab["joints_2d"]).abs().mean().item()
reproj_clean = criterion._project_3d_to_norm_2d(kp_c, cb["cam_trans"], cb)
r_clean = (reproj_clean - cb["joints_2d"]).abs().mean().item()
print(f" reproj(counter-rot + M) residual: clean {r_clean:.4f} | augmented {r_aug:.4f} (normalised units)")
ok_reproj = r_aug < r_clean * 1.5 + 5e-3
print("\n--- Flip validity of the (unmasked) loss_3d_native ---")
flip_ds = SAM3DStudentDataset(cfg.data_root, augment=True,
max_images=256, per_dataset_caps=cfg.per_dataset_caps,
geom_p=1.0, geom_rot_deg=0.0, geom_scale_range=0.0,
geom_trans=0.0, geom_flip_p=1.0,
image_size=cfg.image_size)
flip_loader = DataLoader(flip_ds, batch_size=32, shuffle=False, num_workers=0)
fb = {k: v.to(device) for k, v in next(iter(flip_loader)).items()}
with torch.no_grad():
# mirror the un-augmented MHR geometry the same way the dataset mirrors labels
kp_f = mhr_module.get_native_keypoints(fb["mhr_model_params"], fb["shape_params"])
kp_f = torch.stack([-kp_f[..., 0], kp_f[..., 1], kp_f[..., 2]], dim=-1)[:, FLIP_PERM]
d_flip = (kp_f - fb["joints_3d"]).norm(dim=-1).mean().item() * 1000.0
print(f" mirrored MHR keypoints vs mirrored labels: {d_flip:.3f} mm "
f"(a mirrored pose is representable by MHR, so the loss stays valid)")
ok_flip = d_flip < d_clean * 1.5 + 1.0
# NOTE on the oracle. _mock_preds_from feeds back the TARGET MHR parameters,
# which are never rotated by the dataset (there is no closed form for the
# rotated root parameters). So on a rotated batch the "perfect student" is
# not actually perfect for any MHR-derived quantity: its geometry is the
# un-rotated body while the labels are rotated. The rotation pathway itself
# is already validated above (regressed-vs-label residual, 1.4 mm rotated
# exactly as clean). The masked-loss check therefore runs on a batch with
# scale + translation only, where the target parameters ARE the right answer.
print("\n--- Perfect Student (rotated batch — informational only) ---")
l_rot = criterion(_mock_preds_from(ab), ab)
for k, v in l_rot.items():
print(f" {k:<20}: {v.item():.6f}")
print(" (loss_3d_native / loss_mhr_joints are non-zero here BY CONSTRUCTION:")
print(" the oracle parameters are un-rotated. See the note in run_self_tests.)")
print("\n--- Perfect Student (scale+translate batch, masked losses) ---")
st_ds = SAM3DStudentDataset(cfg.data_root, augment=True,
max_images=256, per_dataset_caps=cfg.per_dataset_caps,
geom_p=1.0, geom_rot_deg=0.0,
geom_scale_range=cfg.geom_scale_range,
geom_trans=cfg.geom_trans, geom_flip_p=0.0,
image_size=cfg.image_size)
st_loader = DataLoader(st_ds, batch_size=32, shuffle=False, num_workers=0)
sb = {k: v.to(device) for k, v in next(iter(st_loader)).items()}
l_aug = criterion(_mock_preds_from(sb), sb)
for k, v in l_aug.items():
print(f" {k:<20}: {v.item():.6f}")
keys = ['loss_2d_native', 'loss_3d_native', 'loss_cam', 'loss_reproj', 'loss_mhr_joints']
ok_loss = all(l_aug[k].item() < max(5 * l_clean.get(k, torch.tensor(1e-3)).item(), 1e-3)
for k in keys if k in l_aug)
if not ok_loss:
for k in keys:
if k in l_aug:
lim = max(5 * l_clean.get(k, torch.tensor(1e-3)).item(), 1e-3)
if l_aug[k].item() >= lim:
print(f" ❌ {k}: {l_aug[k].item():.6f} >= {lim:.6f}")
print()
print(f" affine regressor : {'✅' if ok_affine else '❌'}")
print(f" regressor fidelity : {'✅' if ok_reg else '❌'}")
print(f" structural-consistency : {'✅' if ok_struct else '❌'}")
print(f" reprojection-pathway : {'✅' if ok_reproj else '❌'}")
print(f" flip-validity of 3D loss: {'✅' if ok_flip else '❌'}")
print(f" masked perfect-student : {'✅' if ok_loss else '❌'}")
if ok_affine and ok_reg and ok_struct and ok_reproj and ok_flip and ok_loss:
print("\n✅ SELF-TEST PASSED: augmentation is consistent with the MHR pipeline.")
else:
print("\n❌ SELF-TEST FAILED — do not launch a long run.")
def run_overfit_test(steps=3000, subset=8, lr=5e-4):
print(f"--- 1-Batch Overfit Test ({subset} images, {steps} steps) ---")
print("Notice: Geometric augmentations are FORCED OFF to allow loss_pose to stabilize the mesh.")
# The FK warm-up is expressed in optimiser steps and is sized for a real run
# (~0.3 epoch at batch 128). Scale it to this short test so the ramp actually
# completes and the full-strength loss is exercised.
warm = max(1, steps // 5) if cfg.kp3d_warmup_steps > 0 else 0
print(f"Notice: FK-loss warm-up scaled to {warm} steps for this {steps}-step test "
f"(cfg default is {cfg.kp3d_warmup_steps} for a real run).")
model = InstantHMRStudent(cfg, pretrained=True).to(device)
criterion = DistillationLoss(cfg, mhr_module)
criterion.warmup_steps = warm
optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.0)
scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
# 1. SURGICAL INSERTION: Create a dedicated, non-augmented dataset for the overfit test
overfit_dataset = SAM3DStudentDataset(
cfg.data_root,
augment=True, # Keep color/blur augs to prevent total collapse
max_images=subset,
per_dataset_caps=cfg.per_dataset_caps,
geom_p=0.0, # STRICTLY DISABLE SPATIAL AUGS
geom_flip_p=0.0, # STRICTLY DISABLE FLIPS
image_size=cfg.image_size
)
overfit_loader = DataLoader(overfit_dataset, batch_size=subset, shuffle=False, num_workers=0)
batch = next(iter(overfit_loader))
batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
def reg(ls):
# loss_pose_root only exists under cfg.pose_split; the .get keeps the
# regression figure comparable between the two settings.
return (ls['loss_2d_native'].item() + ls['loss_3d_native'].item()
+ ls['loss_cam'].item() + ls['loss_pose'].item()
+ ls.get('loss_pose_root', torch.zeros(())).item())
first_reg = last_reg = last_pa = None
best_pa = float('inf') # 8-sample overfit is noisy at constant LR; judge the best fit reached
best_pa52 = float('inf')
n_skipped = 0
model.train()
for step in range(steps):
optimizer.zero_grad(set_to_none=True)
with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
preds = model(batch["image"], batch["cliff_cond"])
preds = {k: v.float() for k, v in preds.items()}
losses = criterion(preds, batch)
lv = losses['total_loss'].item()
if math.isnan(lv) or math.isinf(lv) or lv > cfg.anomaly_loss_threshold:
n_skipped += 1
continue
scaler.scale(losses['total_loss']).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
scaler.step(optimizer)
scaler.update()
criterion.set_step(step + 1)
if first_reg is None:
first_reg = reg(losses)
last_reg = reg(losses)
if step % 50 == 0 or step == steps - 1:
model.eval()
with torch.no_grad():
p = {k: v.float() for k, v in model(batch["image"], batch["cliff_cond"]).items()}
m = evaluate_hmr_batch(p, batch, cfg, mhr_module)
model.train()
# criterion is now Mesh_PA_MPJPE (the 70 MHR-derived keypoints) —
# the old Head_3D_PA_MPJPE has no meaning without the 3D head.
# Threshold left at 40 mm, unchanged from the original script.
last_pa = m['Mesh_PA_MPJPE']
best_pa = min(best_pa, last_pa)
best_pa52 = min(best_pa52, m['Mesh52_PA_MPJPE'])
print(f" step {step:04d} | tot {losses['total_loss'].item():.3f} "
f"| 3d {losses['loss_3d_native'].item():.4f} "
f"| pose {losses['loss_pose'].item():.4f} "
f"| shape {losses.get('loss_shape', torch.tensor(0)).item():.4f} "
f"| mhr {losses['loss_mhr_joints'].item():.4f} "
f"| Mesh [MPJPE: {m['Mesh_MPJPE']:.1f} | PA: {m['Mesh_PA_MPJPE']:.1f}] mm "
f"| body PA: {m['Mesh_PA_MPJPE_body']:.1f} mm "
f"| Mesh52 [MPJPE: {m['Mesh52_MPJPE']:.1f} | PA: {m['Mesh52_PA_MPJPE']:.1f}] mm")
# THRESHOLD CHANGED, 40.0 -> 55.0 mm, and this is deliberate — the quantity
# under the bar is not the same quantity any more:
# * old: Head_3D_PA_MPJPE, a free coordinate head with no other
# supervision, which overfits 8 images trivially (9.6-18 mm measured).
# * new: Mesh_PA_MPJPE over all 70 MHR-derived keypoints, which includes
# toe tips / finger tips / face points that the mesh must reach through
# forward kinematics.
# On the IDENTICAL 52-anchor mesh metric and the same 1000 steps, this script
# reaches 23.8 / 26.6 / 32.4 mm across three seeds where
# the earlier trainer reaches 34.5 mm — i.e. the mesh got
# BETTER, the bar just now measures the mesh instead of a bypass head.
# Observed 70-keypoint spread across seeds is 34-49 mm; 55 mm keeps headroom
# without being vacuous.
ok = (last_reg < first_reg * 0.10) and (best_pa < 55.0)
print(f"\n{'✅ SUCCESS' if ok else '❌ FAIL'}: regression loss {first_reg:.4f} -> {last_reg:.5f} "
f"| best Mesh PA-MPJPE {best_pa:.1f} mm (final {last_pa:.1f}, bar 55.0) "
f"| best Mesh52 PA-MPJPE {best_pa52:.1f} mm (old-script metric, its value: 34.5) "
f"| anomalous steps skipped: {n_skipped}")
return ok
def fit_keypoint_regressor(cfg, mhr, out_path, per_dataset=6000, val_frac=0.2, rcond=1e-5, seed=0):
"""Least-squares fit of the (70, 127) skeleton -> native-keypoint map.
The fit is parametrised so that every row of W sums to EXACTLY 1: instead of
regressing the keypoint on the raw joints, we regress (kp - J_ref) on
(J_j - J_ref) and recover w_ref = 1 - sum(other weights). An affine
combination commutes with any rigid transform, which is what makes the
augmentation counter-rotations valid on the regressed keypoints.
The design matrix is rank-deficient (rank 116 of 127 — some MHR joints are
exact linear combinations of others), so the pseudo-inverse is truncated at
`rcond` instead of solved directly; the default keeps rank 73 and bounds
|W| at ~4, which is the knee of the accuracy / conditioning trade-off.
"""
import random as _random
rng = _random.Random(seed)
root = Path(cfg.data_root)
dirs = SAM3DStudentDataset._find_dataset_dirs(root)
if not dirs:
raise FileNotFoundError(f"no annotations/ + images/ sub-folders under '{root}'")
files = []
for ann_dir, _ in dirs:
f = sorted(ann_dir.glob("*.npz"))
rng.shuffle(f)
files += f[:per_dataset]
print(f" {ann_dir.parent.name}: {min(len(f), per_dataset)} of {len(f)}")
rng.shuffle(files)
print(f" total {len(files)} samples")
MP, SH, KP = [], [], []
for f in files:
a = np.load(f)
MP.append(a["mhr_model_params"]); SH.append(a["shape_params"]); KP.append(a["joints_3d"])
MP = torch.from_numpy(np.stack(MP)).float()
SH = torch.from_numpy(np.stack(SH)).float()
K = torch.from_numpy(np.stack(KP)).double()
J = []
with torch.no_grad():
for i in range(0, len(MP), 64):
j = mhr.get_joints(MP[i:i + 64].to(device), SH[i:i + 64].to(device))[..., :3]
J.append(mhr.to_vision(j).double().cpu())
J = torch.cat(J)
n = len(J); ntr = int(n * (1.0 - val_frac)); REF = 0
def feats(X):
C = X - X[:, REF:REF + 1, :]
C = torch.cat([C[:, :REF], C[:, REF + 1:]], 1)
return C.permute(0, 2, 1).reshape(-1, 126)
Atr = feats(J[:ntr])
Ytr = (K[:ntr] - J[:ntr, REF:REF + 1, :]).permute(0, 2, 1).reshape(-1, 70)
U, S, Vh = torch.linalg.svd(Atr, full_matrices=False)
keep = S > rcond * S.max()
Si = torch.zeros_like(S); Si[keep] = 1.0 / S[keep]
V = Vh.T @ torch.diag(Si) @ U.T @ Ytr
W = torch.zeros(127, 70, dtype=torch.float64)
W[:REF] = V[:REF]; W[REF + 1:] = V[REF:]
W[REF] = 1.0 - V.sum(0)
pred = (J[ntr:].permute(0, 2, 1).reshape(-1, 127) @ W).reshape(-1, 3, 70).permute(0, 2, 1)
e = (pred - K[ntr:]).norm(dim=-1) * 1000.0
print(f" rank kept {int(keep.sum())}/127 | |W|max {W.abs().max():.2f} | "
f"row sums {W.sum(0).min():.9f}..{W.sum(0).max():.9f}")
print(f" held-out residual: mean {e.mean():.3f} mm | median {e.median():.3f} | "
f"p95 {e.flatten().quantile(0.95):.3f} | p99 {e.flatten().quantile(0.99):.3f} | max {e.max():.2f}")
per_kp = e.mean(0)
worst = torch.argsort(per_kp, descending=True)[:8]
print(" worst keypoints: " + ", ".join(f"{JOINT_NAMES[i]} {per_kp[i]:.1f}mm" for i in worst))
Wnp = W.T.float().numpy().copy() # (70, 127)
np.save(out_path, Wnp)
print(f" saved -> {out_path}")
return Wnp
def parse_args():
p = argparse.ArgumentParser(description="Standalone InstantHMR distillation training.")
p.add_argument("--data_root", type=str, default=None,
help="Entry-point folder containing sub-folders with annotations/ + images/.")
p.add_argument("--output_dir", type=str, default=None,
help="Where checkpoints/logs/exports are written (cfg.log_dir).")
p.add_argument("--mhr_model_path", type=str, default=None,
help="Path to the TorchScript MHR model (mhr_model.pt).")
p.add_argument("--kp_regressor_path", type=str, default=None,
help="Path to the (70,127) skeleton->keypoint regressor .npy.")
p.add_argument("--epochs", type=int, default=None)
p.add_argument("--batch_size", type=int, default=None)
p.add_argument("--lr", type=float, default=None)
p.add_argument("--num_workers", type=int, default=None)
p.add_argument("--max_images", type=int, default=None)
p.add_argument("--harmony4d_cap", type=int, default=None,
help="Max Harmony4D crops kept (randomly sampled). 0 = uncapped/use all.")
p.add_argument("--cap_all", type=int, default=None,
help="Cap EVERY sub-dataset to this many crops (seeded, reproducible). "
"Gives a domain-balanced subset for small-scale ablations, unlike "
"--max_images, which slices a path-sorted list.")
p.add_argument("--no-resume", dest="no_resume", action="store_true",
help="Ignore any existing checkpoint and train from scratch.")
p.add_argument("--self-test", dest="self_test", action="store_true",
help="Run the geometry/MHR-consistency checks and exit.")
p.add_argument("--overfit-test", dest="overfit_test", action="store_true",
help="Run the 1-batch overfit sanity check and exit.")
p.add_argument("--fit-regressor", dest="fit_regressor", action="store_true",
help="Refit the (70,127) skeleton->keypoint regressor from the dataset and exit.")
p.add_argument("--overfit-steps", dest="overfit_steps", type=int, default=3000,
help="Number of steps for --overfit-test (default 3000; see run_overfit_test).")
p.add_argument("--gpu", type=int, default=None, help="GPU count (informational).")
p.add_argument("--exact-landmarks", dest="exact_landmarks", action="store_true",
help="Read the 70 keypoints out of the skinned mesh with the "
"teacher's own fixed mapping, instead of the fitted "
"(70, 127) skeleton-only matrix. Affects loss_3d_native "
"and loss_reproj only; the exported graph is unchanged. "
"Needs assets/mhr_landmarks70.npz.")
p.add_argument("--cont-head", dest="cont_head", action="store_true",
help="Regress the teacher's continuous space (6D root, 260-dim "
"body, 28 scale PCA coefficients, 2x54 hand blocks; 447 "
"numbers) and convert to the 204-vector inside the model, "
"instead of emitting the 204-vector from one nn.Linear. "
"Also switches the root's parameter loss to a rotation "
"matrix metric, which is not separable from the change. "
"Default off: with it off every tensor here is "
"bit-identical to generation 7.")
p.add_argument("--bound-scales", dest="bound_scales", action="store_true",
help="Squash the body-size parameters (root translation and "
"130:204) into the MHR rig's own parameter_limits. "
"Off by default for bit-identity; ON for new runs.")
p.add_argument("--scale-bound-margin", dest="scale_bound_margin", type=float,
default=None, help="How far to widen each limit (default 0.5).")
p.add_argument("--anomaly-safe-fallback", dest="anomaly_safe_fallback",
action="store_true",
help="On an anomalous batch, step on the non-FK terms "
"instead of discarding the batch.")
p.add_argument("--kp3d-warmup-steps", dest="kp3d_warmup_steps", type=int, default=None,
help="Optimiser steps over which the FK-path losses ramp from 0 to full weight.")
p.add_argument("--w_keypoints3d", type=float, default=None,
help="Weight of the MHR-derived 3D keypoint loss (FK path).")
p.add_argument("--losses", choices=("legacy", "rebalanced"), default="legacy",
help="'rebalanced' applies the four loss-budget fixes "
"(w_shape down, pose split + beta, unsquared-Euclidean 3D "
"loss, finger down-weight). 'legacy' reproduces the "
"pre-rebalance recipe bit for bit. See apply_rebalanced_losses.")
p.add_argument("--seed", type=int, default=None,
help="Seed torch/numpy/random (for reproducible sanity runs).")
p.add_argument("--split-wd", dest="split_wd", action="store_true",
help="Exclude depthwise-conv weights, norm params and biases from "
"weight decay (standard for BN/depthwise-heavy backbones).")
p.add_argument("--backbone", type=str, default=None,
help="timm backbone name. Any model whose forward_features() returns a "
"stride-32 (B,C,H,W) map works unchanged (the channel count is read "
"from .num_features). Tested: repvit_m2_3, repvit_m1_5, "
"mobilenetv4_conv_medium, mobilenetv4_hybrid_medium, "
"mobilenetv4_conv_large.")
args, unknown = p.parse_known_args()
if unknown:
print(f"⚠️ Ignoring unrecognized arguments: {unknown}")
return args
cfg = None
train_loader = None
val_loader = None
full_dataset = None
mhr_module = None
def main():
global cfg, train_loader, val_loader, full_dataset, mhr_module
args = parse_args()
cfg = DistillConfig()
if args.data_root is not None: cfg.data_root = args.data_root
if args.output_dir is not None: cfg.log_dir = args.output_dir
if args.mhr_model_path is not None: cfg.mhr_model_path = args.mhr_model_path
if args.kp_regressor_path is not None: cfg.kp_regressor_path = args.kp_regressor_path
if args.epochs is not None: cfg.epochs = args.epochs
if args.batch_size is not None: cfg.batch_size = args.batch_size
if args.lr is not None: cfg.lr = args.lr
if args.num_workers is not None: cfg.num_workers = args.num_workers
if args.max_images is not None: cfg.max_images = args.max_images
if args.cap_all is not None:
cfg.per_dataset_caps = {"*": args.cap_all}
if args.harmony4d_cap is not None:
if args.harmony4d_cap <= 0:
cfg.per_dataset_caps.pop("sam3d_gt_harmony4d", None)
else:
cfg.per_dataset_caps["sam3d_gt_harmony4d"] = args.harmony4d_cap
if args.no_resume: cfg.resume = False
if args.bound_scales: cfg.bound_scales = True
if args.anomaly_safe_fallback: cfg.anomaly_safe_fallback = True
if args.scale_bound_margin is not None: cfg.scale_bound_margin = args.scale_bound_margin
if args.kp3d_warmup_steps is not None: cfg.kp3d_warmup_steps = args.kp3d_warmup_steps
if getattr(args, "cont_head", False):
cfg.cont_head = True
cfg.root_rot_loss = True
if getattr(args, "exact_landmarks", False):
cfg.exact_landmarks = True
if args.losses == "rebalanced": apply_rebalanced_losses(cfg)
# An explicit --w_keypoints3d still wins over the preset.
if args.w_keypoints3d is not None: cfg.w_keypoints3d = args.w_keypoints3d
if args.split_wd: cfg.split_weight_decay = True
if args.backbone is not None:
cfg.backbone = args.backbone
# backbone_feat_dim is informational only: InstantHMRStudent reads the real
# width from backbone.num_features. Keep it in sync so the printout is honest.
try:
import timm as _timm
cfg.backbone_feat_dim = _timm.create_model(
cfg.backbone, pretrained=False, num_classes=0).num_features
except Exception:
pass
if args.seed is not None:
random.seed(args.seed); np.random.seed(args.seed); torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
print(f"seed: {args.seed}")
print("=" * 60)
print(f"Project root : {PROJECT_ROOT}")
print(f"Data root : {cfg.data_root}")
print(f"Output dir : {cfg.log_dir}")
print(f"MHR model : {cfg.mhr_model_path}")
print(f"Backbone : {cfg.backbone} | epochs={cfg.epochs} | batch={cfg.batch_size} | lr={cfg.lr}")
print(f"weight decay : {cfg.weight_decay} | "
f"{'split (no wd on depthwise/norm/bias)' if cfg.split_weight_decay else 'uniform on all params'}")
print(f"max_images : {cfg.max_images if cfg.max_images is not None else 'unlimited (all crops)'}")
print(f"dataset caps : {cfg.per_dataset_caps if cfg.per_dataset_caps else 'none'}")
print(f"geom aug : p={cfg.geom_p} rot±{cfg.geom_rot_deg}° scale±{cfg.geom_scale_range} "
f"trans±{cfg.geom_trans} flip={cfg.geom_flip_p}")
print(f"2D head : SimCC soft-argmax, {cfg.kp2d_bins} bins over ±{cfg.kp2d_range} (w_simcc={cfg.w_simcc})")
print(f"3D pathway : MHR skeleton (127 joints) -> affine (70,127) regressor. No 3D head.")
print(f"kp regressor : {cfg.kp_regressor_path}")
print(f"losses : pose={cfg.w_pose} scale={cfg.w_scale} shape={cfg.w_shape} cam={cfg.w_cam} "
f"kp3d={cfg.w_keypoints3d} kp2d={cfg.w_keypoints2d} simcc={cfg.w_simcc} "
f"mhr_joints={cfg.w_3d_joints} reproj={cfg.w_reproj}")
print("=" * 60)
if not Path(cfg.mhr_model_path).exists():
raise FileNotFoundError(
f"MHR model not found at '{cfg.mhr_model_path}'. "
f"Place mhr_model.pt in the checkpoints/ folder or pass --mhr_model_path."
)
if args.fit_regressor:
mhr_module = MHRForwardPass(cfg.mhr_model_path, device)
print("--- Fitting the (70, 127) skeleton -> native-keypoint regressor ---")
fit_keypoint_regressor(cfg, mhr_module, cfg.kp_regressor_path)
return
W = None
if Path(cfg.kp_regressor_path).exists():
W = np.load(cfg.kp_regressor_path)
print(f"✅ keypoint regressor loaded: {W.shape} from {cfg.kp_regressor_path}")
else:
print(f"⚠️ no regressor at '{cfg.kp_regressor_path}' — fitting one now "
f"(deterministic; rerun with --fit-regressor to redo it).")
W = fit_keypoint_regressor(cfg, MHRForwardPass(cfg.mhr_model_path, device),
cfg.kp_regressor_path)
train_loader, val_loader, full_dataset = build_dataloaders(cfg)
mhr_module = MHRForwardPass(
cfg.mhr_model_path, device, kp_regressor=W,
n_verts=cfg.n_verts if cfg.w_verts > 0 else 0,
landmark_assets=cfg.landmark_assets_path if cfg.exact_landmarks else None)
if args.self_test:
run_self_tests()
return
if args.overfit_test:
run_overfit_test(steps=args.overfit_steps)
return
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
train_instant_hmr()
print(f"Export the best checkpoint with: python tools/export_onnx.py --ckpt {cfg.log_dir}/best_student_model_v3.pth --output model.onnx")
if __name__ == "__main__":
main()