Skip to content
Python

feat: added optimized models for 3D pose without mesh

mohamdev759cb0399114
diff --git a/.gitignore b/.gitignore
index 028a1c3..fb52f5b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,7 @@ models/*.onnx
models/*.pt
models/*.pth
models/mhr_assets/
+commands.md
!models/README.md
# Transfer archives (dataset tarballs)
diff --git a/instanthmr_distill_train/submit_train_distill.sh b/instanthmr_distill_train/submit_train_distill.sh
index 227bcb1..1e11cd5 100755
--- a/instanthmr_distill_train/submit_train_distill.sh
+++ b/instanthmr_distill_train/submit_train_distill.sh
@@ -12,7 +12,7 @@ GPU_REQUIRED=true
# --data_root points at the dataset synced by datasynch_perso (see below).
# It is synced next to this project folder, i.e. ../instanthmr_data once we
# cd into instanthmr_distill_train.
-EXEC="python3 train_distill.py --data_root ../instanthmr_data $useropt"
+EXEC="python3 -u train_distill.py --data_root /datasets/instanthmr_data --num_workers 8 --batch_size 128 $useropt"
# SBATCH --output/--error paths are relative to the submission directory, so
# create the log folder before submitting.
@@ -30,17 +30,16 @@ sbatch ${sbatchopt} << eof
#SBATCH --cpus-per-task=8
#SBATCH --output=instanthmr_distill/stdout.txt
#SBATCH --error=instanthmr_distill/stderr.txt
-#SBATCH --partition=gpu
-#SBATCH --nodelist=dolcino
+#SBATCH --account=dept_rob
+#SBATCH --partition=robgpu
#SBATCH --gres=gpu:1
#SBATCH --mem-per-gpu=50000
#SBATCH --time=4-00:00:00
. /pfcalcul/tools/sbatchHelpers5.sh
-/pfcalcul/work/kchalabi/envs/lstm/datasynch_perso /datasets/instanthmr_data
+/pfcalcul/tools/datasync /datasets/instanthmr_data
-cd instanthmr_distill_train
time $EXEC
diff --git a/instanthmr_distill_train/submit_train_distill_pose3d.sh b/instanthmr_distill_train/submit_train_distill_pose3d.sh
new file mode 100755
index 0000000..dda9c6b
--- /dev/null
+++ b/instanthmr_distill_train/submit_train_distill_pose3d.sh
@@ -0,0 +1,49 @@
+#!/bin/sh -e
+
+echo "example: $0 --gpu 1"
+
+# if `requirements.txt` is present, virtual environment is installed and used
+EXEC="python3 train_distill_pose3d.py"
+
+GPU_REQUIRED=true
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+# --data_root points at the dataset synced by datasynch_perso (see below).
+# It is synced next to this project folder, i.e. ../instanthmr_data once we
+# cd into instanthmr_distill_train.
+EXEC="python3 -u train_distill_pose3d.py --data_root /datasets/instanthmr_data --num_workers 8 --batch_size 128 $useropt"
+
+# SBATCH --output/--error paths are relative to the submission directory, so
+# create the log folder before submitting.
+mkdir -p instanthmr_distill
+
+jobMessage=$(
+######################### SBATCH launcher
+######################### #SBATCH --option # <= this is an enabled paramater
+######################### ##SBATCH --option # <= this is a disabled parameter
+sbatch ${sbatchopt} << eof
+#!/bin/bash
+#SBATCH --job-name="instanthmr_distill"
+#SBATCH --mail-type=ALL
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --output=instanthmr_distill/stdout.txt
+#SBATCH --error=instanthmr_distill/stderr.txt
+#SBATCH --account=dept_rob
+#SBATCH --partition=robgpu
+#SBATCH --gres=gpu:1
+#SBATCH --mem-per-gpu=50000
+#SBATCH --time=4-00:00:00
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+/pfcalcul/tools/datasync /datasets/instanthmr_data
+
+
+time $EXEC
+
+eof
+)
+
+showSubmitted
diff --git a/instanthmr_distill_train/train_distill.py b/instanthmr_distill_train/train_distill.py
index 3889d2d..495d496 100644
--- a/instanthmr_distill_train/train_distill.py
+++ b/instanthmr_distill_train/train_distill.py
@@ -81,7 +81,7 @@ class DistillConfig:
# each with its own annotations/ and images/ directories.
data_root: str = str(DEFAULT_DATA_ROOT)
mhr_model_path: str = str(PROJECT_ROOT / "checkpoints/mhr_model.pt") # path to MHR model
- log_dir: str = str(PROJECT_ROOT / "runs/distill_repvit_cliff_v2") # Bumped to v2!
+ log_dir: str = str(PROJECT_ROOT / "runs/distill_repvit_cliff_v3") # v3: full balanced dataset (Harmony4D capped to 150k)
# --- Data ---
image_size: int = 224
@@ -120,7 +120,8 @@ class DistillConfig:
weight_decay: float = 1e-4
epochs: int = 400
warmup_epochs: int = 3
- grad_clip: float = 5.0
+ grad_clip: float = 1.0 # tighter clip: 5.0 let a bad step blow up the MHR loss
+ anomaly_loss_threshold: float = 100.0 # skip a batch whose loss exceeds this (normal is <2)
use_amp: bool = True
ema_decay: float = 0.9998 # EMA shadow-weights decay (Cell 11)
early_stop_patience: int = 150 # stop after N epochs where NEITHER raw nor ema improved (Cell 11)
@@ -191,7 +192,11 @@ class SAM3DStudentDataset(Dataset):
if augment:
tfm_list.extend([
- transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),
+ # ColorJitter (esp. hue -> RGB/HSV round-trip) is the priciest CPU op in
+ # the pipeline. Applying it on EVERY image was starving the GPU, so run it
+ # half the time via RandomApply — still solid colour regularisation.
+ transforms.RandomApply([transforms.ColorJitter(
+ brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),
RandomPixelate(p=0.2, min_res=48, max_res=112),
transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),
])
@@ -370,21 +375,26 @@ def build_dataloaders(cfg):
generator = torch.Generator().manual_seed(42)
_, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=generator)
+ # Keep workers alive across epochs and prefetch more batches so the GPU
+ # stays fed (helps the ~40% idle time seen in nvidia-smi).
+ loader_kwargs = dict(num_workers=cfg.num_workers, pin_memory=True)
+ if cfg.num_workers > 0:
+ loader_kwargs["persistent_workers"] = True
+ loader_kwargs["prefetch_factor"] = 4
+
train_loader = DataLoader(
train_dataset,
batch_size=cfg.batch_size,
shuffle=True,
- num_workers=cfg.num_workers,
- pin_memory=True,
drop_last=True,
+ **loader_kwargs,
)
val_loader = DataLoader(
val_dataset,
batch_size=cfg.batch_size,
shuffle=False,
- num_workers=cfg.num_workers,
- pin_memory=True,
+ **loader_kwargs,
)
print(f"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}")
@@ -964,9 +974,15 @@ def train_instant_hmr():
preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}
losses = criterion(preds_fp32, batch)
loss = losses['total_loss']
- if math.isnan(loss.item()) or math.isinf(loss.item()):
+ loss_val = loss.item()
+ # Skip NaN/Inf AND anomalously large (but finite) batches — a single
+ # ~1e14 MHR-loss batch would otherwise take a step and diverge training.
+ if math.isnan(loss_val) or math.isinf(loss_val):
print("⚠️ WARNING: NaN/Inf loss detected! Skipping step.")
continue
+ if loss_val > cfg.anomaly_loss_threshold:
+ print(f"⚠️ WARNING: anomalous loss {loss_val:.1f} > {cfg.anomaly_loss_threshold} — skipping step.")
+ continue
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
diff --git a/instanthmr_distill_train/train_distill_pose3d.py b/instanthmr_distill_train/train_distill_pose3d.py
new file mode 100644
index 0000000..6ff06a3
--- /dev/null
+++ b/instanthmr_distill_train/train_distill_pose3d.py
@@ -0,0 +1,708 @@
+#!/usr/bin/env python3
+# ============================================================
+# Keypoint + camera student — NO MHR mesh model / params.
+# ------------------------------------------------------------
+# A stripped-down sibling of train_distill.py. It predicts and
+# supervises the 70 native 2D + 3D keypoints AND the camera
+# translation (cam_trans) — which is NOT an MHR parameter and is
+# required by the demo (joints_3d_cam = joints_3d + cam_trans).
+#
+# Removed vs train_distill.py: the MHR body rig (mhr_model.pt),
+# the mhr_params / shape_params heads, and every MHR-derived loss
+# (mhr_joints / structural / feet / reproj_mhr / mesh metrics) —
+# i.e. exactly the terms that were causing the periodic loss spikes.
+#
+# Model out: cam_trans [B,3], joints_2d [B,70,2] ([-1,1] crop),
+# joints_3d [B,70,3] (root-centred metres).
+# Loss: SmoothL1(2D) + SmoothL1(3D) + MSE(cam) + L1(reproj).
+# Metrics: MPJPE / PA-MPJPE on the 3D head.
+# Export: 5 ONNX outputs in demo order (mhr_params & shape_params
+# zero-filled) so it works with the existing demo.py.
+#
+# Sanity: python3 train_distill_pose3d.py --overfit-test --data_root ../data
+# Full: python3 train_distill_pose3d.py --data_root /datasets/instanthmr_data --num_workers 8
+# ============================================================
+import os
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import argparse
+import math
+import random
+import gc
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.optim.lr_scheduler as lr_scheduler
+from torch.utils.data import Dataset, DataLoader, random_split
+from torch.optim.swa_utils import AveragedModel
+
+import torchvision.transforms as transforms
+import torchvision.transforms.functional as F_t
+from torchvision.transforms import InterpolationMode
+
+import timm
+from tqdm import tqdm
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"Using device: {device}")
+
+PROJECT_ROOT = Path(__file__).resolve().parent
+DEFAULT_DATA_ROOT = "/pfcalcul/datasets/instanthmr_data/"
+
+
+# ============================================================
+# Configuration
+# ============================================================
+@dataclass
+class PoseConfig:
+ data_root: str = str(DEFAULT_DATA_ROOT)
+ log_dir: str = str(PROJECT_ROOT / "runs/distill_pose3d_v1")
+
+ # --- Data ---
+ image_size: int = 224
+ max_images: int | None = None
+ per_dataset_caps: dict = field(default_factory=lambda: {"sam3d_gt_harmony4d": 300_000})
+ val_split: float = 0.1
+ num_workers: int = 4
+ augment: bool = True
+
+ # --- Model ---
+ backbone: str = "repvit_m2_3"
+ d_model: int = 512
+ n_heads: int = 8
+ n_decoder_layers: int = 4
+ dropout: float = 0.1
+ cliff_dim: int = 3
+ cam_dim: int = 3
+ num_joints: int = 70
+
+ # --- Training ---
+ batch_size: int = 64
+ lr: float = 3e-4
+ weight_decay: float = 1e-4
+ epochs: int = 150
+ grad_clip: float = 1.0
+ anomaly_loss_threshold: float = 100.0
+ use_amp: bool = True
+ ema_decay: float = 0.9998
+ early_stop_patience: int = 150
+ resume: bool = True
+
+ # --- Loss weights ---
+ w_keypoints2d: float = 10.0
+ w_keypoints3d: float = 10.0
+ w_cam: float = 0.1 # direct MSE supervision on cam_trans (stable; no 1/z reprojection)
+
+
+# ============================================================
+# Dataset (same crops as train_distill.py; keeps cam_trans + camera intrinsics)
+# ============================================================
+class RandomPixelate:
+ def __init__(self, p=0.2, min_res=32, max_res=112):
+ self.p, self.min_res, self.max_res = p, min_res, max_res
+
+ def __call__(self, img):
+ if random.random() < self.p:
+ orig_w, orig_h = img.size
+ degraded_h = random.randint(self.min_res, self.max_res)
+ degraded_w = int(orig_w * (degraded_h / orig_h))
+ img = F_t.resize(img, [degraded_h, degraded_w], interpolation=InterpolationMode.NEAREST)
+ img = F_t.resize(img, [orig_h, orig_w], interpolation=InterpolationMode.NEAREST)
+ return img
+
+
+class SAM3DStudentDataset(Dataset):
+ def __init__(self, data_root: str, image_size: int = 224, max_images: int | None = None,
+ augment: bool = True, per_dataset_caps: dict | None = None):
+ super().__init__()
+ self.image_size = image_size
+ self.data_root = Path(data_root)
+
+ tfm_list = [transforms.Resize((image_size, image_size))]
+ if augment:
+ tfm_list.extend([
+ transforms.RandomApply([transforms.ColorJitter(
+ brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),
+ RandomPixelate(p=0.2, min_res=48, max_res=112),
+ transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),
+ ])
+ tfm_list.extend([
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+ ])
+ if augment:
+ tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))
+ self.transform = transforms.Compose(tfm_list)
+
+ self.pairs = []
+ dataset_dirs = self._find_dataset_dirs(self.data_root)
+ if not dataset_dirs:
+ raise FileNotFoundError(f"No annotations/images sub-folders under '{self.data_root}'.")
+
+ cap_rng = random.Random(42)
+ caps = per_dataset_caps or {}
+ for ann_dir, img_dir in dataset_dirs:
+ sub_name = ann_dir.parent.name
+ sub_pairs = []
+ for npz_path in sorted(ann_dir.glob("*.npz")):
+ stem = npz_path.stem
+ img_path = img_dir / f"{stem}.jpg"
+ if not img_path.exists():
+ img_path = img_dir / f"{stem}.png"
+ if img_path.exists():
+ sub_pairs.append((img_path, npz_path))
+ cap = caps.get(sub_name)
+ if cap is not None and len(sub_pairs) > cap:
+ n_before = len(sub_pairs)
+ sub_pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
+ sub_pairs = cap_rng.sample(sub_pairs, cap)
+ print(f" [cap] {sub_name}: sampled {cap:,} of {n_before:,} crops")
+ self.pairs.extend(sub_pairs)
+
+ self.pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
+ if max_images is not None:
+ self.pairs = self.pairs[:max_images]
+ print(f" Found {len(dataset_dirs)} sub-folder(s) under {self.data_root}; "
+ f"{len(self.pairs)} valid (image, npz) pairs (augment={augment}).")
+
+ @staticmethod
+ def _find_dataset_dirs(root: Path):
+ dirs = []
+ if (root / "annotations").is_dir() and (root / "images").is_dir():
+ dirs.append((root / "annotations", root / "images"))
+ if root.is_dir():
+ for sub in sorted(root.iterdir()):
+ if not sub.is_dir():
+ continue
+ ann, img = sub / "annotations", sub / "images"
+ if ann.is_dir() and img.is_dir():
+ dirs.append((ann, img))
+ return dirs
+
+ def __len__(self):
+ return len(self.pairs)
+
+ def __getitem__(self, idx):
+ img_path, npz_path = self.pairs[idx]
+ image = self.transform(Image.open(img_path).convert("RGB"))
+
+ ann = np.load(npz_path)
+ orig_h, orig_w = ann["orig_shape"]
+
+ sq_bbox = ann["bbox_square"]
+ sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]
+ orig_crop_size = sq_bbox[2] - sq_bbox[0]
+ true_scale = self.image_size / max(orig_crop_size, 1.0)
+
+ joints_2d = ann["joints_2d"].copy()
+ joints_2d[:, 0] = (joints_2d[:, 0] - sq_x1) * true_scale
+ joints_2d[:, 1] = (joints_2d[:, 1] - sq_y1) * true_scale
+ joints_2d[:, 0] = (joints_2d[:, 0] / self.image_size) * 2.0 - 1.0
+ joints_2d[:, 1] = (joints_2d[:, 1] / self.image_size) * 2.0 - 1.0
+ joints_2d = torch.from_numpy(joints_2d).float()
+
+ tight_bbox = ann["bbox"]
+ cx = (tight_bbox[0] + tight_bbox[2]) / 2.0
+ cy = (tight_bbox[1] + tight_bbox[3]) / 2.0
+ cx_norm = 2.0 * (cx / orig_w) - 1.0
+ cy_norm = 2.0 * (cy / orig_h) - 1.0
+ b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])
+ b_scale = b_size / max(orig_w, orig_h)
+ cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)
+
+ return {
+ "image": image, # [3,224,224]
+ "cliff_cond": cliff_cond, # [3]
+ "joints_2d": joints_2d, # [70,2] crop [-1,1]
+ "joints_3d": torch.from_numpy(ann["joints_3d"]).float(), # [70,3] root-centred
+ "cam_trans": torch.from_numpy(ann["cam_trans"]).float(), # [3] camera translation
+ }
+
+
+def build_dataloaders(cfg):
+ full_dataset = SAM3DStudentDataset(cfg.data_root, augment=cfg.augment,
+ max_images=cfg.max_images, per_dataset_caps=cfg.per_dataset_caps)
+ val_dataset_clean = SAM3DStudentDataset(cfg.data_root, augment=False,
+ max_images=cfg.max_images, per_dataset_caps=cfg.per_dataset_caps)
+ n_val = int(len(full_dataset) * cfg.val_split)
+ n_train = len(full_dataset) - n_val
+ g = torch.Generator().manual_seed(42)
+ train_dataset, _ = random_split(full_dataset, [n_train, n_val], generator=g)
+ g = torch.Generator().manual_seed(42)
+ _, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=g)
+
+ loader_kwargs = dict(num_workers=cfg.num_workers, pin_memory=True)
+ if cfg.num_workers > 0:
+ loader_kwargs["persistent_workers"] = True
+ loader_kwargs["prefetch_factor"] = 4
+ train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
+ drop_last=True, **loader_kwargs)
+ val_loader = DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False, **loader_kwargs)
+ print(f"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}")
+ return train_loader, val_loader, full_dataset
+
+
+# ============================================================
+# Model — RepViT + decoder + cam head + 2D/3D heads (no MHR head)
+# ============================================================
+def get_2d_sincos_pos_embed(embed_dim, grid_size):
+ grid_h, grid_w = grid_size, grid_size
+ grid_y, grid_x = torch.meshgrid(torch.arange(grid_h), torch.arange(grid_w), indexing='ij')
+ omega = torch.arange(embed_dim // 4).float() / (embed_dim // 4)
+ omega = 1.0 / (10000 ** omega)
+ out_y = grid_y.flatten().unsqueeze(1) * omega.unsqueeze(0)
+ out_x = grid_x.flatten().unsqueeze(1) * omega.unsqueeze(0)
+ pe_y = torch.cat([torch.sin(out_y), torch.cos(out_y)], dim=1)
+ pe_x = torch.cat([torch.sin(out_x), torch.cos(out_x)], dim=1)
+ return torch.cat([pe_y, pe_x], dim=1).unsqueeze(0)
+
+
+class InstantHMRPoseStudent(nn.Module):
+ def __init__(self, cfg, pretrained=True):
+ super().__init__()
+ self.cfg = cfg
+ self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)
+ embed_dim = self.backbone.num_features
+ self.feat_proj = nn.Linear(embed_dim, cfg.d_model)
+
+ self.grid_size = cfg.image_size // 32
+ self.register_buffer("mem_pos_embed", get_2d_sincos_pos_embed(cfg.d_model, self.grid_size))
+
+ # 1 global (camera) + 70 2D + 70 3D query tokens
+ self.num_global = 1
+ self.num_2d = cfg.num_joints
+ self.num_3d = cfg.num_joints
+ self.total_queries = self.num_global + self.num_2d + self.num_3d
+ self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model) * 0.02)
+
+ self.cond_proj = nn.Sequential(
+ nn.Linear(cfg.cliff_dim, 128), nn.GELU(), nn.Linear(128, cfg.d_model))
+
+ decoder_layer = nn.TransformerDecoderLayer(
+ d_model=cfg.d_model, nhead=cfg.n_heads, dim_feedforward=cfg.d_model * 4,
+ dropout=cfg.dropout, activation="gelu", batch_first=True, norm_first=True)
+ self.transformer = nn.TransformerDecoder(decoder_layer, num_layers=cfg.n_decoder_layers)
+
+ # Camera head (translation only) + 2D head + 3D head
+ self.head_cam = nn.Linear(cfg.d_model, cfg.cam_dim)
+ self.head_2d_feat = nn.Sequential(nn.Linear(cfg.d_model, 256), nn.GELU())
+ self.head_2d_xy = nn.Sequential(nn.Linear(256, 2))
+ self.head_3d = nn.Sequential(nn.Linear(cfg.d_model, 256), nn.GELU(), nn.Linear(256, 3))
+
+ # Init camera near (0, 0, 2m) depth; tiny init on spatial heads for stability
+ nn.init.normal_(self.head_cam.weight, mean=0.0, std=1e-4)
+ nn.init.constant_(self.head_cam.bias, 0.0)
+ nn.init.constant_(self.head_cam.bias[-1], 2.0)
+ nn.init.normal_(self.head_2d_xy[0].weight, mean=0.0, std=1e-4)
+ nn.init.constant_(self.head_2d_xy[0].bias, 0.0)
+ nn.init.normal_(self.head_3d[-1].weight, mean=0.0, std=1e-4)
+ nn.init.constant_(self.head_3d[-1].bias, 0.0)
+
+ def forward(self, images, cliff_cond):
+ B = images.shape[0]
+ img_feats = self.backbone.forward_features(images)
+ if img_feats.dim() == 4:
+ img_feats = img_feats.flatten(2).transpose(1, 2)
+ memory = self.feat_proj(img_feats) + self.mem_pos_embed
+
+ queries = self.query_embed.expand(B, -1, -1) + self.cond_proj(cliff_cond).unsqueeze(1)
+ shared = self.transformer(tgt=queries, memory=memory)
+
+ feat_global = shared[:, 0, :]
+ feat_2d = shared[:, 1:1 + self.num_2d, :]
+ feat_3d = shared[:, 1 + self.num_2d:, :]
+
+ pred_cam = self.head_cam(feat_global)
+ pred_2d = self.head_2d_xy(self.head_2d_feat(feat_2d))
+ pred_3d = self.head_3d(feat_3d)
+ return {"cam_trans": pred_cam, "joints_2d": pred_2d, "joints_3d": pred_3d}
+
+
+# ============================================================
+# Loss — keypoints + camera (no MHR)
+# ============================================================
+class Pose3DLoss(nn.Module):
+ def __init__(self, cfg):
+ super().__init__()
+ self.cfg = cfg
+ self.l1 = nn.SmoothL1Loss()
+ self.mse = nn.MSELoss()
+
+ def forward(self, preds, targets):
+ losses = {}
+ losses['loss_2d'] = self.l1(preds["joints_2d"][..., :2], targets["joints_2d"]) * self.cfg.w_keypoints2d
+ losses['loss_3d'] = self.l1(preds["joints_3d"][..., :3], targets["joints_3d"]) * self.cfg.w_keypoints3d
+ # Direct camera supervision only — SmoothL1 (bounded gradient, robust to large
+ # depth targets); MSE here diverges at the overfit LR, no 1/z reprojection term.
+ losses['loss_cam'] = self.l1(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam
+ losses['total_loss'] = losses['loss_2d'] + losses['loss_3d'] + losses['loss_cam']
+ return losses
+
+
+# ============================================================
+# Metrics — MPJPE / PA-MPJPE on the 3D head (mm)
+# ============================================================
+def batched_procrustes_alignment(pred_pts, gt_pts):
+ mu_pred = pred_pts.mean(dim=1, keepdim=True)
+ mu_gt = gt_pts.mean(dim=1, keepdim=True)
+ pred_c, gt_c = pred_pts - mu_pred, gt_pts - mu_gt
+ norm_pred = torch.linalg.norm(pred_c, dim=(1, 2), keepdim=True)
+ norm_gt = torch.linalg.norm(gt_c, dim=(1, 2), keepdim=True)
+ pred_n = pred_c / torch.clamp(norm_pred, min=1e-8)
+ gt_n = gt_c / torch.clamp(norm_gt, min=1e-8)
+ H = torch.bmm(pred_n.transpose(1, 2), gt_n)
+ U, S, Vh = torch.linalg.svd(H)
+ R = torch.bmm(U, Vh)
+ det = torch.linalg.det(R)
+ det_sign = torch.where(det < 0, torch.tensor(-1.0, device=pred_pts.device),
+ torch.tensor(1.0, device=pred_pts.device)).unsqueeze(-1)
+ U_fixed = U.clone()
+ U_fixed[:, :, 2] *= det_sign
+ R_fixed = torch.bmm(U_fixed, Vh)
+ S_fixed = S.clone()
+ S_fixed[:, 2] *= det_sign.squeeze(-1)
+ scale = S_fixed.sum(dim=-1, keepdim=True).unsqueeze(-1) * (norm_gt / torch.clamp(norm_pred, min=1e-8))
+ return scale * torch.bmm(pred_c, R_fixed) + mu_gt
+
+
+def evaluate_pose_batch(preds, targets):
+ metrics = {}
+ pred_3d = preds["joints_3d"][..., :3]
+ gt_3d = targets["joints_3d"][..., :3]
+ pred_rr = pred_3d - pred_3d[:, 0:1, :]
+ gt_rr = gt_3d - gt_3d[:, 0:1, :]
+ metrics['MPJPE'] = torch.linalg.norm(pred_rr - gt_rr, dim=-1).mean().item() * 1000.0
+ pred_al = batched_procrustes_alignment(pred_3d, gt_3d)
+ metrics['PA_MPJPE'] = torch.linalg.norm(pred_al - gt_3d, dim=-1).mean().item() * 1000.0
+ return metrics
+
+
+# ============================================================
+# 1-batch overfit test — proves gradients flow / model can learn
+# ============================================================
+def run_overfit_test(cfg, train_loader, steps=800, subset=8, lr=1e-3):
+ print(f"--- 1-Batch Overfit Test ({subset} images, {steps} steps) ---")
+ model = InstantHMRPoseStudent(cfg, pretrained=True).to(device)
+ criterion = Pose3DLoss(cfg)
+ optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.0)
+ scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
+
+ batch = next(iter(train_loader))
+ batch = {k: (v[:subset].to(device) if isinstance(v, torch.Tensor) else v) for k, v in batch.items()}
+
+ first_loss, last_loss = None, None
+ model.train()
+ for step in range(steps):
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds = model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ loss = losses['total_loss']
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
+ scaler.step(optimizer)
+ scaler.update()
+ if first_loss is None:
+ first_loss = loss.item()
+ last_loss = loss.item()
+ if step % 50 == 0 or step == steps - 1:
+ model.eval()
+ with torch.no_grad():
+ p = {k: v.float() for k, v in model(batch["image"], batch["cliff_cond"]).items()}
+ m = evaluate_pose_batch(p, batch)
+ model.train()
+ print(f" step {step:04d} | loss {loss.item():.5f} | 2d {losses['loss_2d'].item():.4f} "
+ f"| 3d {losses['loss_3d'].item():.4f} | cam {losses['loss_cam'].item():.4f} "
+ f"| MPJPE {m['MPJPE']:.1f} mm | PA {m['PA_MPJPE']:.1f} mm")
+
+ ok = last_loss < first_loss * 0.05
+ print(f"\n{'✅ SUCCESS' if ok else '❌ FAIL'}: loss {first_loss:.4f} -> {last_loss:.5f} "
+ f"({'overfit, gradients flow' if ok else 'did NOT drop enough — check the setup'})")
+ return ok
+
+
+# ============================================================
+# EMA helper (decay warmup)
+# ============================================================
+def make_ema_avg_fn(max_decay):
+ @torch.no_grad()
+ def avg_fn(ema_params, model_params, num_averaged):
+ if not (torch.is_floating_point(ema_params[0]) or torch.is_complex(ema_params[0])):
+ for e, m in zip(ema_params, model_params):
+ e.copy_(m)
+ return
+ n = num_averaged.item() if torch.is_tensor(num_averaged) else float(num_averaged)
+ decay = min(max_decay, (1.0 + n) / (10.0 + n))
+ torch._foreach_lerp_(ema_params, model_params, 1.0 - decay)
+ return avg_fn
+
+
+# ============================================================
+# Training loop — EMA + early stopping + guards, keypoints + camera
+# ============================================================
+def train_pose3d(cfg, train_loader, val_loader):
+ print(f"--- Training {cfg.backbone} (pose3d) | EMA={cfg.ema_decay} | patience={cfg.early_stop_patience} ---")
+ os.makedirs(cfg.log_dir, exist_ok=True)
+
+ model = InstantHMRPoseStudent(cfg, pretrained=True).to(device)
+ criterion = Pose3DLoss(cfg)
+ optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
+ scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
+ steps_per_epoch = len(train_loader)
+ scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=cfg.lr, epochs=cfg.epochs,
+ steps_per_epoch=steps_per_epoch, pct_start=0.1)
+
+ best_raw_path = os.path.join(cfg.log_dir, "best_pose3d_raw.pth")
+ best_ema_path = os.path.join(cfg.log_dir, "best_pose3d_ema.pth")
+ best_ckpt_path = os.path.join(cfg.log_dir, "best_pose3d.pth")
+
+ start_epoch = 0
+ best_raw_pa = best_ema_pa = best_overall_pa = float('inf')
+ epochs_no_improve = 0
+
+ if cfg.resume and os.path.exists(best_ckpt_path):
+ print(f"🔄 Resuming from {best_ckpt_path}")
+ ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=False)
+ model.load_state_dict(ckpt["model_state_dict"], strict=False)
+ for k, obj in [("optimizer_state_dict", optimizer), ("scaler_state_dict", scaler),
+ ("scheduler_state_dict", scheduler)]:
+ if k in ckpt:
+ obj.load_state_dict(ckpt[k])
+ start_epoch = ckpt.get("epoch", -1) + 1
+ best_overall_pa = best_raw_pa = best_ema_pa = ckpt.get("val_pa_mpjpe", float('inf'))
+ print(f"✅ Resumed at epoch {start_epoch}. Best PA-MPJPE {best_overall_pa:.1f} mm.")
+ else:
+ print("🚀 Training from scratch.")
+
+ ema_model = AveragedModel(model, multi_avg_fn=make_ema_avg_fn(cfg.ema_decay), use_buffers=True)
+
+ @torch.no_grad()
+ def run_validation(eval_model, tag, epoch):
+ eval_model.eval()
+ agg = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'cam': 0.0, 'MPJPE': 0.0, 'PA_MPJPE': 0.0}
+ n = 0
+ vbar = tqdm(val_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]", leave=False)
+ for batch in vbar:
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds = eval_model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ if math.isnan(losses['total_loss'].item()) or math.isinf(losses['total_loss'].item()):
+ continue
+ agg['total'] += losses['total_loss'].item()
+ agg['2d'] += losses['loss_2d'].item()
+ agg['3d'] += losses['loss_3d'].item()
+ agg['cam'] += losses['loss_cam'].item()
+ bm = evaluate_pose_batch(preds, batch)
+ agg['MPJPE'] += bm['MPJPE']
+ agg['PA_MPJPE'] += bm['PA_MPJPE']
+ n += 1
+ vbar.set_postfix({'PA-MPJPE': f"{agg['PA_MPJPE']/max(n,1):.1f}mm"})
+ vbar.close()
+ return {k: v / max(n, 1) for k, v in agg.items()}, n
+
+ for epoch in range(start_epoch, cfg.epochs):
+ model.train()
+ tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'cam': 0.0}
+ nb = 0
+ pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Train]")
+ for batch in pbar:
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds_fp16 = model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds_fp16.items()}
+ losses = criterion(preds, batch)
+ loss = losses['total_loss']
+ loss_val = loss.item()
+ if math.isnan(loss_val) or math.isinf(loss_val):
+ print("⚠️ WARNING: NaN/Inf loss detected! Skipping step.")
+ continue
+ if loss_val > cfg.anomaly_loss_threshold:
+ print(f"⚠️ WARNING: anomalous loss {loss_val:.1f} > {cfg.anomaly_loss_threshold} — skipping step.")
+ continue
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
+ scale_before = scaler.get_scale()
+ scaler.step(optimizer)
+ scaler.update()
+ if scale_before <= scaler.get_scale():
+ scheduler.step()
+ ema_model.update_parameters(model)
+ tm['total'] += loss_val
+ tm['2d'] += losses['loss_2d'].item()
+ tm['3d'] += losses['loss_3d'].item()
+ tm['cam'] += losses['loss_cam'].item()
+ nb += 1
+ pbar.set_postfix({'Tot': f"{loss_val:.4f}"})
+ pbar.close()
+ avg_train = {k: v / max(nb, 1) for k, v in tm.items()}
+
+ raw_val, n_raw = run_validation(model, "raw", epoch)
+ ema_val, n_ema = run_validation(ema_model, "ema", epoch)
+
+ print(f"\n📈 Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}")
+ print(f" [Train] Tot: {avg_train['total']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f} | cam: {avg_train['cam']:.4f}")
+ print(f" [Val RAW] Tot: {raw_val['total']:.4f} | PA-MPJPE: {raw_val['PA_MPJPE']:.1f} | MPJPE: {raw_val['MPJPE']:.1f} mm")
+ print(f" [Val EMA] Tot: {ema_val['total']:.4f} | PA-MPJPE: {ema_val['PA_MPJPE']:.1f} | MPJPE: {ema_val['MPJPE']:.1f} mm")
+
+ def save_dict(m, val_metric, source):
+ return {'epoch': epoch, 'model_state_dict': m.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ 'scaler_state_dict': scaler.state_dict(),
+ 'scheduler_state_dict': scheduler.state_dict(),
+ 'val_pa_mpjpe': val_metric, 'source': source}
+
+ improved_raw = n_raw > 0 and raw_val['PA_MPJPE'] < best_raw_pa
+ improved_ema = n_ema > 0 and ema_val['PA_MPJPE'] < best_ema_pa
+ if improved_raw:
+ best_raw_pa = raw_val['PA_MPJPE']
+ torch.save(save_dict(model, best_raw_pa, 'raw'), best_raw_path)
+ if improved_ema:
+ best_ema_pa = ema_val['PA_MPJPE']
+ torch.save(save_dict(ema_model.module, best_ema_pa, 'ema'), best_ema_path)
+
+ current = min(raw_val['PA_MPJPE'], ema_val['PA_MPJPE'])
+ if current < best_overall_pa - 1e-4:
+ best_overall_pa = current
+ use_ema = ema_val['PA_MPJPE'] <= raw_val['PA_MPJPE']
+ better = ema_model.module if use_ema else model
+ torch.save(save_dict(better, best_overall_pa, 'ema' if use_ema else 'raw'), best_ckpt_path)
+ print(f"💾 New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}")
+
+ if improved_raw or improved_ema:
+ epochs_no_improve = 0
+ else:
+ epochs_no_improve += 1
+ print(f" ⏳ Neither RAW nor EMA improved: {epochs_no_improve}/{cfg.early_stop_patience} "
+ f"(best raw {best_raw_pa:.1f} / ema {best_ema_pa:.1f} mm)")
+ if epochs_no_improve >= cfg.early_stop_patience:
+ print(f"\n🛑 Early stopping at epoch {epoch+1}.")
+ break
+
+ print(f"\n🎉 Training complete! Best PA-MPJPE: {best_overall_pa:.1f} mm")
+
+
+# ============================================================
+# ONNX export — 5 outputs in demo order (MHR slots zero-filled)
+# ============================================================
+class PoseDeployWrapper(nn.Module):
+ """Emits the demo's 5-output contract so the ONNX drops into demo.py.
+ mhr_params/shape_params are zeros (this model has no MHR mesh); the demo
+ only reads them when --mhr-assets is set, which you won't use here."""
+ def __init__(self, model):
+ super().__init__()
+ self.model = model
+
+ def forward(self, image, cliff_cond):
+ out = self.model(image, cliff_cond)
+ cam = out["cam_trans"]
+ B = cam.shape[0]
+ mhr_params = cam.new_zeros(B, 204)
+ shape_params = cam.new_zeros(B, 45)
+ return mhr_params, shape_params, cam, out["joints_2d"], out["joints_3d"]
+
+
+def export_onnx(cfg):
+ import onnx # noqa: F401
+ export_dir = Path(cfg.log_dir) / "export"
+ export_dir.mkdir(parents=True, exist_ok=True)
+ onnx_path = export_dir / "instanthmr_pose3d.onnx"
+ ckpt_path = os.path.join(cfg.log_dir, "best_pose3d.pth")
+ if not os.path.exists(ckpt_path):
+ print("⚠️ No checkpoint to export at", ckpt_path)
+ return
+ model = InstantHMRPoseStudent(cfg, pretrained=False).to(device)
+ ckpt = torch.load(ckpt_path, map_location=device, weights_only=True)
+ state = {k.replace('_orig_mod.', ''): v for k, v in ckpt['model_state_dict'].items()}
+ model.load_state_dict(state, strict=False)
+ deploy = PoseDeployWrapper(model).eval()
+
+ keys = ["mhr_params", "shape_params", "cam_trans", "joints_2d", "joints_3d"]
+ dummy_img = torch.randn(1, 3, cfg.image_size, cfg.image_size, device=device)
+ dummy_cliff = torch.randn(1, 3, device=device)
+ dyn = {"image": {0: "batch"}, "cliff_cond": {0: "batch"}}
+ for k in keys:
+ dyn[k] = {0: "batch"}
+ torch.onnx.export(deploy, (dummy_img, dummy_cliff), str(onnx_path),
+ input_names=["image", "cliff_cond"], output_names=keys,
+ dynamic_axes=dyn, opset_version=17)
+ print(f"✅ Exported {onnx_path} ({onnx_path.stat().st_size/1e6:.1f} MB)")
+ print(f" Run the demo with: python demo.py --camera 0 --model {onnx_path}")
+
+
+# ============================================================
+# Entry point
+# ============================================================
+def parse_args():
+ p = argparse.ArgumentParser(description="Keypoint + camera student (no MHR mesh).")
+ p.add_argument("--data_root", type=str, default=None)
+ p.add_argument("--output_dir", type=str, default=None)
+ p.add_argument("--epochs", type=int, default=None)
+ p.add_argument("--batch_size", type=int, default=None)
+ p.add_argument("--lr", type=float, default=None)
+ p.add_argument("--num_workers", type=int, default=None)
+ p.add_argument("--max_images", type=int, default=None)
+ p.add_argument("--harmony4d_cap", type=int, default=None, help="Max Harmony4D crops (0 = uncapped).")
+ p.add_argument("--overfit-test", dest="overfit_test", action="store_true",
+ help="Run the 1-batch overfit sanity check and exit.")
+ p.add_argument("--no-resume", dest="no_resume", action="store_true")
+ p.add_argument("--no-export", dest="no_export", action="store_true")
+ p.add_argument("--gpu", type=int, default=None)
+ args, unknown = p.parse_known_args()
+ if unknown:
+ print(f"⚠️ Ignoring unrecognized arguments: {unknown}")
+ return args
+
+
+def main():
+ args = parse_args()
+ cfg = PoseConfig()
+ if args.data_root is not None: cfg.data_root = args.data_root
+ if args.output_dir is not None: cfg.log_dir = args.output_dir
+ if args.epochs is not None: cfg.epochs = args.epochs
+ if args.batch_size is not None: cfg.batch_size = args.batch_size
+ if args.lr is not None: cfg.lr = args.lr
+ if args.num_workers is not None: cfg.num_workers = args.num_workers
+ if args.max_images is not None: cfg.max_images = args.max_images
+ if args.harmony4d_cap is not None:
+ if args.harmony4d_cap <= 0:
+ cfg.per_dataset_caps.pop("sam3d_gt_harmony4d", None)
+ else:
+ cfg.per_dataset_caps["sam3d_gt_harmony4d"] = args.harmony4d_cap
+ if args.no_resume: cfg.resume = False
+
+ print("=" * 60)
+ print(f"Data root : {cfg.data_root}")
+ print(f"Output dir : {cfg.log_dir}")
+ print(f"Backbone : {cfg.backbone} | epochs={cfg.epochs} | batch={cfg.batch_size} | lr={cfg.lr}")
+ print(f"Loss : 2D + 3D keypoints + camera (NO MHR mesh)")
+ print("=" * 60)
+
+ train_loader, val_loader, _ = build_dataloaders(cfg)
+
+ if args.overfit_test:
+ run_overfit_test(cfg, train_loader)
+ return
+
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ train_pose3d(cfg, train_loader, val_loader)
+ if not args.no_export:
+ export_onnx(cfg)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/instanthmr_distill_train/train_distill_pose3d_optimized.py b/instanthmr_distill_train/train_distill_pose3d_optimized.py
new file mode 100644
index 0000000..57667ca
--- /dev/null
+++ b/instanthmr_distill_train/train_distill_pose3d_optimized.py
@@ -0,0 +1,861 @@
+#!/usr/bin/env python3
+# ============================================================
+# Keypoint + camera student — OPTIMIZED (geometric aug + soft-argmax).
+# ------------------------------------------------------------
+# Same task as train_distill_pose3d.py (70 2D + 3D keypoints + cam_trans,
+# NO MHR), with two SOTA changes:
+#
+# (1) GEOMETRIC AUGMENTATION — random rotation / scale / translation +
+# horizontal flip, propagated CORRECTLY to the 2D targets, the 3D
+# targets, and cam_trans (verified: image<->2D warp matches to the
+# pixel; rotating 3D XY + cam by R=[[c,s],[-s,c]] reprojects onto the
+# rotated 2D to 1e-7; flip mirrors X and swaps left<->right joints).
+# (2) SOFT-ARGMAX 2D HEAD — the 2D tokens emit 1D bin distributions
+# (SimCC-style) and soft-argmax to coordinates: a light heatmap-style
+# spatial prior at ~the same compute as the old MLP regression.
+#
+# Loss: SmoothL1(2D) + SmoothL1(3D) + SmoothL1(cam).
+# Metrics: MPJPE / PA-MPJPE on the 3D head.
+# Export: 5 ONNX outputs in demo order (mhr_params & shape_params zero-filled).
+#
+# Sanity: python3 train_distill_pose3d_optimized.py --overfit-test --data_root ../data
+# Full: python3 train_distill_pose3d_optimized.py --data_root /datasets/instanthmr_data --num_workers 8
+# ============================================================
+import os
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import argparse
+import math
+import random
+import gc
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+import cv2
+from PIL import Image
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torch.optim.lr_scheduler as lr_scheduler
+from torch.utils.data import Dataset, DataLoader, random_split
+from torch.optim.swa_utils import AveragedModel
+
+import torchvision.transforms as transforms
+import torchvision.transforms.functional as F_t
+from torchvision.transforms import InterpolationMode
+
+import timm
+from tqdm import tqdm
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+print(f"Using device: {device}")
+
+PROJECT_ROOT = Path(__file__).resolve().parent
+DEFAULT_DATA_ROOT = "/pfcalcul/datasets/instanthmr_data/"
+
+# MHR70 joint ordering (mirror of instanthmr/skeleton.py, embedded so this
+# script is self-contained on the cluster). Used to build the horizontal-flip
+# left<->right permutation by name.
+JOINT_NAMES = [
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
+ "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle",
+ "left_big_toe_tip", "left_small_toe_tip", "left_heel",
+ "right_big_toe_tip", "right_small_toe_tip", "right_heel",
+ "right_thumb_tip", "right_thumb_first_joint", "right_thumb_second_joint", "right_thumb_third_joint",
+ "right_index_tip", "right_index_first_joint", "right_index_second_joint", "right_index_third_joint",
+ "right_middle_tip", "right_middle_first_joint", "right_middle_second_joint", "right_middle_third_joint",
+ "right_ring_tip", "right_ring_first_joint", "right_ring_second_joint", "right_ring_third_joint",
+ "right_pinky_tip", "right_pinky_first_joint", "right_pinky_second_joint", "right_pinky_third_joint",
+ "right_wrist",
+ "left_thumb_tip", "left_thumb_first_joint", "left_thumb_second_joint", "left_thumb_third_joint",
+ "left_index_tip", "left_index_first_joint", "left_index_second_joint", "left_index_third_joint",
+ "left_middle_tip", "left_middle_first_joint", "left_middle_second_joint", "left_middle_third_joint",
+ "left_ring_tip", "left_ring_first_joint", "left_ring_second_joint", "left_ring_third_joint",
+ "left_pinky_tip", "left_pinky_first_joint", "left_pinky_second_joint", "left_pinky_third_joint",
+ "left_wrist",
+ "left_olecranon", "right_olecranon", "left_cubital_fossa", "right_cubital_fossa",
+ "left_acromion", "right_acromion", "neck",
+]
+
+
+def _build_flip_perm(names):
+ """Left<->right joint index permutation for horizontal flip (central joints fixed)."""
+ idx = {n: i for i, n in enumerate(names)}
+ perm = list(range(len(names)))
+ for i, n in enumerate(names):
+ if n.startswith("left_"):
+ perm[i] = idx.get("right_" + n[5:], i)
+ elif n.startswith("right_"):
+ perm[i] = idx.get("left_" + n[6:], i)
+ perm = np.array(perm, dtype=np.int64)
+ assert (perm[perm] == np.arange(len(perm))).all(), "flip permutation is not an involution"
+ return perm
+
+
+FLIP_PERM = _build_flip_perm(JOINT_NAMES)
+
+
+# ============================================================
+# Configuration
+# ============================================================
+@dataclass
+class PoseConfig:
+ data_root: str = str(DEFAULT_DATA_ROOT)
+ log_dir: str = str(PROJECT_ROOT / "runs/distill_pose3d_v1")
+
+ # --- Data ---
+ image_size: int = 224
+ max_images: int | None = None
+ per_dataset_caps: dict = field(default_factory=lambda: {"sam3d_gt_harmony4d": 300_000})
+ val_split: float = 0.1
+ num_workers: int = 4
+ augment: bool = True
+
+ # --- Geometric augmentation (change #1) ---
+ geom_rot_deg: float = 30.0 # random in-plane rotation ±deg
+ geom_scale_range: float = 0.25 # random scale in [1-r, 1+r]
+ geom_trans: float = 0.08 # random translation, fraction of half-crop
+ geom_flip_p: float = 0.5 # horizontal flip probability (with L/R joint swap)
+
+ # --- Model ---
+ backbone: str = "repvit_m2_3"
+ d_model: int = 512
+ n_heads: int = 8
+ n_decoder_layers: int = 4
+ dropout: float = 0.1
+ cliff_dim: int = 3
+ cam_dim: int = 3
+ num_joints: int = 70
+ kp2d_bins: int = 64 # soft-argmax 2D head bins per axis (change #2)
+
+ # --- Training ---
+ batch_size: int = 64
+ lr: float = 3e-4
+ weight_decay: float = 1e-4
+ epochs: int = 150
+ grad_clip: float = 1.0
+ anomaly_loss_threshold: float = 100.0
+ use_amp: bool = True
+ ema_decay: float = 0.9998
+ early_stop_patience: int = 150
+ resume: bool = True
+
+ # --- Loss weights ---
+ w_keypoints2d: float = 10.0
+ w_keypoints3d: float = 10.0
+ w_cam: float = 0.1 # direct SmoothL1 supervision on cam_trans
+ w_simcc: float = 1.0 # SimCC bin-classification weight (has an entropy floor)
+
+
+# ============================================================
+# Dataset (same crops as train_distill.py; keeps cam_trans + camera intrinsics)
+# ============================================================
+class RandomPixelate:
+ def __init__(self, p=0.2, min_res=32, max_res=112):
+ self.p, self.min_res, self.max_res = p, min_res, max_res
+
+ def __call__(self, img):
+ if random.random() < self.p:
+ orig_w, orig_h = img.size
+ degraded_h = random.randint(self.min_res, self.max_res)
+ degraded_w = int(orig_w * (degraded_h / orig_h))
+ img = F_t.resize(img, [degraded_h, degraded_w], interpolation=InterpolationMode.NEAREST)
+ img = F_t.resize(img, [orig_h, orig_w], interpolation=InterpolationMode.NEAREST)
+ return img
+
+
+class SAM3DStudentDataset(Dataset):
+ def __init__(self, data_root: str, image_size: int = 224, max_images: int | None = None,
+ augment: bool = True, per_dataset_caps: dict | None = None,
+ geom_rot_deg: float = 30.0, geom_scale_range: float = 0.25,
+ geom_trans: float = 0.08, geom_flip_p: float = 0.5):
+ super().__init__()
+ self.image_size = image_size
+ self.data_root = Path(data_root)
+ # geometric augmentation only on the train split (augment=True)
+ self.augment = augment
+ self.geom_rot_deg = geom_rot_deg
+ self.geom_scale_range = geom_scale_range
+ self.geom_trans = geom_trans
+ self.geom_flip_p = geom_flip_p
+
+ # `self.transform` is now PHOTOMETRIC only — geometric aug happens in
+ # __getitem__ jointly on the image + 2D/3D/cam targets (before ToTensor).
+ tfm_list = [transforms.Resize((image_size, image_size))]
+ if augment:
+ tfm_list.extend([
+ transforms.RandomApply([transforms.ColorJitter(
+ brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)], p=0.5),
+ RandomPixelate(p=0.2, min_res=48, max_res=112),
+ transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2),
+ ])
+ tfm_list.extend([
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+ ])
+ if augment:
+ tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))
+ self.transform = transforms.Compose(tfm_list)
+
+ self.pairs = []
+ dataset_dirs = self._find_dataset_dirs(self.data_root)
+ if not dataset_dirs:
+ raise FileNotFoundError(f"No annotations/images sub-folders under '{self.data_root}'.")
+
+ cap_rng = random.Random(42)
+ caps = per_dataset_caps or {}
+ for ann_dir, img_dir in dataset_dirs:
+ sub_name = ann_dir.parent.name
+ sub_pairs = []
+ for npz_path in sorted(ann_dir.glob("*.npz")):
+ stem = npz_path.stem
+ img_path = img_dir / f"{stem}.jpg"
+ if not img_path.exists():
+ img_path = img_dir / f"{stem}.png"
+ if img_path.exists():
+ sub_pairs.append((img_path, npz_path))
+ cap = caps.get(sub_name)
+ if cap is not None and len(sub_pairs) > cap:
+ n_before = len(sub_pairs)
+ sub_pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
+ sub_pairs = cap_rng.sample(sub_pairs, cap)
+ print(f" [cap] {sub_name}: sampled {cap:,} of {n_before:,} crops")
+ self.pairs.extend(sub_pairs)
+
+ self.pairs.sort(key=lambda p: (str(p[1]), str(p[0])))
+ if max_images is not None:
+ self.pairs = self.pairs[:max_images]
+ print(f" Found {len(dataset_dirs)} sub-folder(s) under {self.data_root}; "
+ f"{len(self.pairs)} valid (image, npz) pairs (augment={augment}).")
+
+ @staticmethod
+ def _find_dataset_dirs(root: Path):
+ dirs = []
+ if (root / "annotations").is_dir() and (root / "images").is_dir():
+ dirs.append((root / "annotations", root / "images"))
+ if root.is_dir():
+ for sub in sorted(root.iterdir()):
+ if not sub.is_dir():
+ continue
+ ann, img = sub / "annotations", sub / "images"
+ if ann.is_dir() and img.is_dir():
+ dirs.append((ann, img))
+ return dirs
+
+ def __len__(self):
+ return len(self.pairs)
+
+ def _geometric_aug(self, img_np, j2, j3, cam):
+ """Jointly warp image + 2D/3D/cam. Verified: image<->2D warp matches to the
+ pixel; 3D XY & cam rotate by R=[[c,s],[-s,c]] consistently with the 2D."""
+ W = H = self.image_size
+
+ # --- horizontal flip: mirror X + swap left/right joints ---
+ if random.random() < self.geom_flip_p:
+ img_np = np.ascontiguousarray(img_np[:, ::-1, :])
+ j2[:, 0] = -j2[:, 0]
+ j3[:, 0] = -j3[:, 0]
+ cam[0] = -cam[0]
+ j2 = j2[FLIP_PERM]
+ j3 = j3[FLIP_PERM]
+
+ # --- rotation + scale + translation (single cv2 affine) ---
+ angle = random.uniform(-self.geom_rot_deg, self.geom_rot_deg)
+ scale = random.uniform(1.0 - self.geom_scale_range, 1.0 + self.geom_scale_range)
+ dx = random.uniform(-self.geom_trans, self.geom_trans)
+ dy = random.uniform(-self.geom_trans, self.geom_trans)
+
+ M = cv2.getRotationMatrix2D((W / 2.0, H / 2.0), angle, scale)
+ M[0, 2] += dx * (W / 2.0)
+ M[1, 2] += dy * (H / 2.0)
+ img_np = cv2.warpAffine(img_np, M, (W, H), flags=cv2.INTER_LINEAR,
+ borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
+
+ # 2D: apply the SAME affine M in pixel space, back to normalised [-1,1]
+ px = (j2[:, 0] + 1.0) * 0.5 * W
+ py = (j2[:, 1] + 1.0) * 0.5 * H
+ px2 = M[0, 0] * px + M[0, 1] * py + M[0, 2]
+ py2 = M[1, 0] * px + M[1, 1] * py + M[1, 2]
+ j2[:, 0] = px2 / (0.5 * W) - 1.0
+ j2[:, 1] = py2 / (0.5 * H) - 1.0
+
+ # 3D: rotate the XY plane about the optical axis (scale/translate don't touch 3D)
+ rad = math.radians(angle)
+ c, s = math.cos(rad), math.sin(rad)
+ R = np.array([[c, s], [-s, c]], dtype=np.float32) # matches cv2 rotation (scale=1)
+ j3[:, :2] = j3[:, :2] @ R.T
+ cam[:2] = R @ cam[:2]
+ return img_np, j2, j3, cam
+
+ def __getitem__(self, idx):
+ img_path, npz_path = self.pairs[idx]
+ img_pil = Image.open(img_path).convert("RGB")
+
+ ann = np.load(npz_path)
+ orig_h, orig_w = ann["orig_shape"]
+
+ sq_bbox = ann["bbox_square"]
+ sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]
+ orig_crop_size = sq_bbox[2] - sq_bbox[0]
+ true_scale = self.image_size / max(orig_crop_size, 1.0)
+
+ joints_2d = ann["joints_2d"].copy().astype(np.float32)
+ joints_2d[:, 0] = (joints_2d[:, 0] - sq_x1) * true_scale
+ joints_2d[:, 1] = (joints_2d[:, 1] - sq_y1) * true_scale
+ joints_2d[:, 0] = (joints_2d[:, 0] / self.image_size) * 2.0 - 1.0
+ joints_2d[:, 1] = (joints_2d[:, 1] / self.image_size) * 2.0 - 1.0
+
+ joints_3d = ann["joints_3d"].astype(np.float32).copy()
+ cam_trans = ann["cam_trans"].astype(np.float32).copy()
+
+ # --- geometric augmentation (train only), jointly on image + targets ---
+ if self.augment:
+ img_np = np.asarray(img_pil.resize((self.image_size, self.image_size)))
+ img_np, joints_2d, joints_3d, cam_trans = self._geometric_aug(img_np, joints_2d, joints_3d, cam_trans)
+ img_pil = Image.fromarray(img_np)
+
+ image = self.transform(img_pil) # photometric + ToTensor + Normalize
+
+ tight_bbox = ann["bbox"]
+ cx = (tight_bbox[0] + tight_bbox[2]) / 2.0
+ cy = (tight_bbox[1] + tight_bbox[3]) / 2.0
+ cx_norm = 2.0 * (cx / orig_w) - 1.0
+ cy_norm = 2.0 * (cy / orig_h) - 1.0
+ b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])
+ b_scale = b_size / max(orig_w, orig_h)
+ cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)
+
+ return {
+ "image": image, # [3,224,224]
+ "cliff_cond": cliff_cond, # [3]
+ "joints_2d": torch.from_numpy(joints_2d).float(), # [70,2] crop [-1,1]
+ "joints_3d": torch.from_numpy(joints_3d).float(), # [70,3] root-centred
+ "cam_trans": torch.from_numpy(cam_trans).float(), # [3] camera translation
+ }
+
+
+def build_dataloaders(cfg):
+ geom = dict(geom_rot_deg=cfg.geom_rot_deg, geom_scale_range=cfg.geom_scale_range,
+ geom_trans=cfg.geom_trans, geom_flip_p=cfg.geom_flip_p)
+ full_dataset = SAM3DStudentDataset(cfg.data_root, augment=cfg.augment,
+ max_images=cfg.max_images, per_dataset_caps=cfg.per_dataset_caps, **geom)
+ val_dataset_clean = SAM3DStudentDataset(cfg.data_root, augment=False,
+ max_images=cfg.max_images, per_dataset_caps=cfg.per_dataset_caps, **geom)
+ n_val = int(len(full_dataset) * cfg.val_split)
+ n_train = len(full_dataset) - n_val
+ g = torch.Generator().manual_seed(42)
+ train_dataset, _ = random_split(full_dataset, [n_train, n_val], generator=g)
+ g = torch.Generator().manual_seed(42)
+ _, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=g)
+
+ loader_kwargs = dict(num_workers=cfg.num_workers, pin_memory=True)
+ if cfg.num_workers > 0:
+ loader_kwargs["persistent_workers"] = True
+ loader_kwargs["prefetch_factor"] = 4
+ train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
+ drop_last=True, **loader_kwargs)
+ val_loader = DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False, **loader_kwargs)
+ print(f"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}")
+ return train_loader, val_loader, full_dataset
+
+
+# ============================================================
+# Model — RepViT + decoder + cam head + 2D/3D heads (no MHR head)
+# ============================================================
+def get_2d_sincos_pos_embed(embed_dim, grid_size):
+ grid_h, grid_w = grid_size, grid_size
+ grid_y, grid_x = torch.meshgrid(torch.arange(grid_h), torch.arange(grid_w), indexing='ij')
+ omega = torch.arange(embed_dim // 4).float() / (embed_dim // 4)
+ omega = 1.0 / (10000 ** omega)
+ out_y = grid_y.flatten().unsqueeze(1) * omega.unsqueeze(0)
+ out_x = grid_x.flatten().unsqueeze(1) * omega.unsqueeze(0)
+ pe_y = torch.cat([torch.sin(out_y), torch.cos(out_y)], dim=1)
+ pe_x = torch.cat([torch.sin(out_x), torch.cos(out_x)], dim=1)
+ return torch.cat([pe_y, pe_x], dim=1).unsqueeze(0)
+
+
+class SoftArgmax2DHead(nn.Module):
+ """SimCC-style 2D head: each joint token emits 1D bin distributions for x and y,
+ then soft-argmax to a continuous [-1,1] coordinate. A light heatmap-style spatial
+ prior at ~the same compute as an MLP regressor. Fully ONNX-exportable."""
+ def __init__(self, d_in, num_bins=64):
+ super().__init__()
+ self.num_bins = num_bins
+ self.proj = nn.Sequential(nn.Linear(d_in, 256), nn.GELU())
+ self.to_logits = nn.Linear(256, 2 * num_bins)
+ self.register_buffer("bin_centers", torch.linspace(-1.0, 1.0, num_bins))
+
+ def forward(self, feat): # feat: [B, J, d_in]
+ logits = self.to_logits(self.proj(feat)) # [B, J, 2*bins]
+ logits = logits.unflatten(-1, (2, self.num_bins)) # [B, J, 2, bins]
+ prob = torch.softmax(logits, dim=-1)
+ coords = (prob * self.bin_centers).sum(dim=-1) # [B, J, 2] in [-1,1]
+ return coords, logits # logits used by the SimCC loss
+
+
+class InstantHMRPoseStudent(nn.Module):
+ def __init__(self, cfg, pretrained=True):
+ super().__init__()
+ self.cfg = cfg
+ self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)
+ embed_dim = self.backbone.num_features
+ self.feat_proj = nn.Linear(embed_dim, cfg.d_model)
+
+ self.grid_size = cfg.image_size // 32
+ self.register_buffer("mem_pos_embed", get_2d_sincos_pos_embed(cfg.d_model, self.grid_size))
+
+ # 1 global (camera) + 70 2D + 70 3D query tokens
+ self.num_global = 1
+ self.num_2d = cfg.num_joints
+ self.num_3d = cfg.num_joints
+ self.total_queries = self.num_global + self.num_2d + self.num_3d
+ self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model) * 0.02)
+
+ self.cond_proj = nn.Sequential(
+ nn.Linear(cfg.cliff_dim, 128), nn.GELU(), nn.Linear(128, cfg.d_model))
+
+ decoder_layer = nn.TransformerDecoderLayer(
+ d_model=cfg.d_model, nhead=cfg.n_heads, dim_feedforward=cfg.d_model * 4,
+ dropout=cfg.dropout, activation="gelu", batch_first=True, norm_first=True)
+ self.transformer = nn.TransformerDecoder(decoder_layer, num_layers=cfg.n_decoder_layers)
+
+ # Camera head (translation only) + soft-argmax 2D head + 3D head
+ self.head_cam = nn.Linear(cfg.d_model, cfg.cam_dim)
+ self.head_2d = SoftArgmax2DHead(cfg.d_model, cfg.kp2d_bins)
+ self.head_3d = nn.Sequential(nn.Linear(cfg.d_model, 256), nn.GELU(), nn.Linear(256, 3))
+
+ # Init camera near (0, 0, 2m) depth; tiny init on the 3D head for stability
+ nn.init.normal_(self.head_cam.weight, mean=0.0, std=1e-4)
+ nn.init.constant_(self.head_cam.bias, 0.0)
+ nn.init.constant_(self.head_cam.bias[-1], 2.0)
+ nn.init.normal_(self.head_3d[-1].weight, mean=0.0, std=1e-4)
+ nn.init.constant_(self.head_3d[-1].bias, 0.0)
+
+ def forward(self, images, cliff_cond):
+ B = images.shape[0]
+ img_feats = self.backbone.forward_features(images)
+ if img_feats.dim() == 4:
+ img_feats = img_feats.flatten(2).transpose(1, 2)
+ memory = self.feat_proj(img_feats) + self.mem_pos_embed
+
+ queries = self.query_embed.expand(B, -1, -1) + self.cond_proj(cliff_cond).unsqueeze(1)
+ shared = self.transformer(tgt=queries, memory=memory)
+
+ feat_global = shared[:, 0, :]
+ feat_2d = shared[:, 1:1 + self.num_2d, :]
+ feat_3d = shared[:, 1 + self.num_2d:, :]
+
+ pred_cam = self.head_cam(feat_global)
+ pred_2d, logits_2d = self.head_2d(feat_2d)
+ pred_3d = self.head_3d(feat_3d)
+ return {"cam_trans": pred_cam, "joints_2d": pred_2d,
+ "joints_2d_logits": logits_2d, "joints_3d": pred_3d}
+
+
+# ============================================================
+# Loss — keypoints + camera (no MHR)
+# ============================================================
+class Pose3DLoss(nn.Module):
+ def __init__(self, cfg):
+ super().__init__()
+ self.cfg = cfg
+ self.l1 = nn.SmoothL1Loss()
+ bins = torch.linspace(-1.0, 1.0, cfg.kp2d_bins)
+ self.register_buffer("bin_centers", bins)
+ self.simcc_sigma = 2.0 * (2.0 / (cfg.kp2d_bins - 1)) # ~2 bins wide Gaussian label
+
+ def _simcc_ce(self, logits, tgt_2d):
+ """SimCC classification loss: CE between the predicted bin distribution and a
+ Gaussian soft-label centred on the target coordinate (sharpens the soft-argmax)."""
+ # logits: [B,J,2,bins] tgt_2d: [B,J,2] in [-1,1]
+ d = self.bin_centers.view(1, 1, 1, -1) - tgt_2d.unsqueeze(-1) # [B,J,2,bins]
+ label = torch.exp(-(d ** 2) / (2.0 * self.simcc_sigma ** 2))
+ label = label / label.sum(dim=-1, keepdim=True).clamp_min(1e-8)
+ logp = F.log_softmax(logits, dim=-1)
+ return -(label * logp).sum(dim=-1).mean()
+
+ def forward(self, preds, targets):
+ losses = {}
+ # 2D: soft-argmax coordinate SmoothL1 (accurate read-out) + SimCC classification (sharpen)
+ losses['loss_2d'] = self.l1(preds["joints_2d"][..., :2], targets["joints_2d"]) * self.cfg.w_keypoints2d
+ losses['loss_2d_simcc'] = self._simcc_ce(preds["joints_2d_logits"], targets["joints_2d"]) * self.cfg.w_simcc
+ losses['loss_3d'] = self.l1(preds["joints_3d"][..., :3], targets["joints_3d"]) * self.cfg.w_keypoints3d
+ # Direct camera supervision — SmoothL1 (bounded gradient, robust to large depths).
+ losses['loss_cam'] = self.l1(preds["cam_trans"], targets["cam_trans"]) * self.cfg.w_cam
+ losses['total_loss'] = (losses['loss_2d'] + losses['loss_2d_simcc']
+ + losses['loss_3d'] + losses['loss_cam'])
+ return losses
+
+
+# ============================================================
+# Metrics — MPJPE / PA-MPJPE on the 3D head (mm)
+# ============================================================
+def batched_procrustes_alignment(pred_pts, gt_pts):
+ mu_pred = pred_pts.mean(dim=1, keepdim=True)
+ mu_gt = gt_pts.mean(dim=1, keepdim=True)
+ pred_c, gt_c = pred_pts - mu_pred, gt_pts - mu_gt
+ norm_pred = torch.linalg.norm(pred_c, dim=(1, 2), keepdim=True)
+ norm_gt = torch.linalg.norm(gt_c, dim=(1, 2), keepdim=True)
+ pred_n = pred_c / torch.clamp(norm_pred, min=1e-8)
+ gt_n = gt_c / torch.clamp(norm_gt, min=1e-8)
+ H = torch.bmm(pred_n.transpose(1, 2), gt_n)
+ U, S, Vh = torch.linalg.svd(H)
+ R = torch.bmm(U, Vh)
+ det = torch.linalg.det(R)
+ det_sign = torch.where(det < 0, torch.tensor(-1.0, device=pred_pts.device),
+ torch.tensor(1.0, device=pred_pts.device)).unsqueeze(-1)
+ U_fixed = U.clone()
+ U_fixed[:, :, 2] *= det_sign
+ R_fixed = torch.bmm(U_fixed, Vh)
+ S_fixed = S.clone()
+ S_fixed[:, 2] *= det_sign.squeeze(-1)
+ scale = S_fixed.sum(dim=-1, keepdim=True).unsqueeze(-1) * (norm_gt / torch.clamp(norm_pred, min=1e-8))
+ return scale * torch.bmm(pred_c, R_fixed) + mu_gt
+
+
+def evaluate_pose_batch(preds, targets):
+ metrics = {}
+ pred_3d = preds["joints_3d"][..., :3]
+ gt_3d = targets["joints_3d"][..., :3]
+ pred_rr = pred_3d - pred_3d[:, 0:1, :]
+ gt_rr = gt_3d - gt_3d[:, 0:1, :]
+ metrics['MPJPE'] = torch.linalg.norm(pred_rr - gt_rr, dim=-1).mean().item() * 1000.0
+ pred_al = batched_procrustes_alignment(pred_3d, gt_3d)
+ metrics['PA_MPJPE'] = torch.linalg.norm(pred_al - gt_3d, dim=-1).mean().item() * 1000.0
+ return metrics
+
+
+# ============================================================
+# 1-batch overfit test — proves gradients flow / model can learn
+# ============================================================
+def run_overfit_test(cfg, train_loader, steps=800, subset=8, lr=1e-3):
+ print(f"--- 1-Batch Overfit Test ({subset} images, {steps} steps) ---")
+ model = InstantHMRPoseStudent(cfg, pretrained=True).to(device)
+ criterion = Pose3DLoss(cfg).to(device)
+ optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.0)
+ scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
+
+ batch = next(iter(train_loader))
+ batch = {k: (v[:subset].to(device) if isinstance(v, torch.Tensor) else v) for k, v in batch.items()}
+
+ # Judge overfit on the REGRESSION losses (2D coord + 3D + cam), which reach ~0.
+ # The SimCC cross-entropy has an irreducible entropy floor, so the *total* can't.
+ def reg_loss(ls):
+ return ls['loss_2d'].item() + ls['loss_3d'].item() + ls['loss_cam'].item()
+
+ first_reg, last_reg, last_pa = None, None, None
+ model.train()
+ for step in range(steps):
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds = model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ loss = losses['total_loss']
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
+ scaler.step(optimizer)
+ scaler.update()
+ if first_reg is None:
+ first_reg = reg_loss(losses)
+ last_reg = reg_loss(losses)
+ if step % 50 == 0 or step == steps - 1:
+ model.eval()
+ with torch.no_grad():
+ p = {k: v.float() for k, v in model(batch["image"], batch["cliff_cond"]).items()}
+ m = evaluate_pose_batch(p, batch)
+ model.train()
+ last_pa = m['PA_MPJPE']
+ print(f" step {step:04d} | total {loss.item():.4f} | 2d {losses['loss_2d'].item():.4f} "
+ f"| simcc {losses['loss_2d_simcc'].item():.3f} | 3d {losses['loss_3d'].item():.4f} "
+ f"| cam {losses['loss_cam'].item():.4f} | MPJPE {m['MPJPE']:.1f} mm | PA {m['PA_MPJPE']:.1f} mm")
+
+ ok = (last_reg < first_reg * 0.05) and (last_pa is not None and last_pa < 40.0)
+ print(f"\n{'✅ SUCCESS' if ok else '❌ FAIL'}: regression loss {first_reg:.4f} -> {last_reg:.5f}"
+ f" | final PA-MPJPE {last_pa:.1f} mm "
+ f"({'overfit, gradients flow' if ok else 'did NOT fit — check the setup'})")
+ return ok
+
+
+# ============================================================
+# EMA helper (decay warmup)
+# ============================================================
+def make_ema_avg_fn(max_decay):
+ @torch.no_grad()
+ def avg_fn(ema_params, model_params, num_averaged):
+ if not (torch.is_floating_point(ema_params[0]) or torch.is_complex(ema_params[0])):
+ for e, m in zip(ema_params, model_params):
+ e.copy_(m)
+ return
+ n = num_averaged.item() if torch.is_tensor(num_averaged) else float(num_averaged)
+ decay = min(max_decay, (1.0 + n) / (10.0 + n))
+ torch._foreach_lerp_(ema_params, model_params, 1.0 - decay)
+ return avg_fn
+
+
+# ============================================================
+# Training loop — EMA + early stopping + guards, keypoints + camera
+# ============================================================
+def train_pose3d(cfg, train_loader, val_loader):
+ print(f"--- Training {cfg.backbone} (pose3d) | EMA={cfg.ema_decay} | patience={cfg.early_stop_patience} ---")
+ os.makedirs(cfg.log_dir, exist_ok=True)
+
+ model = InstantHMRPoseStudent(cfg, pretrained=True).to(device)
+ criterion = Pose3DLoss(cfg).to(device)
+ optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
+ scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)
+ steps_per_epoch = len(train_loader)
+ scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=cfg.lr, epochs=cfg.epochs,
+ steps_per_epoch=steps_per_epoch, pct_start=0.1)
+
+ best_raw_path = os.path.join(cfg.log_dir, "best_pose3d_raw.pth")
+ best_ema_path = os.path.join(cfg.log_dir, "best_pose3d_ema.pth")
+ best_ckpt_path = os.path.join(cfg.log_dir, "best_pose3d.pth")
+
+ start_epoch = 0
+ best_raw_pa = best_ema_pa = best_overall_pa = float('inf')
+ epochs_no_improve = 0
+
+ if cfg.resume and os.path.exists(best_ckpt_path):
+ print(f"🔄 Resuming from {best_ckpt_path}")
+ ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=False)
+ model.load_state_dict(ckpt["model_state_dict"], strict=False)
+ for k, obj in [("optimizer_state_dict", optimizer), ("scaler_state_dict", scaler),
+ ("scheduler_state_dict", scheduler)]:
+ if k in ckpt:
+ obj.load_state_dict(ckpt[k])
+ start_epoch = ckpt.get("epoch", -1) + 1
+ best_overall_pa = best_raw_pa = best_ema_pa = ckpt.get("val_pa_mpjpe", float('inf'))
+ print(f"✅ Resumed at epoch {start_epoch}. Best PA-MPJPE {best_overall_pa:.1f} mm.")
+ else:
+ print("🚀 Training from scratch.")
+
+ ema_model = AveragedModel(model, multi_avg_fn=make_ema_avg_fn(cfg.ema_decay), use_buffers=True)
+
+ @torch.no_grad()
+ def run_validation(eval_model, tag, epoch):
+ eval_model.eval()
+ agg = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'cam': 0.0, 'MPJPE': 0.0, 'PA_MPJPE': 0.0}
+ n = 0
+ vbar = tqdm(val_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]", leave=False)
+ for batch in vbar:
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds = eval_model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds.items()}
+ losses = criterion(preds, batch)
+ if math.isnan(losses['total_loss'].item()) or math.isinf(losses['total_loss'].item()):
+ continue
+ agg['total'] += losses['total_loss'].item()
+ agg['2d'] += losses['loss_2d'].item()
+ agg['3d'] += losses['loss_3d'].item()
+ agg['cam'] += losses['loss_cam'].item()
+ bm = evaluate_pose_batch(preds, batch)
+ agg['MPJPE'] += bm['MPJPE']
+ agg['PA_MPJPE'] += bm['PA_MPJPE']
+ n += 1
+ vbar.set_postfix({'PA-MPJPE': f"{agg['PA_MPJPE']/max(n,1):.1f}mm"})
+ vbar.close()
+ return {k: v / max(n, 1) for k, v in agg.items()}, n
+
+ for epoch in range(start_epoch, cfg.epochs):
+ model.train()
+ tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'cam': 0.0}
+ nb = 0
+ pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Train]")
+ for batch in pbar:
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):
+ preds_fp16 = model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds_fp16.items()}
+ losses = criterion(preds, batch)
+ loss = losses['total_loss']
+ loss_val = loss.item()
+ if math.isnan(loss_val) or math.isinf(loss_val):
+ print("⚠️ WARNING: NaN/Inf loss detected! Skipping step.")
+ continue
+ if loss_val > cfg.anomaly_loss_threshold:
+ print(f"⚠️ WARNING: anomalous loss {loss_val:.1f} > {cfg.anomaly_loss_threshold} — skipping step.")
+ continue
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
+ scale_before = scaler.get_scale()
+ scaler.step(optimizer)
+ scaler.update()
+ if scale_before <= scaler.get_scale():
+ scheduler.step()
+ ema_model.update_parameters(model)
+ tm['total'] += loss_val
+ tm['2d'] += losses['loss_2d'].item()
+ tm['3d'] += losses['loss_3d'].item()
+ tm['cam'] += losses['loss_cam'].item()
+ nb += 1
+ pbar.set_postfix({'Tot': f"{loss_val:.4f}"})
+ pbar.close()
+ avg_train = {k: v / max(nb, 1) for k, v in tm.items()}
+
+ raw_val, n_raw = run_validation(model, "raw", epoch)
+ ema_val, n_ema = run_validation(ema_model, "ema", epoch)
+
+ print(f"\n📈 Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}")
+ print(f" [Train] Tot: {avg_train['total']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f} | cam: {avg_train['cam']:.4f}")
+ print(f" [Val RAW] Tot: {raw_val['total']:.4f} | PA-MPJPE: {raw_val['PA_MPJPE']:.1f} | MPJPE: {raw_val['MPJPE']:.1f} mm")
+ print(f" [Val EMA] Tot: {ema_val['total']:.4f} | PA-MPJPE: {ema_val['PA_MPJPE']:.1f} | MPJPE: {ema_val['MPJPE']:.1f} mm")
+
+ def save_dict(m, val_metric, source):
+ return {'epoch': epoch, 'model_state_dict': m.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ 'scaler_state_dict': scaler.state_dict(),
+ 'scheduler_state_dict': scheduler.state_dict(),
+ 'val_pa_mpjpe': val_metric, 'source': source}
+
+ improved_raw = n_raw > 0 and raw_val['PA_MPJPE'] < best_raw_pa
+ improved_ema = n_ema > 0 and ema_val['PA_MPJPE'] < best_ema_pa
+ if improved_raw:
+ best_raw_pa = raw_val['PA_MPJPE']
+ torch.save(save_dict(model, best_raw_pa, 'raw'), best_raw_path)
+ if improved_ema:
+ best_ema_pa = ema_val['PA_MPJPE']
+ torch.save(save_dict(ema_model.module, best_ema_pa, 'ema'), best_ema_path)
+
+ current = min(raw_val['PA_MPJPE'], ema_val['PA_MPJPE'])
+ if current < best_overall_pa - 1e-4:
+ best_overall_pa = current
+ use_ema = ema_val['PA_MPJPE'] <= raw_val['PA_MPJPE']
+ better = ema_model.module if use_ema else model
+ torch.save(save_dict(better, best_overall_pa, 'ema' if use_ema else 'raw'), best_ckpt_path)
+ print(f"💾 New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}")
+
+ if improved_raw or improved_ema:
+ epochs_no_improve = 0
+ else:
+ epochs_no_improve += 1
+ print(f" ⏳ Neither RAW nor EMA improved: {epochs_no_improve}/{cfg.early_stop_patience} "
+ f"(best raw {best_raw_pa:.1f} / ema {best_ema_pa:.1f} mm)")
+ if epochs_no_improve >= cfg.early_stop_patience:
+ print(f"\n🛑 Early stopping at epoch {epoch+1}.")
+ break
+
+ print(f"\n🎉 Training complete! Best PA-MPJPE: {best_overall_pa:.1f} mm")
+
+
+# ============================================================
+# ONNX export — 5 outputs in demo order (MHR slots zero-filled)
+# ============================================================
+class PoseDeployWrapper(nn.Module):
+ """Emits the demo's 5-output contract so the ONNX drops into demo.py.
+ mhr_params/shape_params are zeros (this model has no MHR mesh); the demo
+ only reads them when --mhr-assets is set, which you won't use here."""
+ def __init__(self, model):
+ super().__init__()
+ self.model = model
+
+ def forward(self, image, cliff_cond):
+ out = self.model(image, cliff_cond)
+ cam = out["cam_trans"]
+ B = cam.shape[0]
+ mhr_params = cam.new_zeros(B, 204)
+ shape_params = cam.new_zeros(B, 45)
+ return mhr_params, shape_params, cam, out["joints_2d"], out["joints_3d"]
+
+
+def export_onnx(cfg):
+ import onnx # noqa: F401
+ export_dir = Path(cfg.log_dir) / "export"
+ export_dir.mkdir(parents=True, exist_ok=True)
+ onnx_path = export_dir / "instanthmr_pose3d.onnx"
+ ckpt_path = os.path.join(cfg.log_dir, "best_pose3d.pth")
+ if not os.path.exists(ckpt_path):
+ print("⚠️ No checkpoint to export at", ckpt_path)
+ return
+ model = InstantHMRPoseStudent(cfg, pretrained=False).to(device)
+ ckpt = torch.load(ckpt_path, map_location=device, weights_only=True)
+ state = {k.replace('_orig_mod.', ''): v for k, v in ckpt['model_state_dict'].items()}
+ model.load_state_dict(state, strict=False)
+ deploy = PoseDeployWrapper(model).eval()
+
+ keys = ["mhr_params", "shape_params", "cam_trans", "joints_2d", "joints_3d"]
+ dummy_img = torch.randn(1, 3, cfg.image_size, cfg.image_size, device=device)
+ dummy_cliff = torch.randn(1, 3, device=device)
+ dyn = {"image": {0: "batch"}, "cliff_cond": {0: "batch"}}
+ for k in keys:
+ dyn[k] = {0: "batch"}
+ torch.onnx.export(deploy, (dummy_img, dummy_cliff), str(onnx_path),
+ input_names=["image", "cliff_cond"], output_names=keys,
+ dynamic_axes=dyn, opset_version=17)
+ print(f"✅ Exported {onnx_path} ({onnx_path.stat().st_size/1e6:.1f} MB)")
+ print(f" Run the demo with: python demo.py --camera 0 --model {onnx_path}")
+
+
+# ============================================================
+# Entry point
+# ============================================================
+def parse_args():
+ p = argparse.ArgumentParser(description="Keypoint + camera student (no MHR mesh).")
+ p.add_argument("--data_root", type=str, default=None)
+ p.add_argument("--output_dir", type=str, default=None)
+ p.add_argument("--epochs", type=int, default=None)
+ p.add_argument("--batch_size", type=int, default=None)
+ p.add_argument("--lr", type=float, default=None)
+ p.add_argument("--num_workers", type=int, default=None)
+ p.add_argument("--max_images", type=int, default=None)
+ p.add_argument("--harmony4d_cap", type=int, default=None, help="Max Harmony4D crops (0 = uncapped).")
+ p.add_argument("--overfit-test", dest="overfit_test", action="store_true",
+ help="Run the 1-batch overfit sanity check and exit.")
+ p.add_argument("--no-resume", dest="no_resume", action="store_true")
+ p.add_argument("--no-export", dest="no_export", action="store_true")
+ p.add_argument("--gpu", type=int, default=None)
+ args, unknown = p.parse_known_args()
+ if unknown:
+ print(f"⚠️ Ignoring unrecognized arguments: {unknown}")
+ return args
+
+
+def main():
+ args = parse_args()
+ cfg = PoseConfig()
+ if args.data_root is not None: cfg.data_root = args.data_root
+ if args.output_dir is not None: cfg.log_dir = args.output_dir
+ if args.epochs is not None: cfg.epochs = args.epochs
+ if args.batch_size is not None: cfg.batch_size = args.batch_size
+ if args.lr is not None: cfg.lr = args.lr
+ if args.num_workers is not None: cfg.num_workers = args.num_workers
+ if args.max_images is not None: cfg.max_images = args.max_images
+ if args.harmony4d_cap is not None:
+ if args.harmony4d_cap <= 0:
+ cfg.per_dataset_caps.pop("sam3d_gt_harmony4d", None)
+ else:
+ cfg.per_dataset_caps["sam3d_gt_harmony4d"] = args.harmony4d_cap
+ if args.no_resume: cfg.resume = False
+
+ print("=" * 60)
+ print(f"Data root : {cfg.data_root}")
+ print(f"Output dir : {cfg.log_dir}")
+ print(f"Backbone : {cfg.backbone} | epochs={cfg.epochs} | batch={cfg.batch_size} | lr={cfg.lr}")
+ print(f"Loss : 2D + 3D keypoints + camera (NO MHR mesh)")
+ print("=" * 60)
+
+ train_loader, val_loader, _ = build_dataloaders(cfg)
+
+ if args.overfit_test:
+ run_overfit_test(cfg, train_loader)
+ return
+
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ train_pose3d(cfg, train_loader, val_loader)
+ if not args.no_export:
+ export_onnx(cfg)
+
+
+if __name__ == "__main__":
+ main()