Skip to content
Python

feat: adjusted onnx converter, corrected pose flip augmentation bug, added fine tuning script

mohamdevca2b2623aca1
diff --git a/instanthmr_distill_train/submit_train_distill_finetune_dolcino.sh b/instanthmr_distill_train/submit_train_distill_finetune_dolcino.sh
new file mode 100755
index 0000000..7229f93
--- /dev/null
+++ b/instanthmr_distill_train/submit_train_distill_finetune_dolcino.sh
@@ -0,0 +1,71 @@
+#!/bin/sh -e
+
+echo "example: $0 --gpu 1"
+
+# if `requirements.txt` is present, virtual environment is installed and used
+EXEC="python3 train_distill_optimized_correctives_fine_tune.py"
+
+GPU_REQUIRED=true
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+# --data_root points at the dataset synced by datasynch_perso (see below).
+# It is synced next to this project folder, i.e. ../instanthmr_data once we
+# cd into instanthmr_distill_train.
+#
+# Fine-tune to repair the 2D keypoint head after the flip-label fix
+# (joints_2d[FLIP_PERM] was missing from the geometric augmentation).
+# --init_ckpt is the checkpoint we START FROM; results land in --output_dir.
+# --reset_2d_head is important: the existing 2D head encodes the OLD left/right
+# convention, so fine-tuning it in place has to unlearn that on ~40% of samples
+# and thrashes. Resetting it makes the head learn the corrected convention from
+# scratch on frozen-quality features, which converges cleanly.
+# trunk lr 1e-5 / head lr 3e-4; --pa_tolerance rejects any checkpoint whose
+# Mesh PA-MPJPE regresses more than 2 mm from the measured baseline.
+EXEC="python3 -u train_distill_optimized_correctives_fine_tune.py \
+ --data_root /datasets/instanthmr_data \
+ --init_ckpt runs/distill_optimized_v2/best_student_model_ema.pth \
+ --output_dir runs/distill_optimized_v2_ft \
+ --num_workers 8 --batch_size 128 \
+ --reset_2d_head \
+ --epochs 15 --lr 1e-5 --head_2d_lr_mult 30 \
+ --pa_tolerance 2.0 --early_stop_patience 4 $useropt"
+
+# Zero-risk fallback if the PA guard blocks everything: freezes all but head_2d_*,
+# so the 3D pose provably cannot change (measured: PA held to 0.0 mm over 8 epochs).
+# Slower to converge — the decoder cannot adapt with it.
+# ... --train_2d_only --reset_2d_head --lr 3e-4 --head_2d_lr_mult 1 --epochs 30
+
+# SBATCH --output/--error paths are relative to the submission directory, so
+# create the log folder before submitting.
+mkdir -p instanthmr_distill
+
+jobMessage=$(
+######################### SBATCH launcher
+######################### #SBATCH --option # <= this is an enabled paramater
+######################### ##SBATCH --option # <= this is a disabled parameter
+sbatch ${sbatchopt} << eof
+#!/bin/bash
+#SBATCH --job-name="instanthmr_finetune"
+#SBATCH --mail-type=ALL
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --output=instanthmr_distill/stdout_finetune_dolcino.txt
+#SBATCH --error=instanthmr_distill/stderr_finetune_dolcino.txt
+#SBATCH --account=dept_rob
+#SBATCH --partition=robgpu
+#SBATCH --gres=gpu:1
+#SBATCH --mem-per-gpu=50000
+#SBATCH --time=2-00:00:00
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+/pfcalcul/tools/datasync /datasets/instanthmr_data
+
+
+time $EXEC
+
+eof
+)
+
+showSubmitted
diff --git a/instanthmr_distill_train/submit_train_distill_pose3d_dolcino.sh b/instanthmr_distill_train/submit_train_distill_pose3d_dolcino.sh
new file mode 100755
index 0000000..d76ed0c
--- /dev/null
+++ b/instanthmr_distill_train/submit_train_distill_pose3d_dolcino.sh
@@ -0,0 +1,49 @@
+#!/bin/sh -e
+
+echo "example: $0 --gpu 1"
+
+# if `requirements.txt` is present, virtual environment is installed and used
+EXEC="python3 train_distill_optimized_correctives.py"
+
+GPU_REQUIRED=true
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+# --data_root points at the dataset synced by datasynch_perso (see below).
+# It is synced next to this project folder, i.e. ../instanthmr_data once we
+# cd into instanthmr_distill_train.
+EXEC="python3 -u train_distill_optimized_correctives.py --data_root /datasets/instanthmr_data --num_workers 8 --batch_size 128 $useropt"
+
+# SBATCH --output/--error paths are relative to the submission directory, so
+# create the log folder before submitting.
+mkdir -p instanthmr_distill
+
+jobMessage=$(
+######################### SBATCH launcher
+######################### #SBATCH --option # <= this is an enabled paramater
+######################### ##SBATCH --option # <= this is a disabled parameter
+sbatch ${sbatchopt} << eof
+#!/bin/bash
+#SBATCH --job-name="instanthmr_distill"
+#SBATCH --mail-type=ALL
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --output=instanthmr_distill/stdout_dolcino.txt
+#SBATCH --error=instanthmr_distill/stderr_dolcino.txt
+#SBATCH --account=dept_rob
+#SBATCH --partition=robgpu
+#SBATCH --gres=gpu:1
+#SBATCH --mem-per-gpu=50000
+#SBATCH --time=4-00:00:00
+
+. /pfcalcul/tools/sbatchHelpers5.sh
+
+/pfcalcul/tools/datasync /datasets/instanthmr_data
+
+
+time $EXEC
+
+eof
+)
+
+showSubmitted
diff --git a/instanthmr_distill_train/train_distill_optimized.py b/instanthmr_distill_train/train_distill_optimized.py
index 09a05b1..6bfc78d 100644
--- a/instanthmr_distill_train/train_distill_optimized.py
+++ b/instanthmr_distill_train/train_distill_optimized.py
@@ -454,6 +454,12 @@ class SAM3DStudentDataset(Dataset):
if aug_flip > 0.5:
joints_3d[:, 0] = -joints_3d[:, 0]
joints_3d = joints_3d[FLIP_PERM]
+ # joints_2d was already mirrored geometrically by M_total, but a
+ # mirrored image LOOKS like the opposite-handed person, so the
+ # left/right labels must be swapped too - exactly as for joints_3d.
+ # Without this the 2D head gets contradictory left/right supervision
+ # and collapses symmetric joints toward the body midline.
+ joints_2d = joints_2d[FLIP_PERM]
cam_trans[0] = -cam_trans[0]
cx = orig_w - cx # mirror bbox centre in full frame
rad = math.radians(angle)
diff --git a/instanthmr_distill_train/train_distill_optimized_correctives.py b/instanthmr_distill_train/train_distill_optimized_correctives.py
index d3d1a8f..e57563b 100644
--- a/instanthmr_distill_train/train_distill_optimized_correctives.py
+++ b/instanthmr_distill_train/train_distill_optimized_correctives.py
@@ -373,6 +373,12 @@ class SAM3DStudentDataset(Dataset):
if aug_flip > 0.5:
joints_3d[:, 0] = -joints_3d[:, 0]
joints_3d = joints_3d[FLIP_PERM]
+ # joints_2d was already mirrored geometrically by M_total, but a
+ # mirrored image LOOKS like the opposite-handed person, so the
+ # left/right labels must be swapped too - exactly as for joints_3d.
+ # Without this the 2D head gets contradictory left/right supervision
+ # and collapses symmetric joints toward the body midline.
+ joints_2d = joints_2d[FLIP_PERM]
cam_trans[0] = -cam_trans[0]
cx = orig_w - cx
rad = math.radians(angle)
diff --git a/instanthmr_distill_train/train_distill_optimized_correctives_fine_tune.py b/instanthmr_distill_train/train_distill_optimized_correctives_fine_tune.py
new file mode 100644
index 0000000..e14be7c
--- /dev/null
+++ b/instanthmr_distill_train/train_distill_optimized_correctives_fine_tune.py
@@ -0,0 +1,556 @@
+#!/usr/bin/env python3
+"""
+Fine-tune an InstantHMR student to repair the 2D keypoint head.
+
+Why this script exists
+----------------------
+The geometric augmentation in ``train_distill_optimized_correctives.py`` mirrored
+``joints_2d`` with ``M_total`` but never applied ``FLIP_PERM`` to it, while
+``joints_3d`` *was* permuted. On flipped samples the 2D head was therefore told
+that a joint which LOOKS like a right shoulder is the left shoulder. Since
+``loss_2d_native`` is an L1 on the decoded coordinate at w=10 (vs the SimCC CE at
+w=1), the L1-optimal answer to two contradictory targets is their midpoint, so
+left/right symmetric joints collapsed toward the body midline — the "shoulders
+joining at the neck" artefact. Only x is affected (the mirror is about the
+vertical axis) and only the 2D head, because every MHR/pose/shape loss is masked
+by ``m_noflip``/``m_ident`` and ``loss_3d_native`` uses correctly permuted targets.
+
+That label bug is now fixed at its source. This script does NOT redefine the
+model, dataset or losses — it imports them from
+``train_distill_optimized_correctives.py`` so the fix and every future change stay
+in exactly one place. What it adds is a fine-tuning regime built for this repair:
+
+ * starts from an existing checkpoint instead of ImageNet weights;
+ * discriminative learning rates — a high LR on ``head_2d_*`` to pull it out of
+ the collapsed minimum, a low LR everywhere else so the (good) 3D head, MHR
+ params and backbone are not disturbed;
+ * **2D validation metrics**, because the stock loop selects checkpoints on
+ ``Mesh_PA_MPJPE`` — a 3D metric that is blind to exactly the thing being
+ repaired. Selection here is on 2D error, guarded by a hard limit on how far
+ ``Mesh_PA_MPJPE`` may regress;
+ * a ``SymCollapse_%`` metric that measures the actual symptom (symmetric joint
+ pairs whose predicted x-width is under 70% of ground truth), so you can watch
+ the artefact disappear;
+ * crash/preemption-safe resume from ``last_finetune.pth``.
+
+Checkpoints are written in the same format the stock loop uses, so
+``tools/pth_to_onnx.py`` consumes them unchanged.
+
+What was measured (RTX 4070, balanced 36k-crop subset, 40 val batches)
+---------------------------------------------------------------------
+A probe feeding each image and its mirror shows which convention each head
+learned. The 3D head is cleanly equivariant under the left/right swap
+(residual 0.018, 11.9x margin) — it already uses the "apparent" convention.
+The 2D head follows the opposite convention and only weakly (1.4x margin,
+6x larger residual): it is stuck between the two, which is the collapse.
+
+Fine-tuning the existing 2D head in place therefore has to REVERSE its
+handedness on the ~40% of samples that get flipped, and it thrashes:
+
+ in-place, lr 5e-5, 4 epochs 2D 2.05 -> 8.75 px SymCollapse 2% -> 67%
+
+Resetting the head first, so it learns the corrected convention from scratch
+instead of unlearning the old one, converges monotonically:
+
+ --train_2d_only --reset_2d_head, 8 epochs 2D 8.01 -> 4.98 px Sym 44% -> 14%
+ --reset_2d_head (decoder adapts), 3 epochs 2D 2.32 -> 2.18 px Sym 2.6% -> 2.2%
+
+Letting the transformer decoder adapt is ~10x more sample-efficient — its 2D
+query tokens carry the old convention as well, so freezing them caps how far
+the head can recover. Hence the default recipe: --reset_2d_head, a high LR on
+head_2d_*, a low LR on the trunk, and the PA guard to catch 3D drift.
+--train_2d_only remains the zero-risk fallback: with it, Mesh PA-MPJPE held at
+exactly 23.9 mm for all 8 epochs.
+
+Note these numbers come from a 36k-crop subset, which is ~1/24th of the corpus
+and cannot reach the accuracy of the 359-epoch full-corpus baseline. They show
+direction and stability, not the final quality of a full run.
+
+Usage
+-----
+ python3 -u train_distill_optimized_correctives_fine_tune.py \
+ --data_root /datasets/instanthmr_data --batch_size 128 --num_workers 8 \
+ --reset_2d_head
+"""
+import argparse
+import gc
+import math
+import os
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.optim as optim
+import torch.optim.lr_scheduler as lr_scheduler
+from torch.optim.swa_utils import AveragedModel
+from tqdm import tqdm
+
+# Model / dataset / losses / metrics all come from the main training script so
+# there is exactly one definition of each.
+import train_distill_optimized_correctives as T
+
+PROJECT_ROOT = Path(__file__).resolve().parent
+device = T.device
+
+# Left/right joint pairs, derived from the same permutation the augmentation uses.
+SYM_PAIRS = [(i, int(T.FLIP_PERM[i])) for i in range(len(T.JOINT_NAMES))
+ if i < int(T.FLIP_PERM[i])]
+SYM_A = [a for a, _ in SYM_PAIRS]
+SYM_B = [b for _, b in SYM_PAIRS]
+
+
+# ============================================================
+# 2D metrics — what the stock loop is missing
+# ============================================================
+def evaluate_2d_batch(preds, targets, cfg):
+ """2D accuracy + a direct measurement of the symmetric-collapse artefact.
+
+ ``joints_2d`` lives in normalised [-1, 1] crop space, so one unit is
+ ``image_size / 2`` pixels of the 224x224 crop.
+ """
+ p2 = preds["joints_2d"][..., :2].float()
+ t2 = targets["joints_2d"][..., :2].float()
+
+ px_per_unit = cfg.image_size / 2.0
+ err_px = torch.linalg.norm(p2 - t2, dim=-1) * px_per_unit # [B, J]
+
+ m = {
+ "KP2D_L2_px": err_px.mean().item(),
+ "[email protected]": (err_px < 0.05 * cfg.image_size).float().mean().item() * 100.0,
+ }
+
+ # Symmetric-pair x-width: the quantity that collapses when left/right
+ # supervision is contradictory. Pairs that genuinely overlap in x (limbs
+ # crossed, person in profile) carry no signal, so they are excluded.
+ wp = (p2[:, SYM_A, 0] - p2[:, SYM_B, 0]).abs()
+ wt = (t2[:, SYM_A, 0] - t2[:, SYM_B, 0]).abs()
+ valid = wt > 0.05
+ n_valid = valid.sum().clamp_min(1)
+ ratio = wp / wt.clamp_min(1e-6)
+ m["SymWidth_ratio"] = (ratio * valid).sum().item() / n_valid.item()
+ m["SymCollapse_%"] = (((ratio < 0.7) & valid).sum().item() / n_valid.item()) * 100.0
+ return m
+
+
+# ============================================================
+# Parameter groups
+# ============================================================
+def build_param_groups(model, base_lr, head_2d_lr_mult, freeze_backbone, train_2d_only):
+ """High LR on the 2D head, low LR on everything else.
+
+ The 3D head, the MHR/shape/camera head and the backbone are already good —
+ they only need enough LR to stay consistent with the repaired 2D head.
+
+ ``train_2d_only`` freezes every parameter the flip bug could not have
+ touched, leaving just ``head_2d_*``. Because the backbone, decoder, 3D head
+ and MHR head are then bit-identical to the input checkpoint, Mesh/Head
+ PA-MPJPE cannot move at all — the repair is structurally incapable of
+ regressing the 3D pose. It is the safest mode and the fastest per step.
+ """
+ head_2d, trunk, frozen = [], [], []
+ for name, p in model.named_parameters():
+ if not p.requires_grad:
+ continue
+ is_2d = name.startswith("head_2d_")
+ if is_2d:
+ head_2d.append(p)
+ elif train_2d_only or (freeze_backbone and name.startswith("backbone.")):
+ p.requires_grad_(False)
+ frozen.append(name)
+ else:
+ trunk.append(p)
+
+ groups = [{"params": head_2d, "lr": base_lr * head_2d_lr_mult, "name": "head_2d"}]
+ if trunk:
+ groups.append({"params": trunk, "lr": base_lr, "name": "trunk"})
+ n_head = sum(p.numel() for p in head_2d)
+ n_trunk = sum(p.numel() for p in trunk)
+ msg = f" param groups: head_2d {n_head/1e6:.2f}M @ lr {base_lr*head_2d_lr_mult:.2e}"
+ if trunk:
+ msg += f" | trunk {n_trunk/1e6:.2f}M @ lr {base_lr:.2e}"
+ if frozen:
+ msg += f" | FROZEN {len(frozen)} tensors"
+ if train_2d_only:
+ msg += " (train_2d_only: 3D pose cannot change)"
+ print(msg)
+ return groups
+
+
+def load_init_checkpoint(model, path):
+ ckpt = torch.load(path, map_location=device, weights_only=False)
+ state = ckpt.get("model_state_dict", ckpt)
+ state = {k.replace("_orig_mod.", ""): v for k, v in state.items()}
+ missing, unexpected = model.load_state_dict(state, strict=False)
+ if missing:
+ raise SystemExit(
+ f"[error] {len(missing)} parameter(s) missing from {path}:\n"
+ f" {missing[:8]}\n"
+ f" The checkpoint does not match this architecture — refusing to "
+ f"fine-tune from randomly initialised weights."
+ )
+ if unexpected:
+ print(f" ⚠️ ignoring {len(unexpected)} unexpected key(s): {unexpected[:6]}")
+ print(f" init from {path}")
+ print(f" (epoch {ckpt.get('epoch', '?')} | PA-MPJPE {ckpt.get('val_pa_mpjpe', '?')} "
+ f"| source {ckpt.get('source', '?')})")
+ return ckpt
+
+
+# ============================================================
+# Fine-tuning loop
+# ============================================================
+def finetune(cfg, args, train_loader, val_loader, mhr_module):
+ os.makedirs(cfg.log_dir, exist_ok=True)
+
+ best_raw_path = os.path.join(cfg.log_dir, "best_finetune_raw.pth")
+ best_ema_path = os.path.join(cfg.log_dir, "best_finetune_ema.pth")
+ best_path = os.path.join(cfg.log_dir, "best_finetune.pth")
+ last_path = os.path.join(cfg.log_dir, "last_finetune.pth")
+
+ model = T.InstantHMRStudent(cfg, pretrained=False).to(device)
+ load_init_checkpoint(model, args.init_ckpt)
+ criterion = T.DistillationLoss(cfg, mhr_module)
+ scaler = torch.amp.GradScaler(device="cuda", enabled=cfg.use_amp)
+
+ @torch.no_grad()
+ def run_validation(eval_model, tag, epoch):
+ eval_model.eval()
+ acc, n = {}, 0
+ limit = args.val_max_batches if args.val_max_batches > 0 else len(val_loader)
+ vbar = tqdm(val_loader, total=min(limit, len(val_loader)),
+ desc=f"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]", leave=False)
+ for i, batch in enumerate(vbar):
+ if i >= limit:
+ break
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v
+ for k, v in batch.items()}
+ with torch.amp.autocast(device_type="cuda", dtype=torch.float16, enabled=cfg.use_amp):
+ preds_fp16 = eval_model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds_fp16.items()}
+ losses = criterion(preds, batch)
+ tot = losses["total_loss"].item()
+ if math.isnan(tot) or math.isinf(tot):
+ continue
+ m = T.evaluate_hmr_batch(preds, batch, cfg, mhr_module)
+ m.update(evaluate_2d_batch(preds, batch, cfg))
+ m["loss"] = tot
+ for k, v in m.items():
+ acc[k] = acc.get(k, 0.0) + v
+ n += 1
+ vbar.set_postfix({"2D px": f"{acc.get('KP2D_L2_px',0)/max(n,1):.2f}",
+ "PA": f"{acc.get('Mesh_PA_MPJPE',0)/max(n,1):.1f}"})
+ vbar.close()
+ return {k: v / max(n, 1) for k, v in acc.items()}, n
+
+ def fmt(tag, m):
+ return (f" [{tag}] 2D: {m.get('KP2D_L2_px', 0):.2f} px | [email protected]: {m.get('[email protected]', 0):.1f}% "
+ f"| SymCollapse: {m.get('SymCollapse_%', 0):.2f}% | SymW: {m.get('SymWidth_ratio', 0):.3f} "
+ f"| Mesh PA: {m.get('Mesh_PA_MPJPE', 0):.1f} | Head PA: {m.get('Head_3D_PA_MPJPE', 0):.1f} mm")
+
+ # ---- Baseline: measure the starting point before touching the weights ----
+ start_epoch = 0
+ best_2d = float("inf")
+ best_raw_2d = float("inf")
+ best_ema_2d = float("inf")
+ epochs_no_improve = 0
+
+ resuming = args.resume and os.path.exists(last_path)
+
+ if not resuming:
+ print("\n📏 Baseline (before fine-tuning):")
+ baseline, nb = run_validation(model, "baseline", -1)
+ print(fmt("BASE", baseline))
+ if nb == 0:
+ raise SystemExit("[error] baseline validation produced no batches.")
+ if args.reset_2d_head:
+ # Done AFTER the baseline so the printed starting point is the real
+ # model, and BEFORE the EMA is built so the shadow starts from the
+ # reset head rather than the collapsed one.
+ print(" 🔄 re-initialising head_2d_logits — the 2D head relearns "
+ "left/right from scratch instead of unlearning the old convention")
+ torch.nn.init.normal_(model.head_2d_logits.weight, mean=0.0, std=1e-4)
+ torch.nn.init.constant_(model.head_2d_logits.bias, 0.0)
+
+ groups = build_param_groups(model, cfg.lr, args.head_2d_lr_mult,
+ args.freeze_backbone, args.train_2d_only)
+ optimizer = optim.AdamW(groups, lr=cfg.lr, weight_decay=cfg.weight_decay)
+ scheduler = lr_scheduler.OneCycleLR(
+ optimizer, max_lr=[g["lr"] for g in groups],
+ epochs=cfg.epochs, steps_per_epoch=len(train_loader),
+ pct_start=args.pct_start,
+ )
+ ema_model = AveragedModel(model, multi_avg_fn=T.make_ema_avg_fn(args.ema_decay),
+ use_buffers=True)
+
+ if resuming:
+ ck = torch.load(last_path, map_location=device, weights_only=False)
+ model.load_state_dict(ck["model_state_dict"])
+ ema_model.module.load_state_dict(ck["ema_state_dict"])
+ optimizer.load_state_dict(ck["optimizer_state_dict"])
+ scheduler.load_state_dict(ck["scheduler_state_dict"])
+ scaler.load_state_dict(ck["scaler_state_dict"])
+ start_epoch = ck["epoch"] + 1
+ best_2d = ck.get("best_2d", float("inf"))
+ best_raw_2d = ck.get("best_raw_2d", float("inf"))
+ best_ema_2d = ck.get("best_ema_2d", float("inf"))
+ epochs_no_improve = ck.get("epochs_no_improve", 0)
+ baseline = ck["baseline"]
+ print(f"🔄 Resumed fine-tune from epoch {start_epoch} (best 2D {best_2d:.3f} px)")
+
+ pa_ceiling = baseline["Mesh_PA_MPJPE"] + args.pa_tolerance
+ print(f"\n selection metric : KP2D_L2_px (lower is better)")
+ print(f" guard : Mesh PA-MPJPE must stay <= {pa_ceiling:.1f} mm "
+ f"(baseline {baseline['Mesh_PA_MPJPE']:.1f} + tol {args.pa_tolerance})\n")
+
+ def set_train_mode(m):
+ """Frozen params are not enough: BatchNorm running stats and dropout still
+ move in train() mode, which would drift the 3D pose. Under
+ --train_2d_only keep every other module in eval()."""
+ if args.train_2d_only:
+ m.eval()
+ m.head_2d_feat.train()
+ m.head_2d_logits.train()
+ else:
+ m.train()
+
+ for epoch in range(start_epoch, cfg.epochs):
+ set_train_mode(model)
+ tm, nb = {}, 0
+ pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{cfg.epochs} [Train]")
+ for batch in pbar:
+ batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v
+ for k, v in batch.items()}
+ optimizer.zero_grad(set_to_none=True)
+ with torch.amp.autocast(device_type="cuda", dtype=torch.float16, enabled=cfg.use_amp):
+ preds_fp16 = model(batch["image"], batch["cliff_cond"])
+ preds = {k: v.float() for k, v in preds_fp16.items()}
+ losses = criterion(preds, batch)
+ loss = losses["total_loss"]
+ lv = loss.item()
+ if math.isnan(lv) or math.isinf(lv):
+ print("⚠️ NaN/Inf loss — skipping step.")
+ continue
+ if lv > cfg.anomaly_loss_threshold:
+ print(f"⚠️ anomalous loss {lv:.1f} — skipping step.")
+ continue
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
+ s_before = scaler.get_scale()
+ scaler.step(optimizer)
+ scaler.update()
+ if s_before <= scaler.get_scale():
+ scheduler.step()
+ ema_model.update_parameters(model)
+ for k in ("total_loss", "loss_2d_native", "loss_2d_simcc", "loss_3d_native"):
+ tm[k] = tm.get(k, 0.0) + losses.get(k, torch.tensor(0.0)).item()
+ nb += 1
+ pbar.set_postfix({"Tot": f"{lv:.3f}", "2D": f"{losses.get('loss_2d_native', torch.tensor(0.0)).item():.3f}"})
+ pbar.close()
+ tr = {k: v / max(nb, 1) for k, v in tm.items()}
+
+ raw, n_raw = run_validation(model, "raw", epoch)
+ ema, n_ema = run_validation(ema_model, "ema", epoch)
+
+ lrs = scheduler.get_last_lr()
+ print(f"\n📈 Epoch {epoch+1}/{cfg.epochs} | LR head_2d {lrs[0]:.2e} / trunk {lrs[-1]:.2e}")
+ print(f" [Train] Tot: {tr['total_loss']:.4f} | 2D: {tr['loss_2d_native']:.4f} "
+ f"| simcc: {tr['loss_2d_simcc']:.4f} | 3D: {tr['loss_3d_native']:.4f}")
+ print(fmt("Val RAW", raw))
+ print(fmt("Val EMA", ema))
+ print(f" Δ vs baseline: 2D {raw['KP2D_L2_px']-baseline['KP2D_L2_px']:+.2f} px (raw) / "
+ f"{ema['KP2D_L2_px']-baseline['KP2D_L2_px']:+.2f} px (ema) | "
+ f"SymCollapse {baseline['SymCollapse_%']:.2f}% → "
+ f"{min(raw['SymCollapse_%'], ema['SymCollapse_%']):.2f}%")
+
+ def save_dict(m_to_save, metrics, source):
+ return {
+ "epoch": epoch,
+ "model_state_dict": m_to_save.state_dict(),
+ "optimizer_state_dict": optimizer.state_dict(),
+ "scaler_state_dict": scaler.state_dict(),
+ "scheduler_state_dict": scheduler.state_dict(),
+ "val_pa_mpjpe": metrics["Mesh_PA_MPJPE"],
+ "val_kp2d_px": metrics["KP2D_L2_px"],
+ "val_metrics": metrics,
+ "source": source,
+ "finetune_of": str(args.init_ckpt),
+ }
+
+ # A checkpoint only counts if the 3D head has not regressed past the guard.
+ raw_ok = n_raw > 0 and raw["Mesh_PA_MPJPE"] <= pa_ceiling
+ ema_ok = n_ema > 0 and ema["Mesh_PA_MPJPE"] <= pa_ceiling
+ if not raw_ok and not ema_ok:
+ print(f" ⚠️ PA guard blocked both: raw {raw['Mesh_PA_MPJPE']:.1f} / "
+ f"ema {ema['Mesh_PA_MPJPE']:.1f} mm > {pa_ceiling:.1f} mm")
+
+ improved_raw = raw_ok and raw["KP2D_L2_px"] < best_raw_2d
+ improved_ema = ema_ok and ema["KP2D_L2_px"] < best_ema_2d
+ if improved_raw:
+ best_raw_2d = raw["KP2D_L2_px"]
+ torch.save(save_dict(model, raw, "raw"), best_raw_path)
+ if improved_ema:
+ best_ema_2d = ema["KP2D_L2_px"]
+ torch.save(save_dict(ema_model.module, ema, "ema"), best_ema_path)
+
+ cands = [(raw["KP2D_L2_px"], model, raw, "raw")] if raw_ok else []
+ if ema_ok:
+ cands.append((ema["KP2D_L2_px"], ema_model.module, ema, "ema"))
+ if cands:
+ val, mdl, met, src = min(cands, key=lambda c: c[0])
+ if val < best_2d - 1e-4:
+ best_2d = val
+ torch.save(save_dict(mdl, met, src), best_path)
+ print(f"💾 New best 2D: {best_2d:.3f} px ({src.upper()}, "
+ f"Mesh PA {met['Mesh_PA_MPJPE']:.1f} mm) -> {os.path.basename(best_path)}")
+
+ torch.save({**save_dict(model, raw, "last"),
+ "ema_state_dict": ema_model.module.state_dict(),
+ "baseline": baseline, "best_2d": best_2d,
+ "best_raw_2d": best_raw_2d, "best_ema_2d": best_ema_2d,
+ "epochs_no_improve": epochs_no_improve}, last_path)
+
+ if improved_raw or improved_ema:
+ epochs_no_improve = 0
+ else:
+ epochs_no_improve += 1
+ print(f" ⏳ no 2D improvement: {epochs_no_improve}/{args.early_stop_patience}")
+ if epochs_no_improve >= args.early_stop_patience:
+ print(f"\n🛑 Early stopping at epoch {epoch+1}.")
+ break
+
+ print(f"\n🎉 Fine-tuning complete.")
+ print(f" baseline 2D : {baseline['KP2D_L2_px']:.3f} px | "
+ f"SymCollapse {baseline['SymCollapse_%']:.2f}% | Mesh PA {baseline['Mesh_PA_MPJPE']:.1f} mm")
+ print(f" best 2D : {best_2d:.3f} px -> {best_path}")
+ print(f"\n Export with:\n"
+ f" python tools/pth_to_onnx.py --ckpt {best_path} --output models/instanthmr.onnx")
+
+
+# ============================================================
+# CLI
+# ============================================================
+def parse_args():
+ p = argparse.ArgumentParser(
+ description="Fine-tune InstantHMR to repair the 2D head after the flip-label fix.")
+ p.add_argument("--data_root", type=str, default=None)
+ p.add_argument("--output_dir", type=str,
+ default=str(PROJECT_ROOT / "runs/distill_optimized_v2_ft"))
+ p.add_argument("--init_ckpt", type=str,
+ default=str(PROJECT_ROOT / "runs/distill_optimized_v2/best_student_model_ema.pth"),
+ help="Checkpoint to fine-tune FROM (not the output).")
+ p.add_argument("--mhr_model_path", type=str, default=None)
+
+ p.add_argument("--epochs", type=int, default=15)
+ p.add_argument("--batch_size", type=int, default=None)
+ p.add_argument("--num_workers", type=int, default=None)
+ p.add_argument("--lr", type=float, default=1e-5,
+ help="Trunk LR. The 2D head gets lr * --head_2d_lr_mult.")
+ p.add_argument("--head_2d_lr_mult", type=float, default=30.0,
+ help="LR multiplier for head_2d_* (default 30 -> 3e-4 at the default lr).")
+ p.add_argument("--pct_start", type=float, default=0.05, help="OneCycle warmup fraction.")
+ p.add_argument("--ema_decay", type=float, default=0.9995)
+ p.add_argument("--early_stop_patience", type=int, default=4)
+
+ p.add_argument("--train_2d_only", action="store_true",
+ help="Train ONLY head_2d_* and freeze everything else. The 3D pose then "
+ "cannot change at all — safest way to repair the 2D head.")
+ p.add_argument("--freeze_backbone", action="store_true",
+ help="Freeze the RepViT backbone (faster, more conservative).")
+ p.add_argument("--reset_2d_head", action="store_true",
+ help="Re-initialise head_2d_logits instead of fine-tuning it.")
+ p.add_argument("--pa_tolerance", type=float, default=2.0,
+ help="Max Mesh PA-MPJPE regression (mm) allowed vs baseline before a "
+ "checkpoint is rejected.")
+ p.add_argument("--val_max_batches", type=int, default=150,
+ help="Cap validation batches per pass for speed (0 = full val set).")
+
+ p.add_argument("--per_dataset_cap", type=int, default=None,
+ help="Cap EVERY sub-dataset to N randomly-sampled crops, for a BALANCED "
+ "subset. Prefer this over --max_images for quick runs.")
+ p.add_argument("--max_images", type=int, default=None,
+ help="Truncate to the first N crops. NOTE: this is a path-sorted PREFIX, so "
+ "it draws from one sub-dataset only — it biases any short run. Use "
+ "--per_dataset_cap instead unless you specifically want a prefix.")
+ p.add_argument("--harmony4d_cap", type=int, default=None)
+ p.add_argument("--no-resume", dest="resume", action="store_false",
+ help="Ignore last_finetune.pth and restart the fine-tune.")
+ p.add_argument("--gpu", type=int, default=None, help="GPU count (informational).")
+ args, unknown = p.parse_known_args()
+ if unknown:
+ print(f"⚠️ Ignoring unrecognized arguments: {unknown}")
+ return args
+
+
+def main():
+ args = parse_args()
+ cfg = T.DistillConfig()
+
+ if args.data_root is not None: cfg.data_root = args.data_root
+ if args.mhr_model_path is not None: cfg.mhr_model_path = args.mhr_model_path
+ if args.batch_size is not None: cfg.batch_size = args.batch_size
+ if args.num_workers is not None: cfg.num_workers = args.num_workers
+ if args.max_images is not None:
+ cfg.max_images = args.max_images
+ print("⚠️ --max_images takes a PATH-SORTED PREFIX (one sub-dataset), which biases "
+ "training. Use --per_dataset_cap for a balanced subset.")
+ if args.per_dataset_cap is not None:
+ subs = sorted(d.name for d in Path(cfg.data_root).iterdir()
+ if (d / "annotations").is_dir() and (d / "images").is_dir())
+ cfg.per_dataset_caps = {s_: args.per_dataset_cap for s_ in subs}
+ print(f" balanced subset: {args.per_dataset_cap:,} crops max from each of "
+ f"{len(subs)} sub-dataset(s)")
+ if args.harmony4d_cap is not None:
+ if args.harmony4d_cap <= 0:
+ cfg.per_dataset_caps.pop("sam3d_gt_harmony4d", None)
+ else:
+ cfg.per_dataset_caps["sam3d_gt_harmony4d"] = args.harmony4d_cap
+ cfg.log_dir = args.output_dir
+ cfg.epochs = args.epochs
+ cfg.lr = args.lr
+ cfg.resume = False # this script manages its own resume via last_finetune.pth
+
+ print("=" * 70)
+ print("InstantHMR 2D-head repair fine-tune")
+ print("=" * 70)
+ print(f"Data root : {cfg.data_root}")
+ print(f"Init ckpt : {args.init_ckpt}")
+ print(f"Output dir : {cfg.log_dir}")
+ print(f"MHR model : {cfg.mhr_model_path}")
+ print(f"epochs={cfg.epochs} | batch={cfg.batch_size} | trunk_lr={cfg.lr} "
+ f"| head_2d_lr={cfg.lr * args.head_2d_lr_mult:.2e}")
+ print(f"geom aug : p={cfg.geom_p} rot±{cfg.geom_rot_deg}° scale±{cfg.geom_scale_range} "
+ f"trans±{cfg.geom_trans} flip={cfg.geom_flip_p}")
+ print(f"flip fix : joints_2d[FLIP_PERM] applied "
+ f"({len(SYM_PAIRS)} left/right pairs, "
+ f"{int((T.FLIP_PERM != np.arange(len(T.FLIP_PERM))).sum())}/{len(T.FLIP_PERM)} joints swapped)")
+ print(f"train_2d_only={args.train_2d_only} | freeze_bb={args.freeze_backbone} "
+ f"| reset_2d_head={args.reset_2d_head} | pa_tolerance={args.pa_tolerance} mm")
+ print("=" * 70)
+
+ if not args.reset_2d_head:
+ print("⚠️ --reset_2d_head is NOT set. The existing 2D head encodes the OLD "
+ "left/right convention,\n so fine-tuning it in place must UNLEARN that on "
+ "~40% of samples; measured runs thrash\n (2D error and symmetric collapse both "
+ "got worse) instead of converging. Strongly recommended.\n")
+
+ if not Path(args.init_ckpt).exists():
+ raise SystemExit(f"[error] init checkpoint not found: {args.init_ckpt}")
+ if not Path(cfg.mhr_model_path).exists():
+ raise SystemExit(
+ f"[error] MHR model not found at '{cfg.mhr_model_path}'. "
+ f"Pass --mhr_model_path.")
+
+ train_loader, val_loader, _ = T.build_dataloaders(cfg)
+ mhr_module = T.MHRForwardPass(cfg.mhr_model_path, device)
+
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+ finetune(cfg, args, train_loader, val_loader, mhr_module)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/pth_to_onnx.py b/tools/pth_to_onnx.py
index 7e27316..aaf5b53 100644
--- a/tools/pth_to_onnx.py
+++ b/tools/pth_to_onnx.py
@@ -3,13 +3,24 @@
Convert a trained InstantHMR student checkpoint (.pth) into the ONNX file the
demo expects at ``models/instanthmr.onnx``.
-It reuses the exact model architecture and deploy wrapper from
-``instanthmr_distill_train/train_distill.py`` and the FP32 export recipe from
-notebook Cell 15, so the graph I/O matches ``instanthmr/inference.py``:
+It reuses the exact model architecture and deploy wrapper from the training
+script that produced the checkpoint, plus the FP32 export recipe from notebook
+Cell 15, so the graph I/O matches ``instanthmr/inference.py``:
inputs : image (N, 3, 224, 224), cliff_cond (N, 3)
outputs: mhr_params, shape_params, cam_trans, joints_2d, joints_3d (in this order)
+Architecture selection
+----------------------
+The student head changed between training scripts: ``train_distill.py``
+regresses 2D joints directly (``head_2d_xy``), while the optimized/correctives
+scripts predict them with a SimCC classification head (``head_2d_logits`` +
+``kp2d_bin_centers``). Exporting a SimCC checkpoint through the old architecture
+silently leaves ``head_2d_xy`` at its near-zero init, which collapses every 2D
+keypoint onto the centre of the bbox. ``--arch auto`` (the default) picks the
+right module by inspecting the checkpoint, and a missing-key check makes any
+future mismatch a hard error instead of a silent one.
+
Usage
-----
# default: convert the EMA checkpoint -> models/instanthmr.onnx
@@ -17,10 +28,11 @@ Usage
# or point at a specific checkpoint / output
python tools/pth_to_onnx.py \
- --ckpt instanthmr_distill_train/runs/distill_repvit_cliff_v2/best_student_model_ema.pth \
+ --ckpt instanthmr_distill_train/runs/distill_optimized_v2/best_student_model_ema.pth \
--output models/instanthmr.onnx
"""
import argparse
+import importlib
import sys
from pathlib import Path
@@ -29,19 +41,44 @@ import torch
REPO = Path(__file__).resolve().parent.parent
TRAIN_DIR = REPO / "instanthmr_distill_train"
-# The model class + deploy wrapper live in the standalone training script.
+# The model class + deploy wrapper live in the standalone training scripts.
sys.path.insert(0, str(TRAIN_DIR))
-import train_distill as T # noqa: E402 (InstantHMRStudent, DistillConfig, HMRDeployWrapper)
+
+# arch name -> training module that defines the matching InstantHMRStudent.
+ARCH_MODULES = {
+ "correctives": "train_distill_optimized_correctives", # SimCC 2D head (head_2d_logits)
+ "optimized": "train_distill_optimized", # SimCC 2D head, identical weights
+ "legacy": "train_distill", # direct 2D regression (head_2d_xy)
+}
+
+
+def detect_arch(state: dict) -> str:
+ """Pick the architecture that matches the checkpoint's 2D head."""
+ if "head_2d_logits.weight" in state:
+ # `optimized` and `correctives` share identical parameters (they differ
+ # only by an fp32 softmax cast), so either module loads such a
+ # checkpoint; prefer the newer one.
+ return "correctives"
+ if "head_2d_xy.0.weight" in state:
+ return "legacy"
+ raise SystemExit(
+ "[error] cannot infer architecture: checkpoint has neither "
+ "'head_2d_logits.weight' nor 'head_2d_xy.0.weight'. Pass --arch explicitly."
+ )
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--ckpt", type=Path,
- default=TRAIN_DIR / "runs/distill_repvit_cliff_v2/best_student_model_ema.pth",
+ default=TRAIN_DIR / "runs/distill_optimized_v2/best_student_model_ema.pth",
help="Path to the .pth checkpoint (default: the EMA checkpoint).")
ap.add_argument("--output", type=Path, default=REPO / "models" / "instanthmr.onnx",
help="Where to write the ONNX (default: models/instanthmr.onnx).")
+ ap.add_argument("--arch", choices=["auto", *ARCH_MODULES], default="auto",
+ help="Student architecture / training module (default: auto-detect).")
+ ap.add_argument("--allow-missing", action="store_true",
+ help="Export even if checkpoint weights are missing (NOT recommended).")
ap.add_argument("--opset", type=int, default=17)
args = ap.parse_args()
@@ -49,20 +86,42 @@ def main() -> None:
sys.exit(f"[error] checkpoint not found: {args.ckpt}")
device = torch.device("cpu") # export on CPU — portable, no CUDA needed
- cfg = T.DistillConfig() # default architecture (matches training)
- # --- Build the model and load weights ---
- model = T.InstantHMRStudent(cfg, pretrained=False).to(device).eval()
ckpt = torch.load(args.ckpt, map_location=device, weights_only=False)
state = ckpt.get("model_state_dict", ckpt)
state = {k.replace("_orig_mod.", ""): v for k, v in state.items()} # strip torch.compile prefix
+
+ arch = detect_arch(state) if args.arch == "auto" else args.arch
+ T = importlib.import_module(ARCH_MODULES[arch])
+ print(f"Architecture: {arch} (from {ARCH_MODULES[arch]}.py)")
+
+ cfg = T.DistillConfig() # default architecture (matches training)
+
+ # SimCC head geometry lives in the checkpoint — mirror it so the graph and
+ # the bin centres agree with what was actually trained.
+ if "head_2d_logits.bias" in state:
+ cfg.kp2d_bins = state["head_2d_logits.bias"].shape[0] // 2
+ if "kp2d_bin_centers" in state:
+ cfg.kp2d_range = float(state["kp2d_bin_centers"][-1])
+ print(f" SimCC head: {cfg.kp2d_bins} bins over [-{cfg.kp2d_range}, {cfg.kp2d_range}]")
+
+ # --- Build the model and load weights ---
+ model = T.InstantHMRStudent(cfg, pretrained=False).to(device).eval()
missing, unexpected = model.load_state_dict(state, strict=False)
print(f"Loaded {args.ckpt.name} | epoch {ckpt.get('epoch', '?')} "
f"| PA-MPJPE {ckpt.get('val_pa_mpjpe', '?')} | source {ckpt.get('source', '?')}")
- if missing:
- print(f" missing keys : {len(missing)}")
if unexpected:
- print(f" unexpected keys: {len(unexpected)}")
+ print(f" unexpected keys: {len(unexpected)} -> {unexpected[:8]}")
+ if missing:
+ print(f" missing keys : {len(missing)} -> {missing[:8]}")
+ if not args.allow_missing:
+ sys.exit(
+ f"\n[error] {len(missing)} parameter(s) had no weights in the checkpoint, so they "
+ f"would be exported at random init.\n"
+ f" This usually means --arch is wrong for this checkpoint "
+ f"(detected: {arch}).\n"
+ f" Re-run with the right --arch, or --allow-missing to force it."
+ )
deploy = T.HMRDeployWrapper(model).eval()
@@ -91,7 +150,15 @@ def main() -> None:
outs = sess.run(None, {in_names[0]: dummy_img.numpy(), in_names[1]: dummy_cliff.numpy()})
print(f" onnxruntime OK | inputs {in_names}")
for o, v in zip(sess.get_outputs(), outs):
- print(f" {o.name:14s} {list(np.asarray(v).shape)}")
+ v = np.asarray(v)
+ print(f" {o.name:14s} {list(v.shape)} std={v.std():.4f}")
+ # A dead 2D head (wrong arch, weights left at init) predicts every joint
+ # at the crop centre; real predictions spread over normalised [-1, 1].
+ j2d = np.asarray(outs[output_keys.index("joints_2d")])
+ if j2d.std() < 0.01:
+ print(f"\n[warning] joints_2d std is {j2d.std():.5f} — the 2D keypoints are collapsed "
+ f"onto the crop centre.\n The exported 2D head is almost certainly "
+ f"untrained (wrong --arch?).")
except Exception as e: # noqa: BLE001
print(f" (onnxruntime sanity check skipped: {e})")