Skip to content
Python

feat: added training from ground truth data. fixed out of memory issues

mohamdev0a78ed8baf73
diff --git a/.gitignore b/.gitignore
index 72fb3f4..c643b2d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ rf-detr*.pth
data/
annotations/
images/
+checkpoints/
# Model weights — too large for git, use Git LFS or release artifacts
models/*.onnx
diff --git a/README.md b/README.md
index d4334a7..a4d6550 100644
--- a/README.md
+++ b/README.md
@@ -148,6 +148,60 @@ The InstantHMR ONNX weights are hosted on HuggingFace:
Download `instanthmr.onnx` and place it under `models/`, or pass any path
via `--model`.
+## Training
+
+InstantHMR can be trained from two different label sources. **Both produce the
+exact same `images/*.png` + `annotations/*.npz` layout** that
+[`notebooks/distill_transformer_decoder.ipynb`](notebooks/distill_transformer_decoder.ipynb)
+consumes, so the training notebook is identical either way:
+
+1. **Distillation (default).** Run the SAM3D teacher over your own images with
+ [`tools/annotate_dataset.py`](tools/annotate_dataset.py) to generate
+ per-person pseudo-labels — see [`docs/annotation.md`](docs/annotation.md).
+2. **Original ground-truth annotations.** Train directly on the released
+ [`facebook/sam-3d-body-dataset`](https://huggingface.co/datasets/facebook/sam-3d-body-dataset)
+ — the human MHR fits used to build SAM 3D Body — converted to the same
+ `.npz` schema by [`tools/parquet_to_npz.py`](tools/parquet_to_npz.py).
+
+### Train on the original annotations (COCO example)
+
+The dataset is gated: first **request access** on the
+[dataset page](https://huggingface.co/datasets/facebook/sam-3d-body-dataset)
+and `huggingface-cli login`. The annotation parquets ship **without images**, so
+you download the source images once per sub-dataset (COCO and MPII are the
+easiest — a single `wget`; others may need extra access / undistortion, see the
+[dataset setup guides](https://github.com/facebookresearch/sam-3d-body/tree/main/data)).
+
+```bash
+# 1. Annotations — download the coco_train split (uses Meta's sam-3d-body repo)
+git clone https://github.com/facebookresearch/sam-3d-body
+python sam-3d-body/data/scripts/download.py \
+ --save_dir $ANN_DIR --splits coco_train
+
+# 2. Images — COCO 2014 train (the split the parquet filenames reference)
+mkdir -p $COCO_IMG_DIR && cd $COCO_IMG_DIR
+wget http://images.cocodataset.org/zips/train2014.zip && unzip -q train2014.zip
+cd -
+
+# 3. Convert parquet -> per-crop images/ + annotations/ (.npz)
+python tools/parquet_to_npz.py \
+ --annotation_dir $ANN_DIR/coco_train \
+ --image_dir $COCO_IMG_DIR \
+ --output_dir data/sam3d_gt_coco \
+ --validate
+
+# 4. Point the notebook's config at the output and train:
+# cfg.images_dir = "data/sam3d_gt_coco/images"
+# cfg.annotations_dir = "data/sam3d_gt_coco/annotations"
+```
+
+The converter recovers `cam_trans` from the MHR params, derives the 1.2× square
+crop, skips low-quality fits (`mhr_valid`), and writes the same keys
+`tools/annotate_dataset.py` does. `--validate` asserts that each person's
+`joints_3d + cam_trans` reprojects onto the stored `joints_2d` to < 1 px. Other
+splits work the same way — swap `coco_train` for `mpii_train`,
+`harmony4d_train`, etc., and point `--image_dir` at that dataset's images.
+
## Documentation
- **[`docs/architecture.md`](docs/architecture.md)** — what the network
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynb
index 2d43919..0b03bf5 100644
--- a/notebooks/distill_transformer_decoder.ipynb
+++ b/notebooks/distill_transformer_decoder.ipynb
@@ -5,7 +5,23 @@
"execution_count": null,
"id": "ad5fd1be",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']\n",
+ "Using device: cuda\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\u001b[0;93m2026-06-25 13:28:09.821708154 [W:onnxruntime:Default, device_discovery.cc:283 GetGpuDevices] Failed to detect devices under \"/sys/class/drm/card0\": device_discovery.cc:93 ReadFileContents Failed to open file: \"/sys/class/drm/card0/device/vendor\"\u001b[m\n"
+ ]
+ }
+ ],
"source": [
"import torch\n",
"import torch.nn as nn\n",
@@ -22,15 +38,28 @@
"\n",
"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "import os; os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
"print(f\"Using device: {device}\")"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"id": "7e2ae437",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Project root: /home/madjel/Projects/gitpackages/sam3d-student-release\n",
+ "Images dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/images\n",
+ "Annotations dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/annotations\n",
+ "MHR model path: /home/madjel/Projects/gitpackages/sam3d-student-release/checkpoints/mhr_model.pt\n",
+ "Config loaded. Backbone: repvit_m2_3. Ready for V3 Training.\n"
+ ]
+ }
+ ],
"source": [
"# ============================================================\n",
"# Cell 1 — Configuration\n",
@@ -38,17 +67,23 @@
"from dataclasses import dataclass \n",
"from pathlib import Path\n",
"\n",
- "PROJECT_ROOT = Path(\"\") #set your path\n",
+ "PROJECT_ROOT = Path(\"/home/madjel/Projects/gitpackages/sam3d-student-release/\") #set your path\n",
"\n",
"@dataclass\n",
"class DistillConfig:\n",
" \"\"\"All hyperparameters in one place.\"\"\"\n",
" # --- Paths ---\n",
- " images_dir: str = str(PROJECT_ROOT / \"dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",
- " annotations_dir: str = str(PROJECT_ROOT / \"dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",
- " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt\") #path to MHR model\n",
+ " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",
+ " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n",
+ " # images_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",
+ " # annotations_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",
+ " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/mhr_model.pt\") #path to MHR model\n",
" log_dir: str = str(PROJECT_ROOT / \"runs/distill_repvit_cliff_v1\") # Bumped to v1!\n",
"\n",
+ " print(f\"Project root: {PROJECT_ROOT}\")\n",
+ " print(f\"Images dir: {images_dir}\")\n",
+ " print(f\"Annotations dir: {annotations_dir}\")\n",
+ " print(f\"MHR model path: {mhr_model_path}\")\n",
" # --- Data ---\n",
" image_size: int = 224\n",
" max_images: int = 100_000 \n",
@@ -1504,6 +1539,13 @@
" \n",
" return metrics\n",
"\n",
+ "import gc, torch\n",
+ "for _n in [\"overfit_model\", \"optimizer\", \"scaler\", \"overfit_batch\", \"overfit_batch_cpu\"]:\n",
+ " globals().pop(_n, None)\n",
+ "gc.collect(); torch.cuda.empty_cache()\n",
+ "print(f\"GPU allocated now: {torch.cuda.memory_allocated()/1e9:.2f} GB\") # expect ~1.5 GB (just mhr_module)\n",
+ "\n",
+ "\n",
"# # ==============================================================================\n",
"# # SANITY CHECK: Evaluating Your Overfit Model\n",
"# # ==============================================================================\n",
@@ -1534,7 +1576,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2ede5144",
+ "id": "d44aeb4a",
"metadata": {},
"outputs": [],
"source": [
diff --git a/tools/parquet_to_npz.py b/tools/parquet_to_npz.py
new file mode 100644
index 0000000..a8b45f2
--- /dev/null
+++ b/tools/parquet_to_npz.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""
+Convert the official ``facebook/sam-3d-body-dataset`` parquet annotations into the
+per-crop ``images/*.png`` + ``annotations/*.npz`` layout consumed by
+``notebooks/distill_transformer_decoder.ipynb``.
+
+This lets you train InstantHMR directly on the original SAM 3D Body ground-truth
+MHR fits, instead of (or alongside) the teacher-distilled pseudo-labels produced
+by ``tools/annotate_dataset.py``. The output ``.npz`` schema is exactly the one
+that script writes, so the training notebook needs **no changes**.
+
+Key facts established by auditing the parquet (see tools/README or docs):
+ * The parquet has no ``cam_trans`` column, but the SAM3D translation is fully
+ recoverable. The global translation is the first three MHR ``model_params``:
+
+ cam_trans = [ model_params[0], -model_params[1], -model_params[2] ] / 10
+
+ and the body-centred joints (distillation convention, vision Y-down) are
+
+ joints_3d = keypoints_3d[:, :3] * [1, -1, -1] # negate Y and Z
+
+ Together these reproject keypoints_3d onto the stored full-frame
+ keypoints_2d through ``cam_int`` at ~0 px (verified on coco_train).
+ * The dataset stores its own per-image focal length in ``cam_int`` (NOT the
+ sqrt(H^2+W^2) heuristic), with principal point at the image centre. We copy
+ the focal straight from ``cam_int`` so reprojection stays consistent.
+ * Persons with ``mhr_valid == False`` have unreliable fits and are skipped by
+ default (the 2D in-the-wild splits — coco/mpii/aic — rely on this flag).
+
+NPZ payload (per crop) — identical to tools/annotate_dataset.py:
+ orig_shape (2,) [H, W] of the full uncropped image
+ bbox (4,) [x1, y1, x2, y2] in original image coords (xyxy)
+ bbox_square (4,) [x1, y1, x2, y2] expanded square crop (1.2x)
+ cam_focal_length (2,) [fx, fy] taken from cam_int diagonal
+ cam_trans (3,) [tx, ty, tz] SAM3D translation, vision frame, metres
+ mhr_model_params (204,) MHR pose + scale params (== parquet model_params)
+ shape_params (45,) MHR identity blendshapes
+ joints_3d (70, 3) body-centred joints, vision Y-down (metres)
+ joints_2d (70, 2) full-frame pixel coords
+
+Usage:
+ # 1. Download annotations for a split (needs HF access to the gated repo):
+ # python data/scripts/download.py --save_dir $ANN_DIR --splits coco_train
+ # 2. Download the matching images (COCO2014 train/val/test).
+ # 3. Convert:
+ python tools/parquet_to_npz.py \\
+ --annotation_dir $ANN_DIR/coco_train \\
+ --image_dir $COCO_IMG_DIR \\
+ --output_dir /data/sam3d_gt_coco \\
+ --validate
+
+Resume-safe: existing ``annotations/<name>.npz`` files are skipped.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+CROP_SIZE = 224
+# Negate Y and Z to map MHR-native (Y-up) coordinates into the vision camera
+# frame (Y-down, Z-forward) used everywhere in the distillation pipeline.
+NEG_YZ = np.array([1.0, -1.0, -1.0], dtype=np.float64)
+# MHR root translation (model_params[:3]) is stored in decimetres.
+TRANS_SCALE = 10.0
+
+
+# ---------------------------------------------------------------------------
+# Crop geometry — kept byte-for-byte in sync with
+# tools/annotate_dataset.py:get_square_crop_padded so the 224 crops produced
+# here are pixel-identical to the distillation crops.
+# ---------------------------------------------------------------------------
+def get_square_crop_padded(image: np.ndarray, bbox: np.ndarray, expand: float = 1.2):
+ """Square crop around *bbox*, black-padded where it leaves the image."""
+ h, w = image.shape[:2]
+ x1, y1, x2, y2 = bbox.astype(float)
+
+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
+ size = max(x2 - x1, y2 - y1) * expand
+ half = size / 2.0
+
+ sq_x1, sq_y1 = int(cx - half), int(cy - half)
+ sq_x2, sq_y2 = int(cx + half), int(cy + half)
+
+ pad_top = max(0, -sq_y1)
+ pad_bottom = max(0, sq_y2 - h)
+ pad_left = max(0, -sq_x1)
+ pad_right = max(0, sq_x2 - w)
+
+ valid_x1, valid_y1 = max(0, sq_x1), max(0, sq_y1)
+ valid_x2, valid_y2 = min(w, sq_x2), min(h, sq_y2)
+ crop_valid = image[valid_y1:valid_y2, valid_x1:valid_x2]
+
+ crop_square = cv2.copyMakeBorder(
+ crop_valid, pad_top, pad_bottom, pad_left, pad_right,
+ cv2.BORDER_CONSTANT, value=(0, 0, 0),
+ )
+ sq_bbox = np.array([sq_x1, sq_y1, sq_x2, sq_y2], dtype=np.float32)
+ return crop_square, sq_bbox
+
+
+# ---------------------------------------------------------------------------
+# Parquet field helpers
+# ---------------------------------------------------------------------------
+def _arr(x) -> np.ndarray:
+ """Parquet cells come back as numpy object/ragged arrays; normalise them."""
+ if hasattr(x, "tolist"):
+ x = x.tolist()
+ return np.array(x)
+
+
+def image_relpath(dataset: str, image: str) -> str:
+ """Mirror data/scripts/create_webdataset.py:get_img_name for image lookup."""
+ if dataset == "coco":
+ # e.g. COCO_train2014_000000312652.jpg -> train2014/COCO_train2014_...jpg
+ _, split, _ = image.split("_")
+ return os.path.join(split, image)
+ if dataset == "mpii":
+ return os.path.join("images", image)
+ if dataset == "aic":
+ return os.path.join("train", "images", image)
+ # 3dpw / harmony4d / egoexo4d / egohumans / sa1b store a usable relative path
+ return image
+
+
+def to_xyxy(bbox: np.ndarray, fmt: str) -> np.ndarray:
+ bbox = bbox.astype(np.float32)
+ if fmt == "xyxy":
+ return bbox
+ if fmt == "xywh":
+ x, y, w, h = bbox
+ return np.array([x, y, x + w, y + h], dtype=np.float32)
+ raise ValueError(f"Unsupported bbox_format: {fmt}")
+
+
+def cam_trans_from_params(model_params: np.ndarray) -> np.ndarray:
+ """Recover SAM3D camera translation (vision frame, metres) from MHR params."""
+ t = model_params[:3].astype(np.float64) * NEG_YZ / TRANS_SCALE
+ return t.astype(np.float32)
+
+
+def reproj_error(joints_3d: np.ndarray, cam_trans: np.ndarray,
+ joints_2d: np.ndarray, focal: np.ndarray,
+ orig_shape: np.ndarray) -> float:
+ """RMS px error of (joints_3d + cam_trans) projected onto joints_2d.
+
+ Uses the exact camera model the notebook loss assumes: focal from cam_int,
+ principal point at the image centre.
+ """
+ h, w = float(orig_shape[0]), float(orig_shape[1])
+ cam = joints_3d + cam_trans[None, :]
+ z = np.clip(cam[:, 2], 1e-3, None)
+ u = focal[0] * cam[:, 0] / z + w / 2.0
+ v = focal[1] * cam[:, 1] / z + h / 2.0
+ d = np.stack([u - joints_2d[:, 0], v - joints_2d[:, 1]], axis=-1)
+ return float(np.sqrt((d ** 2).mean()))
+
+
+# ---------------------------------------------------------------------------
+# Per-person annotation builder
+# ---------------------------------------------------------------------------
+def build_annotation(row, img_h: int, img_w: int) -> dict:
+ model_params = _arr(row["model_params"]).astype(np.float32).ravel() # (204,)
+ shape_params = _arr(row["shape_params"]).astype(np.float32).ravel() # (45,)
+ cam_int = _arr(row["cam_int"]).astype(np.float32).reshape(3, 3)
+
+ kp3 = _arr(row["keypoints_3d"]).astype(np.float32) # (70, 4)
+ kp2 = _arr(row["keypoints_2d"]).astype(np.float32) # (70, 3)
+
+ joints_3d = (kp3[:, :3].astype(np.float64) * NEG_YZ).astype(np.float32) # vision frame
+ joints_2d = kp2[:, :2].astype(np.float32) # full-frame px
+ cam_trans = cam_trans_from_params(model_params) # from model_params[:3] BEFORE zeroing
+ focal = np.array([cam_int[0, 0], cam_int[1, 1]], dtype=np.float32)
+
+ # The dataset bakes the global translation into model_params[:3], so the MHR
+ # rig decodes an already-camera-positioned mesh. The distillation pipeline
+ # (notebook Cells 2 & 8) instead expects a *body-centred* mesh and adds
+ # cam_trans separately. Zero the translation here so the MHR mesh comes out
+ # body-centred — verified to match `joints_3d` to ~1e-7. Without this, the
+ # notebook double-counts cam_trans (loss_structural / loss_feet /
+ # loss_reproj_mhr and the MHR-param visualiser all go wrong).
+ model_params = model_params.copy()
+ model_params[:3] = 0.0
+
+ bbox = to_xyxy(_arr(row["bbox"]), row["bbox_format"])
+
+ return {
+ "orig_shape": np.array([img_h, img_w], dtype=np.int32),
+ "bbox": bbox,
+ "cam_focal_length": focal,
+ "cam_trans": cam_trans,
+ "mhr_model_params": model_params,
+ "shape_params": shape_params,
+ "joints_3d": joints_3d,
+ "joints_2d": joints_2d,
+ }
+
+
+def save_person(full_img_rgb, ann, name, images_dir, annotations_dir):
+ crop_square, sq_bbox = get_square_crop_padded(full_img_rgb, ann["bbox"], expand=1.2)
+ crop_224 = cv2.resize(crop_square, (CROP_SIZE, CROP_SIZE), interpolation=cv2.INTER_LINEAR)
+ cv2.imwrite(str(images_dir / f"{name}.png"), cv2.cvtColor(crop_224, cv2.COLOR_RGB2BGR))
+ ann = dict(ann)
+ ann["bbox_square"] = sq_bbox
+ np.savez(annotations_dir / f"{name}.npz", **ann)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+def parse_args() -> argparse.Namespace:
+ p = argparse.ArgumentParser(
+ description="Convert sam-3d-body-dataset parquet -> distillation .npz crops.",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=__doc__,
+ )
+ p.add_argument("--annotation_dir", required=True,
+ help="Directory of NNNNNN.parquet files for one split (e.g. .../coco_train).")
+ p.add_argument("--image_dir", required=True,
+ help="Root directory of the source dataset images.")
+ p.add_argument("--output_dir", required=True,
+ help="Output dir; images/ and annotations/ are created inside.")
+ p.add_argument("--max_samples", type=int, default=None,
+ help="Stop after producing N crops (counts existing on disk).")
+ p.add_argument("--keep_invalid", action="store_true",
+ help="Keep persons with mhr_valid == False (skipped by default).")
+ p.add_argument("--validate", action="store_true",
+ help="Assert reprojection error < 1px on each saved person.")
+ p.add_argument("--reproj_tol", type=float, default=1.0,
+ help="Max allowed RMS reprojection error in px when --validate (default 1.0).")
+ return p.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+ ann_dir = Path(args.annotation_dir)
+ img_dir = Path(args.image_dir)
+ out_dir = Path(args.output_dir)
+ images_dir = out_dir / "images"
+ annotations_dir = out_dir / "annotations"
+ images_dir.mkdir(parents=True, exist_ok=True)
+ annotations_dir.mkdir(parents=True, exist_ok=True)
+
+ parquets = sorted(ann_dir.glob("*.parquet"))
+ if not parquets:
+ print(f"No parquet files in {ann_dir}", file=sys.stderr)
+ sys.exit(1)
+
+ existing = {p.stem for p in annotations_dir.glob("*.npz")}
+ total = len(existing)
+ stats = {"crops": 0, "skipped_existing": 0, "skipped_invalid": 0,
+ "missing_image": 0, "failed": 0}
+
+ pbar = tqdm(parquets, desc="shards", unit="shard")
+ for pq in pbar:
+ df = pd.read_parquet(pq)
+ img_cache: dict[str, np.ndarray | None] = {}
+
+ for _, row in df.iterrows():
+ dataset = row["dataset"]
+ image = row["image"]
+ person_id = int(row["person_id"])
+ stem = Path(image).stem.replace(os.sep, "_").replace(" ", "_")
+ name = f"{dataset}_{stem}_p{person_id}"
+
+ if name in existing:
+ stats["skipped_existing"] += 1
+ continue
+ if not args.keep_invalid and not bool(row["mhr_valid"]):
+ stats["skipped_invalid"] += 1
+ continue
+
+ if image not in img_cache:
+ ipath = img_dir / image_relpath(dataset, image)
+ bgr = cv2.imread(str(ipath))
+ img_cache[image] = (
+ cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) if bgr is not None else None
+ )
+ img_rgb = img_cache[image]
+ if img_rgb is None:
+ stats["missing_image"] += 1
+ continue
+
+ h, w = img_rgb.shape[:2]
+ try:
+ ann = build_annotation(row, h, w)
+ if args.validate:
+ err = reproj_error(ann["joints_3d"], ann["cam_trans"],
+ ann["joints_2d"], ann["cam_focal_length"],
+ ann["orig_shape"])
+ if err > args.reproj_tol:
+ raise ValueError(f"reproj err {err:.2f}px > {args.reproj_tol}px")
+ save_person(img_rgb, ann, name, images_dir, annotations_dir)
+ except Exception as e: # noqa: BLE001
+ stats["failed"] += 1
+ tqdm.write(f"[fail] {name}: {e}")
+ continue
+
+ existing.add(name)
+ stats["crops"] += 1
+ total += 1
+ if args.max_samples and total >= args.max_samples:
+ pbar.close()
+ _report(stats, total, out_dir)
+ return
+
+ pbar.set_postfix(crops=stats["crops"], invalid=stats["skipped_invalid"],
+ missing=stats["missing_image"])
+
+ _report(stats, total, out_dir)
+
+
+def _report(stats, total, out_dir):
+ print("=" * 56)
+ print("CONVERSION COMPLETE")
+ for k, v in stats.items():
+ print(f" {k:18s}: {v}")
+ print(f" total on disk : {total}")
+ print(f" output : {out_dir}")
+ print("=" * 56)
+
+
+if __name__ == "__main__":
+ main()