Python
feat: added training from ground truth data. fixed out of memory issues
mohamdev0a78ed8baf73
diff --git a/.gitignore b/.gitignoreindex 72fb3f4..c643b2d 100644--- a/.gitignore+++ b/.gitignore@@ -30,6 +30,7 @@ rf-detr*.pthdata/annotations/images/+checkpoints/# Model weights — too large for git, use Git LFS or release artifactsmodels/*.onnxdiff --git a/README.md b/README.mdindex d4334a7..a4d6550 100644--- a/README.md+++ b/README.md@@ -148,6 +148,60 @@ The InstantHMR ONNX weights are hosted on HuggingFace:Download `instanthmr.onnx` and place it under `models/`, or pass any pathvia `--model`.+## Training++InstantHMR can be trained from two different label sources. **Both produce the+exact same `images/*.png` + `annotations/*.npz` layout** that+[`notebooks/distill_transformer_decoder.ipynb`](notebooks/distill_transformer_decoder.ipynb)+consumes, so the training notebook is identical either way:++1. **Distillation (default).** Run the SAM3D teacher over your own images with+ [`tools/annotate_dataset.py`](tools/annotate_dataset.py) to generate+ per-person pseudo-labels — see [`docs/annotation.md`](docs/annotation.md).+2. **Original ground-truth annotations.** Train directly on the released+ [`facebook/sam-3d-body-dataset`](https://huggingface.co/datasets/facebook/sam-3d-body-dataset)+ — the human MHR fits used to build SAM 3D Body — converted to the same+ `.npz` schema by [`tools/parquet_to_npz.py`](tools/parquet_to_npz.py).++### Train on the original annotations (COCO example)++The dataset is gated: first **request access** on the+[dataset page](https://huggingface.co/datasets/facebook/sam-3d-body-dataset)+and `huggingface-cli login`. The annotation parquets ship **without images**, so+you download the source images once per sub-dataset (COCO and MPII are the+easiest — a single `wget`; others may need extra access / undistortion, see the+[dataset setup guides](https://github.com/facebookresearch/sam-3d-body/tree/main/data)).++```bash+# 1. Annotations — download the coco_train split (uses Meta's sam-3d-body repo)+git clone https://github.com/facebookresearch/sam-3d-body+python sam-3d-body/data/scripts/download.py \+ --save_dir $ANN_DIR --splits coco_train++# 2. Images — COCO 2014 train (the split the parquet filenames reference)+mkdir -p $COCO_IMG_DIR && cd $COCO_IMG_DIR+wget http://images.cocodataset.org/zips/train2014.zip && unzip -q train2014.zip+cd -++# 3. Convert parquet -> per-crop images/ + annotations/ (.npz)+python tools/parquet_to_npz.py \+ --annotation_dir $ANN_DIR/coco_train \+ --image_dir $COCO_IMG_DIR \+ --output_dir data/sam3d_gt_coco \+ --validate++# 4. Point the notebook's config at the output and train:+# cfg.images_dir = "data/sam3d_gt_coco/images"+# cfg.annotations_dir = "data/sam3d_gt_coco/annotations"+```++The converter recovers `cam_trans` from the MHR params, derives the 1.2× square+crop, skips low-quality fits (`mhr_valid`), and writes the same keys+`tools/annotate_dataset.py` does. `--validate` asserts that each person's+`joints_3d + cam_trans` reprojects onto the stored `joints_2d` to < 1 px. Other+splits work the same way — swap `coco_train` for `mpii_train`,+`harmony4d_train`, etc., and point `--image_dir` at that dataset's images.+## Documentation- **[`docs/architecture.md`](docs/architecture.md)** — what the networkdiff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynbindex 2d43919..0b03bf5 100644--- a/notebooks/distill_transformer_decoder.ipynb+++ b/notebooks/distill_transformer_decoder.ipynb@@ -5,7 +5,23 @@"execution_count": null,"id": "ad5fd1be","metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']\n",+ "Using device: cuda\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "\u001b[0;93m2026-06-25 13:28:09.821708154 [W:onnxruntime:Default, device_discovery.cc:283 GetGpuDevices] Failed to detect devices under \"/sys/class/drm/card0\": device_discovery.cc:93 ReadFileContents Failed to open file: \"/sys/class/drm/card0/device/vendor\"\u001b[m\n"+ ]+ }+ ],"source": ["import torch\n","import torch.nn as nn\n",@@ -22,15 +38,28 @@"\n","\n","device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "import os; os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n","print(f\"Using device: {device}\")"]},{"cell_type": "code",- "execution_count": null,+ "execution_count": 2,"id": "7e2ae437","metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Project root: /home/madjel/Projects/gitpackages/sam3d-student-release\n",+ "Images dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/images\n",+ "Annotations dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/annotations\n",+ "MHR model path: /home/madjel/Projects/gitpackages/sam3d-student-release/checkpoints/mhr_model.pt\n",+ "Config loaded. Backbone: repvit_m2_3. Ready for V3 Training.\n"+ ]+ }+ ],"source": ["# ============================================================\n","# Cell 1 — Configuration\n",@@ -38,17 +67,23 @@"from dataclasses import dataclass \n","from pathlib import Path\n","\n",- "PROJECT_ROOT = Path(\"\") #set your path\n",+ "PROJECT_ROOT = Path(\"/home/madjel/Projects/gitpackages/sam3d-student-release/\") #set your path\n","\n","@dataclass\n","class DistillConfig:\n"," \"\"\"All hyperparameters in one place.\"\"\"\n"," # --- Paths ---\n",- " images_dir: str = str(PROJECT_ROOT / \"dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",- " annotations_dir: str = str(PROJECT_ROOT / \"dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",- " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt\") #path to MHR model\n",+ " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",+ " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n",+ " # images_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",+ " # annotations_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",+ " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/mhr_model.pt\") #path to MHR model\n"," log_dir: str = str(PROJECT_ROOT / \"runs/distill_repvit_cliff_v1\") # Bumped to v1!\n","\n",+ " print(f\"Project root: {PROJECT_ROOT}\")\n",+ " print(f\"Images dir: {images_dir}\")\n",+ " print(f\"Annotations dir: {annotations_dir}\")\n",+ " print(f\"MHR model path: {mhr_model_path}\")\n"," # --- Data ---\n"," image_size: int = 224\n"," max_images: int = 100_000 \n",@@ -1504,6 +1539,13 @@" \n"," return metrics\n","\n",+ "import gc, torch\n",+ "for _n in [\"overfit_model\", \"optimizer\", \"scaler\", \"overfit_batch\", \"overfit_batch_cpu\"]:\n",+ " globals().pop(_n, None)\n",+ "gc.collect(); torch.cuda.empty_cache()\n",+ "print(f\"GPU allocated now: {torch.cuda.memory_allocated()/1e9:.2f} GB\") # expect ~1.5 GB (just mhr_module)\n",+ "\n",+ "\n","# # ==============================================================================\n","# # SANITY CHECK: Evaluating Your Overfit Model\n","# # ==============================================================================\n",@@ -1534,7 +1576,7 @@{"cell_type": "code","execution_count": null,- "id": "2ede5144",+ "id": "d44aeb4a","metadata": {},"outputs": [],"source": [diff --git a/tools/parquet_to_npz.py b/tools/parquet_to_npz.pynew file mode 100644index 0000000..a8b45f2--- /dev/null+++ b/tools/parquet_to_npz.py@@ -0,0 +1,331 @@+#!/usr/bin/env python3+"""+Convert the official ``facebook/sam-3d-body-dataset`` parquet annotations into the+per-crop ``images/*.png`` + ``annotations/*.npz`` layout consumed by+``notebooks/distill_transformer_decoder.ipynb``.++This lets you train InstantHMR directly on the original SAM 3D Body ground-truth+MHR fits, instead of (or alongside) the teacher-distilled pseudo-labels produced+by ``tools/annotate_dataset.py``. The output ``.npz`` schema is exactly the one+that script writes, so the training notebook needs **no changes**.++Key facts established by auditing the parquet (see tools/README or docs):+ * The parquet has no ``cam_trans`` column, but the SAM3D translation is fully+ recoverable. The global translation is the first three MHR ``model_params``:++ cam_trans = [ model_params[0], -model_params[1], -model_params[2] ] / 10++ and the body-centred joints (distillation convention, vision Y-down) are++ joints_3d = keypoints_3d[:, :3] * [1, -1, -1] # negate Y and Z++ Together these reproject keypoints_3d onto the stored full-frame+ keypoints_2d through ``cam_int`` at ~0 px (verified on coco_train).+ * The dataset stores its own per-image focal length in ``cam_int`` (NOT the+ sqrt(H^2+W^2) heuristic), with principal point at the image centre. We copy+ the focal straight from ``cam_int`` so reprojection stays consistent.+ * Persons with ``mhr_valid == False`` have unreliable fits and are skipped by+ default (the 2D in-the-wild splits — coco/mpii/aic — rely on this flag).++NPZ payload (per crop) — identical to tools/annotate_dataset.py:+ orig_shape (2,) [H, W] of the full uncropped image+ bbox (4,) [x1, y1, x2, y2] in original image coords (xyxy)+ bbox_square (4,) [x1, y1, x2, y2] expanded square crop (1.2x)+ cam_focal_length (2,) [fx, fy] taken from cam_int diagonal+ cam_trans (3,) [tx, ty, tz] SAM3D translation, vision frame, metres+ mhr_model_params (204,) MHR pose + scale params (== parquet model_params)+ shape_params (45,) MHR identity blendshapes+ joints_3d (70, 3) body-centred joints, vision Y-down (metres)+ joints_2d (70, 2) full-frame pixel coords++Usage:+ # 1. Download annotations for a split (needs HF access to the gated repo):+ # python data/scripts/download.py --save_dir $ANN_DIR --splits coco_train+ # 2. Download the matching images (COCO2014 train/val/test).+ # 3. Convert:+ python tools/parquet_to_npz.py \\+ --annotation_dir $ANN_DIR/coco_train \\+ --image_dir $COCO_IMG_DIR \\+ --output_dir /data/sam3d_gt_coco \\+ --validate++Resume-safe: existing ``annotations/<name>.npz`` files are skipped.+"""++from __future__ import annotations++import argparse+import os+import sys+from pathlib import Path++import cv2+import numpy as np+import pandas as pd+from tqdm import tqdm++CROP_SIZE = 224+# Negate Y and Z to map MHR-native (Y-up) coordinates into the vision camera+# frame (Y-down, Z-forward) used everywhere in the distillation pipeline.+NEG_YZ = np.array([1.0, -1.0, -1.0], dtype=np.float64)+# MHR root translation (model_params[:3]) is stored in decimetres.+TRANS_SCALE = 10.0+++# ---------------------------------------------------------------------------+# Crop geometry — kept byte-for-byte in sync with+# tools/annotate_dataset.py:get_square_crop_padded so the 224 crops produced+# here are pixel-identical to the distillation crops.+# ---------------------------------------------------------------------------+def get_square_crop_padded(image: np.ndarray, bbox: np.ndarray, expand: float = 1.2):+ """Square crop around *bbox*, black-padded where it leaves the image."""+ h, w = image.shape[:2]+ x1, y1, x2, y2 = bbox.astype(float)++ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0+ size = max(x2 - x1, y2 - y1) * expand+ half = size / 2.0++ sq_x1, sq_y1 = int(cx - half), int(cy - half)+ sq_x2, sq_y2 = int(cx + half), int(cy + half)++ pad_top = max(0, -sq_y1)+ pad_bottom = max(0, sq_y2 - h)+ pad_left = max(0, -sq_x1)+ pad_right = max(0, sq_x2 - w)++ valid_x1, valid_y1 = max(0, sq_x1), max(0, sq_y1)+ valid_x2, valid_y2 = min(w, sq_x2), min(h, sq_y2)+ crop_valid = image[valid_y1:valid_y2, valid_x1:valid_x2]++ crop_square = cv2.copyMakeBorder(+ crop_valid, pad_top, pad_bottom, pad_left, pad_right,+ cv2.BORDER_CONSTANT, value=(0, 0, 0),+ )+ sq_bbox = np.array([sq_x1, sq_y1, sq_x2, sq_y2], dtype=np.float32)+ return crop_square, sq_bbox+++# ---------------------------------------------------------------------------+# Parquet field helpers+# ---------------------------------------------------------------------------+def _arr(x) -> np.ndarray:+ """Parquet cells come back as numpy object/ragged arrays; normalise them."""+ if hasattr(x, "tolist"):+ x = x.tolist()+ return np.array(x)+++def image_relpath(dataset: str, image: str) -> str:+ """Mirror data/scripts/create_webdataset.py:get_img_name for image lookup."""+ if dataset == "coco":+ # e.g. COCO_train2014_000000312652.jpg -> train2014/COCO_train2014_...jpg+ _, split, _ = image.split("_")+ return os.path.join(split, image)+ if dataset == "mpii":+ return os.path.join("images", image)+ if dataset == "aic":+ return os.path.join("train", "images", image)+ # 3dpw / harmony4d / egoexo4d / egohumans / sa1b store a usable relative path+ return image+++def to_xyxy(bbox: np.ndarray, fmt: str) -> np.ndarray:+ bbox = bbox.astype(np.float32)+ if fmt == "xyxy":+ return bbox+ if fmt == "xywh":+ x, y, w, h = bbox+ return np.array([x, y, x + w, y + h], dtype=np.float32)+ raise ValueError(f"Unsupported bbox_format: {fmt}")+++def cam_trans_from_params(model_params: np.ndarray) -> np.ndarray:+ """Recover SAM3D camera translation (vision frame, metres) from MHR params."""+ t = model_params[:3].astype(np.float64) * NEG_YZ / TRANS_SCALE+ return t.astype(np.float32)+++def reproj_error(joints_3d: np.ndarray, cam_trans: np.ndarray,+ joints_2d: np.ndarray, focal: np.ndarray,+ orig_shape: np.ndarray) -> float:+ """RMS px error of (joints_3d + cam_trans) projected onto joints_2d.++ Uses the exact camera model the notebook loss assumes: focal from cam_int,+ principal point at the image centre.+ """+ h, w = float(orig_shape[0]), float(orig_shape[1])+ cam = joints_3d + cam_trans[None, :]+ z = np.clip(cam[:, 2], 1e-3, None)+ u = focal[0] * cam[:, 0] / z + w / 2.0+ v = focal[1] * cam[:, 1] / z + h / 2.0+ d = np.stack([u - joints_2d[:, 0], v - joints_2d[:, 1]], axis=-1)+ return float(np.sqrt((d ** 2).mean()))+++# ---------------------------------------------------------------------------+# Per-person annotation builder+# ---------------------------------------------------------------------------+def build_annotation(row, img_h: int, img_w: int) -> dict:+ model_params = _arr(row["model_params"]).astype(np.float32).ravel() # (204,)+ shape_params = _arr(row["shape_params"]).astype(np.float32).ravel() # (45,)+ cam_int = _arr(row["cam_int"]).astype(np.float32).reshape(3, 3)++ kp3 = _arr(row["keypoints_3d"]).astype(np.float32) # (70, 4)+ kp2 = _arr(row["keypoints_2d"]).astype(np.float32) # (70, 3)++ joints_3d = (kp3[:, :3].astype(np.float64) * NEG_YZ).astype(np.float32) # vision frame+ joints_2d = kp2[:, :2].astype(np.float32) # full-frame px+ cam_trans = cam_trans_from_params(model_params) # from model_params[:3] BEFORE zeroing+ focal = np.array([cam_int[0, 0], cam_int[1, 1]], dtype=np.float32)++ # The dataset bakes the global translation into model_params[:3], so the MHR+ # rig decodes an already-camera-positioned mesh. The distillation pipeline+ # (notebook Cells 2 & 8) instead expects a *body-centred* mesh and adds+ # cam_trans separately. Zero the translation here so the MHR mesh comes out+ # body-centred — verified to match `joints_3d` to ~1e-7. Without this, the+ # notebook double-counts cam_trans (loss_structural / loss_feet /+ # loss_reproj_mhr and the MHR-param visualiser all go wrong).+ model_params = model_params.copy()+ model_params[:3] = 0.0++ bbox = to_xyxy(_arr(row["bbox"]), row["bbox_format"])++ return {+ "orig_shape": np.array([img_h, img_w], dtype=np.int32),+ "bbox": bbox,+ "cam_focal_length": focal,+ "cam_trans": cam_trans,+ "mhr_model_params": model_params,+ "shape_params": shape_params,+ "joints_3d": joints_3d,+ "joints_2d": joints_2d,+ }+++def save_person(full_img_rgb, ann, name, images_dir, annotations_dir):+ crop_square, sq_bbox = get_square_crop_padded(full_img_rgb, ann["bbox"], expand=1.2)+ crop_224 = cv2.resize(crop_square, (CROP_SIZE, CROP_SIZE), interpolation=cv2.INTER_LINEAR)+ cv2.imwrite(str(images_dir / f"{name}.png"), cv2.cvtColor(crop_224, cv2.COLOR_RGB2BGR))+ ann = dict(ann)+ ann["bbox_square"] = sq_bbox+ np.savez(annotations_dir / f"{name}.npz", **ann)+++# ---------------------------------------------------------------------------+# Main+# ---------------------------------------------------------------------------+def parse_args() -> argparse.Namespace:+ p = argparse.ArgumentParser(+ description="Convert sam-3d-body-dataset parquet -> distillation .npz crops.",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog=__doc__,+ )+ p.add_argument("--annotation_dir", required=True,+ help="Directory of NNNNNN.parquet files for one split (e.g. .../coco_train).")+ p.add_argument("--image_dir", required=True,+ help="Root directory of the source dataset images.")+ p.add_argument("--output_dir", required=True,+ help="Output dir; images/ and annotations/ are created inside.")+ p.add_argument("--max_samples", type=int, default=None,+ help="Stop after producing N crops (counts existing on disk).")+ p.add_argument("--keep_invalid", action="store_true",+ help="Keep persons with mhr_valid == False (skipped by default).")+ p.add_argument("--validate", action="store_true",+ help="Assert reprojection error < 1px on each saved person.")+ p.add_argument("--reproj_tol", type=float, default=1.0,+ help="Max allowed RMS reprojection error in px when --validate (default 1.0).")+ return p.parse_args()+++def main() -> None:+ args = parse_args()+ ann_dir = Path(args.annotation_dir)+ img_dir = Path(args.image_dir)+ out_dir = Path(args.output_dir)+ images_dir = out_dir / "images"+ annotations_dir = out_dir / "annotations"+ images_dir.mkdir(parents=True, exist_ok=True)+ annotations_dir.mkdir(parents=True, exist_ok=True)++ parquets = sorted(ann_dir.glob("*.parquet"))+ if not parquets:+ print(f"No parquet files in {ann_dir}", file=sys.stderr)+ sys.exit(1)++ existing = {p.stem for p in annotations_dir.glob("*.npz")}+ total = len(existing)+ stats = {"crops": 0, "skipped_existing": 0, "skipped_invalid": 0,+ "missing_image": 0, "failed": 0}++ pbar = tqdm(parquets, desc="shards", unit="shard")+ for pq in pbar:+ df = pd.read_parquet(pq)+ img_cache: dict[str, np.ndarray | None] = {}++ for _, row in df.iterrows():+ dataset = row["dataset"]+ image = row["image"]+ person_id = int(row["person_id"])+ stem = Path(image).stem.replace(os.sep, "_").replace(" ", "_")+ name = f"{dataset}_{stem}_p{person_id}"++ if name in existing:+ stats["skipped_existing"] += 1+ continue+ if not args.keep_invalid and not bool(row["mhr_valid"]):+ stats["skipped_invalid"] += 1+ continue++ if image not in img_cache:+ ipath = img_dir / image_relpath(dataset, image)+ bgr = cv2.imread(str(ipath))+ img_cache[image] = (+ cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) if bgr is not None else None+ )+ img_rgb = img_cache[image]+ if img_rgb is None:+ stats["missing_image"] += 1+ continue++ h, w = img_rgb.shape[:2]+ try:+ ann = build_annotation(row, h, w)+ if args.validate:+ err = reproj_error(ann["joints_3d"], ann["cam_trans"],+ ann["joints_2d"], ann["cam_focal_length"],+ ann["orig_shape"])+ if err > args.reproj_tol:+ raise ValueError(f"reproj err {err:.2f}px > {args.reproj_tol}px")+ save_person(img_rgb, ann, name, images_dir, annotations_dir)+ except Exception as e: # noqa: BLE001+ stats["failed"] += 1+ tqdm.write(f"[fail] {name}: {e}")+ continue++ existing.add(name)+ stats["crops"] += 1+ total += 1+ if args.max_samples and total >= args.max_samples:+ pbar.close()+ _report(stats, total, out_dir)+ return++ pbar.set_postfix(crops=stats["crops"], invalid=stats["skipped_invalid"],+ missing=stats["missing_image"])++ _report(stats, total, out_dir)+++def _report(stats, total, out_dir):+ print("=" * 56)+ print("CONVERSION COMPLETE")+ for k, v in stats.items():+ print(f" {k:18s}: {v}")+ print(f" total on disk : {total}")+ print(f" output : {out_dir}")+ print("=" * 56)+++if __name__ == "__main__":+ main()