Python
first commit
mohamdevec1bc434458e
diff --git a/.gitignore b/.gitignorenew file mode 100644index 0000000..39052df--- /dev/null+++ b/.gitignore@@ -0,0 +1,40 @@+# Byte-compiled / cache+__pycache__/+*.py[cod]+*$py.class+.pytest_cache/+.mypy_cache/+.ruff_cache/++# Editor / OS+.vscode/+.idea/+.DS_Store++# Virtual environments+.venv/+venv/+env/++# RF-DETR auto-downloaded weights+rf-detr*.pth+*.pth.lock+*.onnx+*.pth++# Rerun recordings+*.rrd++# Annotation outputs+data/+annotations/+images/++# Model weights — too large for git, use Git LFS or release artifacts+models/*.onnx+models/*.pt+models/*.pth+!models/README.md++# Notebook checkpoints+.ipynb_checkpoints/diff --git a/LICENSE b/LICENSEnew file mode 100644index 0000000..0b55932--- /dev/null+++ b/LICENSE@@ -0,0 +1,216 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for describing the origin of the Work and+ reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may accept and charge a+ fee for, the acceptance of support, warranty, indemnity, or other+ liability obligations and/or rights consistent with this License.+ However, in accepting such obligations, You may act only on Your+ own behalf and on Your sole responsibility, not on behalf of any+ other Contributor, and only if You agree to indemnify, defend, and+ hold each Contributor harmless for any liability incurred by, or+ claims asserted against, such Contributor by reason of your accepting+ any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright 2026 InstantHMR authors++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied. See the License for the specific language governing permissions+ and limitations under the License.++================================================================================++THIRD-PARTY NOTICES+-------------------++The InstantHMR model weights distributed at+https://huggingface.co/momolesang/InstantHMR are released under the SAM+license, since the model is a distillation of Meta's SAM 3D Body+(https://github.com/facebookresearch/sam-3d-body). Refer to that+repository for the full SAM license text and applicable use restrictions.++The demo uses Roboflow's RF-DETR for person detection, which is+distributed under its own license. See https://github.com/roboflow/rf-detr+for terms.diff --git a/README.md b/README.mdnew file mode 100644index 0000000..5b8dafb--- /dev/null+++ b/README.md@@ -0,0 +1,114 @@+# InstantHMR++A lightweight, ONNX-exportable distillation of+[`facebook/sam-3d-body-dinov3`](https://huggingface.co/facebook/sam-3d-body-dinov3)+for **3D human pose estimation and mesh recovery**: a RepViT-M1.5 backbone + a 9-token+cross-attention decoder + CLIFF camera conditioning. Trained to mimic the+SAM3D teacher's per-person 70-keypoint outputs from a single 224×224 crop.++InstantHMR ships as a single `.onnx` file. The demo pipeline pairs it with+**RF-DETR** for person detection — both stages are timed independently so+you always know where the latency is going.++- **Inputs:** `image (N, 3, 224, 224)`, `cliff_cond (N, 3)`.+- **Outputs:** `mhr_params (204)`, `shape_params (45)`, `cam_trans (3)`,+ `joints_2d (70, 2)` (normalised crop coords),+ `joints_3d (70, 3)` (body-centred metres, Y-down).+- **Speed:** ~5 ms / frame (~200 FPS) on a single RTX 4070 with the fp16+ ONNX (CUDA EP); CPU works too (~25 FPS, depending on hardware).++> **Note on the body mesh.** InstantHMR does **not** run the MHR mesh+> decoder — rendering the full body mesh is expensive and we want the+> default pipeline to stay lightweight. The raw `mhr_params` and+> `shape_params` are still exposed on every `HMRPrediction` so you can+> feed them into Meta's MHR TorchScript module from the original+> [SAM 3D Body release](https://github.com/facebookresearch/sam-3d-body)+> if you need a mesh.++## Install++```bash+git clone <this-repo> instanthmr+cd instanthmr++conda create -n instanthmr python=3.11 -y+conda activate instanthmr++pip install -r requirements.txt+```++The first run downloads the RF-DETR (medium) checkpoint into the rfdetr+cache directory automatically. The InstantHMR ONNX weights are released+separately on HuggingFace — see [Model weights](#model-weights) below.++## Run the demo++```bash+# Single image+python demo.py --image path/to/photo.jpg++# Video file+python demo.py --video path/to/clip.mp4++# Live webcam (index 0)+python demo.py --camera 0+```++`python demo.py --help` lists every flag (model path, device, detector+variant, confidence, max persons, frame skip, `.rrd` recording, …).++The demo opens a Rerun viewer with the source image + 2D skeleton, a live+detector / HMR / total-latency plot, and the 3D scene with the predicted+camera frustum.++## Use it from Python++```python+from instanthmr import PosePipeline++pipeline = PosePipeline(+ onnx_path="models/instanthmr.onnx",+ device="cuda",+ detector_variant="medium",+)++result = pipeline.predict(image_rgb)+for r in result.persons:+ print(r.joints_3d_cam.shape) # (70, 3)+ print(r.joints_2d.shape) # (70, 2)+ print(r.mhr_params.shape) # (204,) — for downstream MHR mesh decoding+ print(r.shape_params.shape) # (45,)+```++The 70 keypoints follow the MHR70 ordering — see+[`instanthmr/skeleton.py`](instanthmr/skeleton.py) for joint names and the+canonical skeleton edge list.++## Model weights++The InstantHMR ONNX weights are hosted on HuggingFace:++> **<https://huggingface.co/momolesang/InstantHMR>**++Download `instanthmr.onnx` and place it under `models/`, or pass any path+via `--model`.++## Documentation++- **[`docs/architecture.md`](docs/architecture.md)** — what the network+ looks like (RepViT backbone, 9-query decoder, CLIFF condition).+- **[`docs/annotation.md`](docs/annotation.md)** — how the training data+ was generated with the SAM3D teacher.+- **[`docs/training.md`](docs/training.md)** — how the released checkpoint+ was distilled and exported to ONNX.++## License++The code in this repository is released under the **Apache License 2.0**+(see [LICENSE](LICENSE)).++The model weights distributed at+<https://huggingface.co/momolesang/InstantHMR> are released under the+[SAM license](https://github.com/facebookresearch/sam-3d-body), since+InstantHMR is a distillation of `facebook/sam-3d-body-dinov3`. Please+review the SAM and RF-DETR licenses before downstream use.diff --git a/demo.py b/demo.pynew file mode 100755index 0000000..6b91441--- /dev/null+++ b/demo.py@@ -0,0 +1,307 @@+#!/usr/bin/env python3+"""InstantHMR — demo runner.++Detects persons in a video / image / live camera stream with RF-DETR, runs+InstantHMR on each, and visualises the results in Rerun: 3D joints, camera+pose as a pinhole frustum, the source image with the projected 2D skeleton+drawn on top, and a live latency plot showing RF-DETR vs InstantHMR cost+separately.++Usage+-----+ # Single image+ python demo.py --image path/to/photo.jpg++ # Video file+ python demo.py --video path/to/clip.mp4++ # Live camera (index 0)+ python demo.py --camera 0+"""++from __future__ import annotations++import argparse+import sys+from pathlib import Path++import cv2+import numpy as np++from instanthmr import PosePipeline+from instanthmr.visualizer import RerunVisualizer+++DEFAULT_MODEL = "models/instanthmr.onnx"+++def parse_args() -> argparse.Namespace:+ p = argparse.ArgumentParser(+ description="Run InstantHMR (RF-DETR + ONNX HMR) on a video, image, or camera.",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )++ src = p.add_mutually_exclusive_group(required=True)+ src.add_argument("--video", type=str, help="Path to an input video file.")+ src.add_argument("--image", type=str, help="Path to an input image.")+ src.add_argument(+ "--camera", type=int, metavar="IDX",+ help="Live camera index (0, 1, ...).",+ )++ p.add_argument(+ "--model", type=str, default=DEFAULT_MODEL,+ help=f"Path to the InstantHMR ONNX file (default: {DEFAULT_MODEL}).",+ )+ p.add_argument(+ "--device", type=str, default="cuda",+ help="Inference device for InstantHMR: 'cuda' or 'cpu' (default: cuda).",+ )+ p.add_argument(+ "--detector-variant", type=str, default="medium",+ choices=["nano", "small", "medium", "base", "large"],+ help="RF-DETR size variant (default: medium).",+ )+ p.add_argument(+ "--det-confidence", type=float, default=0.5,+ help="RF-DETR detection confidence threshold (default: 0.5).",+ )+ p.add_argument(+ "--max-persons", type=int, default=5,+ help="Max persons processed per frame (default: 5).",+ )+ p.add_argument(+ "--frame-skip", type=int, default=1,+ help="Process every Nth video frame (default: 1).",+ )+ p.add_argument(+ "--save-rrd", type=str, default=None,+ help="Optional path to save the Rerun recording as a .rrd file.",+ )+ p.add_argument(+ "--no-spawn", action="store_true",+ help="Don't spawn the Rerun viewer GUI (useful when only saving .rrd).",+ )++ return p.parse_args()+++def build_pipeline(args: argparse.Namespace) -> PosePipeline:+ model_path = Path(args.model)+ if not model_path.exists():+ sys.stderr.write(+ f"\n[error] InstantHMR ONNX not found: {model_path}\n"+ " Place 'instanthmr.onnx' under models/, or pass\n"+ " --model /path/to/instanthmr.onnx\n\n"+ )+ sys.exit(1)++ return PosePipeline(+ onnx_path=model_path,+ device=args.device,+ detector_variant=args.detector_variant,+ det_confidence=args.det_confidence,+ max_persons=args.max_persons,+ )+++def build_visualizer(args: argparse.Namespace) -> RerunVisualizer:+ return RerunVisualizer(+ application_id="instanthmr_demo",+ spawn_viewer=not args.no_spawn,+ save_path=args.save_rrd,+ )+++def _print_timings(+ prefix: str,+ detector_ms: float,+ hmr_ms: float,+ total_ms: float,+ n_persons: int,+) -> None:+ fps = 1000.0 / total_ms if total_ms > 0 else 0.0+ print(+ f"{prefix} "+ f"persons={n_persons:>2d} "+ f"RF-DETR={detector_ms:6.1f} ms "+ f"InstantHMR={hmr_ms:6.1f} ms "+ f"total={total_ms:6.1f} ms ({fps:5.1f} fps)"+ )+++# ---------------------------------------------------------------------------+# Source-specific runners+# ---------------------------------------------------------------------------+++def run_image(args: argparse.Namespace) -> None:+ image_path = Path(args.image)+ if not image_path.exists():+ sys.exit(f"[error] image not found: {image_path}")++ bgr = cv2.imread(str(image_path))+ if bgr is None:+ sys.exit(f"[error] cv2.imread failed for {image_path}")+ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)++ pipeline = build_pipeline(args)+ viz = build_visualizer(args)++ # Warm-up: first call includes lazy CUDA / ORT initialisation, so a real+ # latency reading is much more useful from the second call onwards.+ _ = pipeline.predict(rgb)+ result = pipeline.predict(rgb)++ viz.log_frame(+ rgb, result.persons,+ frame_idx=0, timestamp=0.0,+ detector_ms=result.detector_ms,+ hmr_ms=result.hmr_ms,+ total_ms=result.total_ms,+ )+ _print_timings("image:", result.detector_ms, result.hmr_ms, result.total_ms, len(result.persons))+++def run_video(args: argparse.Namespace) -> None:+ video_path = Path(args.video)+ if not video_path.exists():+ sys.exit(f"[error] video not found: {video_path}")++ cap = cv2.VideoCapture(str(video_path))+ if not cap.isOpened():+ sys.exit(f"[error] cv2.VideoCapture failed for {video_path}")++ fps = cap.get(cv2.CAP_PROP_FPS) or 30.0+ total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ print(f"video: {video_path.name}, {total} frames @ {fps:.1f} fps")++ pipeline = build_pipeline(args)+ viz = build_visualizer(args)++ frame_idx = 0+ sent = 0+ det_times: list[float] = []+ hmr_times: list[float] = []+ tot_times: list[float] = []+ try:+ while True:+ ret, bgr = cap.read()+ if not ret:+ break++ if frame_idx % args.frame_skip != 0:+ frame_idx += 1+ continue++ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)++ result = pipeline.predict(rgb)+ det_times.append(result.detector_ms)+ hmr_times.append(result.hmr_ms)+ tot_times.append(result.total_ms)++ viz.log_frame(+ rgb, result.persons,+ frame_idx=frame_idx,+ timestamp=frame_idx / fps,+ detector_ms=result.detector_ms,+ hmr_ms=result.hmr_ms,+ total_ms=result.total_ms,+ )++ sent += 1+ frame_idx += 1++ if sent % 30 == 0:+ _print_timings(+ f" frame {frame_idx:>5d}/{total:<5d}",+ float(np.mean(det_times[-30:])),+ float(np.mean(hmr_times[-30:])),+ float(np.mean(tot_times[-30:])),+ len(result.persons),+ )+ except KeyboardInterrupt:+ print("\ninterrupted")+ finally:+ cap.release()++ if tot_times:+ # Skip the first frame from the average — it bears the warm-up cost.+ warm = slice(1, None) if len(tot_times) > 1 else slice(None)+ print(+ "\nsummary "+ f"frames={sent:d} "+ f"RF-DETR={float(np.mean(det_times[warm])):.1f} ms "+ f"InstantHMR={float(np.mean(hmr_times[warm])):.1f} ms "+ f"total={float(np.mean(tot_times[warm])):.1f} ms "+ f"({1000.0 / float(np.mean(tot_times[warm])):.1f} fps)"+ )+++def run_camera(args: argparse.Namespace) -> None:+ cap = cv2.VideoCapture(args.camera)+ if not cap.isOpened():+ sys.exit(f"[error] cannot open camera index {args.camera}")++ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))+ fps = cap.get(cv2.CAP_PROP_FPS) or 30.0+ print(f"camera {args.camera}: {width}x{height} @ {fps:.1f} fps — Ctrl+C to stop")++ pipeline = build_pipeline(args)+ viz = build_visualizer(args)++ frame_idx = 0+ det_times: list[float] = []+ hmr_times: list[float] = []+ tot_times: list[float] = []+ try:+ while True:+ ret, bgr = cap.read()+ if not ret:+ print("[warn] camera read failed")+ break++ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)+ result = pipeline.predict(rgb)+ det_times.append(result.detector_ms)+ hmr_times.append(result.hmr_ms)+ tot_times.append(result.total_ms)++ viz.log_frame(+ rgb, result.persons,+ frame_idx=frame_idx,+ timestamp=frame_idx / fps,+ detector_ms=result.detector_ms,+ hmr_ms=result.hmr_ms,+ total_ms=result.total_ms,+ )++ if frame_idx and frame_idx % 30 == 0:+ _print_timings(+ f" frame {frame_idx:>5d}",+ float(np.mean(det_times[-30:])),+ float(np.mean(hmr_times[-30:])),+ float(np.mean(tot_times[-30:])),+ len(result.persons),+ )+ frame_idx += 1+ except KeyboardInterrupt:+ print("\nstopped")+ finally:+ cap.release()+++def main() -> None:+ args = parse_args()+ if args.image is not None:+ run_image(args)+ elif args.video is not None:+ run_video(args)+ else:+ run_camera(args)+++if __name__ == "__main__":+ main()diff --git a/docs/annotation.md b/docs/annotation.mdnew file mode 100644index 0000000..7496f5b--- /dev/null+++ b/docs/annotation.md@@ -0,0 +1,109 @@+# Building the training dataset++InstantHMR is supervised entirely by the **SAM3D teacher**+(`facebook/sam-3d-body-dinov3`): for every detector-found person we run+the teacher and store its outputs alongside a tight 224×224 crop. The+training loop in `notebooks/distill_transformer_decoder.ipynb` then learns+to mimic those teacher outputs from the crop alone.++The bundled `tools/annotate_dataset.py` uses YOLO for the+detector during annotation. That choice is independent of the runtime+detector — InstantHMR sees the same 1.2× square crop either way, and the+demo uses RF-DETR at inference time for better recall. If you re-run the+annotation with a different detector, just preserve the crop geometry.++The annotation pipeline is `tools/annotate_dataset.py`.++## What it does++```+root_dir/ output_dir/+├── COCO/ ─┐ ├── images/+├── Harmony4D/ │ YOLO + SAM3D │ COCO_<stem>_p0.png (224×224)+├── Synthetic/ │ ──────────────► │ COCO_<stem>_p0.npz+└── … ─┘ │ …+ └── annotations/+ COCO_<stem>_p0.npz+ …+```++For each detected person (capped at `--max_persons` per image):++1. **YOLO 11n** detects persons (`conf ≥ --confidence`, default 0.7).+2. The **full-resolution** image is passed to SAM3D with the raw bbox.+ SAM3D's CLIFF camera head sees the full frame, so `pred_cam_t` and+ `focal_length` are produced in real perspective coordinates — no manual+ camera math.+3. A 1.2× **square** crop around the bbox is resized to 224×224 and stored+ as a PNG.+4. The teacher outputs are stored as a `.npz` next to the image:++ | Key | Shape | Description |+ |-----|-------|-------------|+ | `orig_shape` | `(2,)` | `[H, W]` of the original frame |+ | `bbox` | `(4,)` | raw YOLO `[x1, y1, x2, y2]` |+ | `bbox_square` | `(4,)` | the 1.2× expanded square crop bounds |+ | `cam_focal_length` | `(2,)` | `[fx, fy]` (≈ √(H²+W²)) |+ | `cam_trans` | `(3,)` | SAM3D translation in camera frame |+ | `mhr_model_params` | `(204,)` | MHR pose + scale |+ | `shape_params` | `(45,)` | MHR identity blendshapes |+ | `joints_3d` | `(70, 3)` | SAM3D `pred_keypoints_3d` |+ | `joints_2d` | `(70, 2)` | SAM3D `pred_keypoints_2d`, in **full-frame** pixels |++The script is **resume-safe**: it scans existing `.npz` filenames at startup+and skips any image whose first crop is already on disk.++## Running it++```bash+# Setup: clone Meta's sam-3d-body and download the dinov3 teacher checkpoint.+git clone https://github.com/facebookresearch/sam-3d-body+huggingface-cli download facebook/sam-3d-body-dinov3 \+ --local-dir checkpoints/sam-3d-body-dinov3++# Annotate a tree of images+python tools/annotate_dataset.py \+ --root_dir /data/training_images \+ --output_dir /data/cliff_annotations \+ --confidence 0.7++# Quick balanced subset (50k crops, shuffled across datasets)+python tools/annotate_dataset.py \+ --root_dir /data/training_images \+ --output_dir /data/cliff_annotations \+ --shuffle --max_samples 50000++# Visually inspect what's getting saved+python tools/annotate_dataset.py \+ --root_dir /data/training_images \+ --output_dir /data/cliff_annotations \+ --use_rerun+```++Useful options:++- `--shuffle` — process images in random order so a partial run still spans+ the full dataset diversity.+- `--max_samples N` — stop after N **crops** are saved (not images), counting+ whatever was already on disk.+- `--timeout 5.0` — per-image SIGALRM; rare GPU hangs don't kill the run.+- `--gc_interval 200` — periodic CUDA cache + GC reset to keep VRAM stable+ across long runs.++## Why a separate cloud script?++This is the same per-person preprocessing the demo uses at inference time+(`InstantHMR._preprocess`). Aligning the training data with the runtime+preprocessing was the single biggest factor in getting InstantHMR to match+SAM3D's accuracy: any drift between annotation crops and inference crops+shows up directly as test-time error.++## Datasets we used++InstantHMR was trained on a mix of:++- **COCO** (in-the-wild humans),+- **Harmony4D** (multi-person studio captures),+- a **synthetic dataset** of rendered humans, and+- domain-specific footage (rugby, aikido) that the model is meant to handle.+diff --git a/docs/architecture.md b/docs/architecture.mdnew file mode 100644index 0000000..4017009--- /dev/null+++ b/docs/architecture.md@@ -0,0 +1,100 @@+# Model architecture++InstantHMR is a 17 M-parameter network that mimics+`facebook/sam-3d-body-dinov3` on the **per-person 3D pose estimation** task,+running ~80× faster on a single GPU and exporting cleanly to ONNX / TFLite /+QNN for mobile and edge deployment.++## High level++```+ RF-DETR ──► square 1.2× crop ──► RepViT backbone ──► cross-attn decoder+ │+ CLIFF cond [cx, cy, scale] ──┘+ ▼+ mhr_params (204), shape (45), cam_trans (3),+ joints_2d (70, 2), joints_3d (70, 3)+```++## Backbone — RepViT-M1.5++`timm.create_model("repvit_m1_5", pretrained=False, features_only=True)`+gives a `(B, 512, 7, 7)` feature map at 224 input. Re-parameterised+inverted-residual blocks make this fast on both desktop and mobile CPUs.++## Decoder — 9-query cross-attention++Two `nn.TransformerDecoderLayer` blocks (`d_model=256`, 4 heads, GELU,+pre-LN) attend the flattened backbone features (49 tokens) with 9 learnable+query tokens:++| Query | Output | Head shape |+|-------|------------------------------|-----------:|+| 0 | camera translation (CLIFF) | (3,) |+| 1 | pose (axis-angle joints) | (136,) |+| 2 | scales | (68,) |+| 3 | identity shape blendshapes | (45,) |+| 4–8 | torso / L arm / R arm / L leg / R leg → joints | (5×256 → 70×3) |++A fixed 2D sin-cos positional embedding is added to the 7×7 token grid.++## CLIFF camera conditioning++The camera-translation head is the only one that consumes the CLIFF+conditioning vector — `[cx_norm, cy_norm, b_scale]` of the detector bbox+(RF-DETR at runtime, YOLO during annotation) in **full-frame** coordinates.+This is what lets InstantHMR predict translation consistent with the+original perspective of the source image, without ever seeing pixels+outside the 224×224 crop.++```+cam_token (B, 256)+cliff_cond (B, 3)+ │ │+ └─────cat─────┘+ │+ LayerNorm + MLP+ │+ ▼+ cam_trans (B, 3)+```++## Output coordinate frame++Both `joints_3d` and `cam_trans` are in metres in the SAM3D camera frame:++- **X right**, **Y down**, **Z forward** (right-handed, Y-down).+- `joints_3d` are body-centred — add `cam_trans` to land in camera space.+- `joints_2d` come from a dedicated 2D head, in **normalised crop coords**+ `[-1, 1]`; the demo's `instanthmr/inference.py` re-projects them to+ full-frame pixels using the bbox-derived crop transform.++## Why distill?++The DINOv3-based teacher is excellent but heavy: its tokenizer + decoder runs+at <20 FPS on a desktop GPU and doesn't fit comfortably on edge hardware.+Distillation gives us InstantHMR — a model that:++- runs at ~150 FPS on a single RTX 4070,+- exports to ONNX / TFLite / QNN, and+- preserves the SAM3D 70-keypoint & MHR outputs, so any downstream code that+ consumed `pred_keypoints_3d` keeps working.++## Detection: RF-DETR++The demo pipeline uses **RF-DETR** (Roboflow's transformer-based detector)+for the upstream person detection step. We picked it over YOLO because:++- it has **stronger recall on small / partially-occluded persons**, which+ matters a lot once the bbox is tight enough to drive a pose model;+- its single-stage transformer head produces calibrated confidence scores,+ so the `--det-confidence` knob behaves predictably; and+- it ships with a `optimize_for_inference()` pass (TorchScript ++ fused ops) that reliably halves detector latency on a desktop GPU.++At inference the demo reports detector and HMR latency **separately**, both+in the console and as scalar plots in the Rerun viewer, so it's always+clear whether RF-DETR or InstantHMR is the bottleneck on a given machine.+The default `--detector-variant medium` is tuned for ~480p–1080p input.+Drop to `nano` / `small` for higher FPS at lower resolution, or step up to+`base` / `large` for crowded scenes.diff --git a/docs/training.md b/docs/training.mdnew file mode 100644index 0000000..3aeeaf1--- /dev/null+++ b/docs/training.md@@ -0,0 +1,112 @@+# Training++The InstantHMR training notebook is+[`notebooks/distill_transformer_decoder.ipynb`](../notebooks/distill_transformer_decoder.ipynb).+It is the source of truth — this page is a tour, not a re-implementation.++## Setup++```bash+pip install torch timm onnx tqdm loguru+jupyter notebook notebooks/distill_transformer_decoder.ipynb+```++You'll need the annotations produced by+[`docs/annotation.md`](annotation.md). The notebook expects the directory+layout:++```+<dataset_root>/+├── images/ *.png (224×224 crops)+└── annotations/ *.npz (teacher outputs)+```++## What the notebook does++1. **Dataset** — loads the `(image, annotation)` pairs, applies light+ augmentation (colour jitter, horizontal flip with paired-joint remapping),+ and yields the 4-tuple `(image, cliff_cond, gt_targets, mask)`.+2. **Architecture** — defines InstantHMR (RepViT-M1.5 backbone + 9-token+ cross-attention decoder) exactly as it ships in this repo's+ `instanthmr/inference.py` (the runtime path is parameter-compatible+ with the trained `.pth`).+3. **Loss** — weighted L1 on each output head:+ - mhr_params, shape_params, cam_trans (parameter regression),+ - joints_3d (mean per-joint position error in camera frame),+ - joints_2d (after re-projection back to full-frame pixels).+4. **Training loop** — AdamW with cosine schedule + warm-up, mixed precision+ on CUDA, gradient clipping at 1.0.+5. **Validation** — held-out split, MPJPE on `joints_3d` reported.+6. **Export** — at the end of training, the model is exported to+ `instanthmr.onnx` with dynamic batch axis using the same input schema+ the demo expects (see [`models/README.md`](../models/README.md)).++## Hyperparameters at a glance++| | |+|---|---|+| Backbone | RepViT-M1.5 (`timm`) |+| Decoder layers | 2 |+| `d_model` | 256 |+| Heads | 4 |+| Dropout | 0.1 |+| Image size | 224 |+| Optimiser | AdamW (β=0.9/0.999, wd=1e-2) |+| LR schedule | cosine + linear warm-up |+| Precision | mixed (fp16 forward, fp32 master) |++The exact values for the released checkpoint live in the notebook's config+cell.++## Re-exporting to ONNX++The notebook's export cell is reproduced below for reference:++```python+dummy_image = torch.randn(1, 3, 224, 224, device=device)+dummy_cliff = torch.randn(1, 3, device=device)++torch.onnx.export(+ model.eval(),+ (dummy_image, dummy_cliff),+ "instanthmr.onnx",+ input_names=["image", "cliff_cond"],+ output_names=[+ "mhr_params", "shape_params", "cam_trans",+ "joints_2d", "joints_3d",+ ],+ dynamic_axes={+ "image": {0: "batch"},+ "cliff_cond": {0: "batch"},+ "mhr_params": {0: "batch"},+ "shape_params": {0: "batch"},+ "cam_trans": {0: "batch"},+ "joints_2d": {0: "batch"},+ "joints_3d": {0: "batch"},+ },+ opset_version=17,+)+```++Verify with `onnxruntime`:++```python+import onnxruntime as ort+s = ort.InferenceSession("instanthmr.onnx",+ providers=["CPUExecutionProvider"])+print([(i.name, i.shape) for i in s.get_inputs()])+print([(o.name, o.shape) for o in s.get_outputs()])+```++## Tips++- **Don't drift the preprocessing.** The demo's+ `InstantHMR._preprocess` (`instanthmr/inference.py`) is the exact+ inverse of the annotation crop: 1.2× square padded crop → 224 → ImageNet+ norm. Any change to either side has to be mirrored on the other.+- **Joints are body-centred.** Don't add `cam_trans` to `joints_3d` before+ computing the loss; the teacher stores them as body-centred, and+ InstantHMR's last layer is trained against that.+- **CLIFF conditioning lives in full-frame coords**, not crop coords.+ `cx_norm = 2·cx/W − 1` etc. Recomputing this from the crop will silently+ degrade the camera head.diff --git a/instanthmr/__init__.py b/instanthmr/__init__.pynew file mode 100644index 0000000..1194ccc--- /dev/null+++ b/instanthmr/__init__.py@@ -0,0 +1,20 @@+"""InstantHMR — standalone 3D human pose inference + Rerun visualization."""++from .inference import InstantHMR, HMRPrediction+from .detector import RFDETRDetector+from .pipeline import PosePipeline, FrameResult+from .skeleton import JOINT_NAMES, SKELETON_EDGES, NUM_JOINTS, edges_for++__all__ = [+ "InstantHMR",+ "HMRPrediction",+ "RFDETRDetector",+ "PosePipeline",+ "FrameResult",+ "JOINT_NAMES",+ "SKELETON_EDGES",+ "NUM_JOINTS",+ "edges_for",+]++__version__ = "0.1.0"diff --git a/instanthmr/detector.py b/instanthmr/detector.pynew file mode 100644index 0000000..c828eb4--- /dev/null+++ b/instanthmr/detector.py@@ -0,0 +1,99 @@+"""RF-DETR person detector.++Wraps Roboflow's RF-DETR (medium variant by default) with a minimal API:+``detect(image_rgb)`` returns a list of+``{"bbox": [x1, y1, x2, y2], "confidence": float}`` dicts sorted by+descending confidence.++RF-DETR is a transformer-based detector with strong COCO recall on small+and partially-occluded persons — a good match for upstream pose+estimation, which is sensitive to bbox localisation.+"""++from __future__ import annotations++import numpy as np+++COCO_PERSON_CLASS_ID = 1 # RF-DETR exposes COCO 1-indexed class ids+++class RFDETRDetector:+ """Person detector backed by RF-DETR.++ Args:+ variant: Which RF-DETR variant to load. One of+ ``"nano"``, ``"small"``, ``"medium"``, ``"base"``, ``"large"``.+ Default ``"medium"`` balances speed and accuracy.+ confidence: Minimum detection score.+ max_persons: Maximum number of persons returned per frame.+ optimize_for_inference: Call RF-DETR's TorchScript / fused-op+ optimisation pass on construction. Adds a few seconds of+ startup but speeds up subsequent ``detect`` calls noticeably.+ """++ _VARIANTS = {+ "nano": "RFDETRNano",+ "small": "RFDETRSmall",+ "medium": "RFDETRMedium",+ "base": "RFDETRBase",+ "large": "RFDETRLarge",+ }++ def __init__(+ self,+ variant: str = "medium",+ confidence: float = 0.5,+ max_persons: int = 5,+ optimize_for_inference: bool = True,+ ):+ import rfdetr++ if variant not in self._VARIANTS:+ raise ValueError(+ f"Unknown RF-DETR variant {variant!r}. "+ f"Choose one of {list(self._VARIANTS)}."+ )++ cls = getattr(rfdetr, self._VARIANTS[variant])+ self._model = cls()+ if optimize_for_inference:+ self._model.optimize_for_inference()++ self._confidence = confidence+ self._max_persons = max_persons+ self._variant = variant++ @property+ def variant(self) -> str:+ return self._variant++ def detect(self, image_rgb: np.ndarray) -> list[dict]:+ """Detect persons in an RGB image.++ Args:+ image_rgb: ``(H, W, 3)`` uint8 RGB array.++ Returns:+ List of detection dicts sorted by descending confidence,+ truncated to ``max_persons``.+ """+ from PIL import Image as PILImage++ pil_image = PILImage.fromarray(image_rgb)+ sv_detections = self._model.predict(pil_image, threshold=self._confidence)++ detections: list[dict] = []+ if len(sv_detections) > 0 and sv_detections.class_id is not None:+ mask = sv_detections.class_id == COCO_PERSON_CLASS_ID+ for bbox, conf in zip(+ sv_detections.xyxy[mask],+ sv_detections.confidence[mask],+ ):+ detections.append({+ "bbox": np.asarray(bbox, dtype=np.float32),+ "confidence": float(conf),+ })++ detections.sort(key=lambda d: d["confidence"], reverse=True)+ return detections[: self._max_persons]diff --git a/instanthmr/inference.py b/instanthmr/inference.pynew file mode 100644index 0000000..13e21a5--- /dev/null+++ b/instanthmr/inference.py@@ -0,0 +1,267 @@+"""ONNX inference for InstantHMR.++InstantHMR takes a 224x224 person crop and a 3-vector CLIFF condition+(bbox center / scale in full-frame coords) and returns mhr_params, shape,+camera translation, and 70 joints in 2D (crop space) and 3D (camera coords,+body-centred, metres, Y-down).++This module wraps the ONNX session and the per-person preprocessing — square+crop, CLIFF cond, ImageNet normalisation, and re-projection of joints back+into full-frame pixels.+"""++from __future__ import annotations++import math+from dataclasses import dataclass+from pathlib import Path+from typing import Optional++import cv2+import numpy as np+++IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)+INPUT_SIZE = 224+CROP_EXPAND = 1.2 # square crop around the detector bbox, matching training+++@dataclass+class HMRPrediction:+ """Per-person outputs from one InstantHMR forward pass.++ All numpy, all in the FULL-frame coordinate system.++ Attributes:+ bbox: (4,) raw detector bbox [x1, y1, x2, y2].+ confidence: detection confidence in [0, 1].+ joints_3d_local: (70, 3) body-centred joints, metres, Y-down.+ joints_3d_cam: (70, 3) joints in camera space (= local + cam_trans).+ joints_2d: (70, 2) joints projected into full-frame pixel coords.+ cam_trans: (3,) camera translation [tx, ty, tz], metres.+ focal_length: (2,) [fx, fy] full-frame virtual focal = sqrt(H^2+W^2).+ principal_point: (2,) [cx, cy] = full-frame centre.+ image_shape: (H, W) of the source frame.+ mhr_params: (204,) raw MHR pose+scale parameters. InstantHMR does+ not run the MHR mesh decoder itself (it is resource-heavy and+ kept out of the default pipeline); these params are exposed so+ you can feed them into Meta's MHR TorchScript module from the+ original SAM 3D Body release if you want a body mesh —+ https://github.com/facebookresearch/sam-3d-body.+ shape_params: (45,) MHR identity shape blendshapes. Same use as+ ``mhr_params`` — pair them and run them through MHR.+ """++ bbox: np.ndarray+ confidence: float+ joints_3d_local: np.ndarray+ joints_3d_cam: np.ndarray+ joints_2d: np.ndarray+ cam_trans: np.ndarray+ focal_length: np.ndarray+ principal_point: np.ndarray+ image_shape: tuple[int, int]+ mhr_params: np.ndarray+ shape_params: np.ndarray+++class InstantHMR:+ """ONNX wrapper for the InstantHMR model.++ Example+ -------+ hmr = InstantHMR("models/instanthmr.onnx")+ out = hmr.predict(image_rgb, bbox=[x1, y1, x2, y2])+ print(out.joints_3d_cam.shape) # (70, 3)+ """++ def __init__(+ self,+ onnx_path: str | Path,+ device: str = "cuda",+ providers: Optional[list[str]] = None,+ ):+ import onnxruntime as ort++ onnx_path = Path(onnx_path)+ if not onnx_path.exists():+ raise FileNotFoundError(f"ONNX model not found: {onnx_path}")++ # ORT >= 1.19 ships preload_dlls() which discovers CUDA / cuDNN+ # bundled by the nvidia-* pip wheels (or by torch) and adds them to+ # the loader's search path. Calling it before session creation is+ # the simplest fix for "libcudnn.so.9: cannot open shared object+ # file" on machines that don't ship system CUDA.+ if hasattr(ort, "preload_dlls"):+ try:+ ort.preload_dlls()+ except Exception:+ # preload_dlls is best-effort — never let it block init.+ pass++ if providers is None:+ providers = self._default_providers(device)++ self.session = ort.InferenceSession(str(onnx_path), providers=providers)+ self.active_provider = self.session.get_providers()[0]++ in_names = [i.name for i in self.session.get_inputs()]+ out_names = [o.name for o in self.session.get_outputs()]+ self._in_image = in_names[0] # "image"+ self._in_cliff = in_names[1] # "cliff_cond"+ # Output order from export: mhr_params, shape_params, cam_trans, joints_2d, joints_3d+ self._out_names = out_names++ # ------------------------------------------------------------------+ # Public API+ # ------------------------------------------------------------------++ def predict(+ self,+ image_rgb: np.ndarray,+ bbox: np.ndarray | list[float],+ confidence: float = 1.0,+ ) -> HMRPrediction:+ """Run InstantHMR on a single person.++ Args:+ image_rgb: (H, W, 3) uint8 RGB full-frame image.+ bbox: tight person bbox [x1, y1, x2, y2] in pixel coords.+ confidence: detection confidence to attach to the output.++ Returns:+ ``HMRPrediction`` with joints in full-frame coordinates.+ """+ image_rgb = np.ascontiguousarray(image_rgb)+ h, w = image_rgb.shape[:2]+ bbox_arr = np.asarray(bbox, dtype=np.float32).reshape(4)++ crop, sq_x1, sq_y1, sq_size, cliff_cond = self._preprocess(+ image_rgb, bbox_arr, h, w+ )++ outs = self.session.run(+ None,+ {+ self._in_image: crop[np.newaxis], # (1, 3, 224, 224)+ self._in_cliff: cliff_cond[np.newaxis], # (1, 3)+ },+ )+ # Order: mhr_params, shape_params, cam_trans, joints_2d, joints_3d+ mhr_params = outs[0][0].astype(np.float32)+ shape_params = outs[1][0].astype(np.float32)+ cam_trans = outs[2][0].astype(np.float32)+ joints_2d_norm = outs[3][0].astype(np.float32) # (70, 2) in [-1, 1]+ joints_3d_local = outs[4][0].astype(np.float32) # (70, 3) body-centred++ # Re-project the 2D head from normalised crop space → full-frame pixels.+ crop_px = (joints_2d_norm + 1.0) * 0.5 * INPUT_SIZE+ scale = sq_size / INPUT_SIZE+ joints_2d = np.stack(+ [crop_px[:, 0] * scale + sq_x1, crop_px[:, 1] * scale + sq_y1],+ axis=-1,+ ).astype(np.float32)++ joints_3d_cam = joints_3d_local + cam_trans++ # Full-frame virtual pinhole: focal = sqrt(H^2 + W^2), principal = centre.+ f = math.sqrt(h * h + w * w)+ focal_length = np.array([f, f], dtype=np.float32)+ principal_point = np.array([w / 2.0, h / 2.0], dtype=np.float32)++ return HMRPrediction(+ bbox=bbox_arr,+ confidence=float(confidence),+ joints_3d_local=joints_3d_local,+ joints_3d_cam=joints_3d_cam,+ joints_2d=joints_2d,+ cam_trans=cam_trans,+ focal_length=focal_length,+ principal_point=principal_point,+ image_shape=(h, w),+ mhr_params=mhr_params,+ shape_params=shape_params,+ )++ # ------------------------------------------------------------------+ # Preprocessing+ # ------------------------------------------------------------------++ @staticmethod+ def _preprocess(+ image_rgb: np.ndarray,+ bbox: np.ndarray,+ h: int,+ w: int,+ ) -> tuple[np.ndarray, float, float, float, np.ndarray]:+ """Square 1.2x crop, ImageNet normalise, and CLIFF conditioning."""+ x1, y1, x2, y2 = bbox.astype(float)+ bw = x2 - x1+ bh = y2 - y1+ cx = (x1 + x2) / 2.0+ cy = (y1 + y2) / 2.0++ # CLIFF conditioning vector (full-frame coords)+ cx_norm = 2.0 * (cx / w) - 1.0+ cy_norm = 2.0 * (cy / h) - 1.0+ b_scale = max(bw, bh) / max(w, h)+ cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)++ sq_size = max(bw, bh) * CROP_EXPAND+ half = sq_size / 2.0+ sq_x1 = cx - half+ sq_y1 = cy - half++ ix1 = int(math.floor(sq_x1))+ iy1 = int(math.floor(sq_y1))+ ix2 = int(math.ceil(sq_x1 + sq_size))+ iy2 = int(math.ceil(sq_y1 + sq_size))++ pad_left = max(0, -ix1)+ pad_top = max(0, -iy1)+ pad_right = max(0, ix2 - w)+ pad_bottom = max(0, iy2 - h)++ src_x1 = max(0, ix1)+ src_y1 = max(0, iy1)+ src_x2 = min(w, ix2)+ src_y2 = min(h, iy2)++ patch = image_rgb[src_y1:src_y2, src_x1:src_x2]+ if pad_left or pad_top or pad_right or pad_bottom:+ patch = cv2.copyMakeBorder(+ patch, pad_top, pad_bottom, pad_left, pad_right,+ cv2.BORDER_CONSTANT, value=(0, 0, 0),+ )++ crop_224 = cv2.resize(patch, (INPUT_SIZE, INPUT_SIZE), interpolation=cv2.INTER_LINEAR)+ crop = crop_224.astype(np.float32) / 255.0+ crop = (crop - IMAGENET_MEAN) / IMAGENET_STD+ crop = np.transpose(crop, (2, 0, 1)).astype(np.float32) # (3, 224, 224)++ return crop, float(sq_x1), float(sq_y1), float(sq_size), cliff_cond++ # ------------------------------------------------------------------+ # Provider selection+ # ------------------------------------------------------------------++ @staticmethod+ def _default_providers(device: str) -> list[str]:+ """Pick ORT execution providers based on the requested *device*.++ TensorRT is intentionally **not** requested by default: the wheel+ distinction between ``onnxruntime-gpu`` and the ``tensorrt`` system+ libraries is fiddly, and asking for an EP whose runtime libraries+ aren't installed produces a verbose error before falling back.+ Pass ``providers=`` explicitly if you want TensorRT.+ """+ import onnxruntime as ort++ available = set(ort.get_available_providers())+ wanted: list[str] = []+ if "cuda" in device.lower():+ if "CUDAExecutionProvider" in available:+ wanted.append("CUDAExecutionProvider")+ wanted.append("CPUExecutionProvider")+ return wanteddiff --git a/instanthmr/pipeline.py b/instanthmr/pipeline.pynew file mode 100644index 0000000..8da5eb5--- /dev/null+++ b/instanthmr/pipeline.py@@ -0,0 +1,126 @@+"""End-to-end pose estimation pipeline.++Combines the RF-DETR person detector with InstantHMR to produce per-person+3D + 2D joints from a single RGB frame, while reporting **separate**+detector and HMR timings so the demo can surface them.+"""++from __future__ import annotations++import time+from dataclasses import dataclass+from pathlib import Path+from typing import Optional++import numpy as np++from .detector import RFDETRDetector+from .inference import InstantHMR, HMRPrediction+++@dataclass+class FrameResult:+ """Output of one ``PosePipeline.predict`` call.++ Attributes:+ persons: Per-person ``HMRPrediction``s (possibly empty).+ detector_ms: Wall-clock time for the detector forward pass.+ hmr_ms: Wall-clock time for InstantHMR across all persons in the+ frame (sum, not average).+ total_ms: ``detector_ms + hmr_ms`` plus any pipeline overhead.+ """++ persons: list[HMRPrediction]+ detector_ms: float+ hmr_ms: float+ total_ms: float+++class PosePipeline:+ """Full RF-DETR → InstantHMR pipeline.++ Args:+ onnx_path: Path to ``instanthmr.onnx``.+ device: ``"cuda"`` or ``"cpu"`` for InstantHMR (RF-DETR auto-selects).+ detector_variant: ``"nano" | "small" | "medium" | "base" | "large"``.+ det_confidence: RF-DETR confidence threshold.+ max_persons: Maximum number of persons returned per frame.++ Example+ -------+ pipeline = PosePipeline("models/instanthmr.onnx")+ result = pipeline.predict(image_rgb)+ print(f"det {result.detector_ms:.1f} ms / hmr {result.hmr_ms:.1f} ms")+ for p in result.persons:+ print(p.joints_3d_cam.shape) # (70, 3)+ """++ def __init__(+ self,+ onnx_path: str | Path,+ device: str = "cuda",+ detector_variant: str = "medium",+ det_confidence: float = 0.5,+ max_persons: int = 5,+ ):+ self.hmr = InstantHMR(onnx_path, device=device)+ self.detector = RFDETRDetector(+ variant=detector_variant,+ confidence=det_confidence,+ max_persons=max_persons,+ )+ self._last_bbox: Optional[np.ndarray] = None++ def predict(self, image_rgb: np.ndarray) -> FrameResult:+ """Detect all persons in *image_rgb* and run InstantHMR on each.++ Returns a :class:`FrameResult` with per-stage timings so callers+ can display detector vs HMR cost separately.+ """+ t_total_start = time.perf_counter()++ # ---- Detector ----+ t0 = time.perf_counter()+ detections = self.detector.detect(image_rgb)+ detector_ms = (time.perf_counter() - t0) * 1000.0++ # Single-frame fallback: reuse the last bbox if the detector drops+ # a frame. Useful for live camera mode where a transient miss is+ # common.+ if not detections and self._last_bbox is not None:+ h, w = image_rgb.shape[:2]+ lb = self._last_bbox+ bw, bh = lb[2] - lb[0], lb[3] - lb[1]+ expand = 0.1+ fb = np.array([+ max(0, lb[0] - bw * expand),+ max(0, lb[1] - bh * expand),+ min(w, lb[2] + bw * expand),+ min(h, lb[3] + bh * expand),+ ], dtype=np.float32)+ detections = [{"bbox": fb, "confidence": 0.0}]++ # ---- InstantHMR (sum across all persons) ----+ outputs: list[HMRPrediction] = []+ t1 = time.perf_counter()+ for det in detections:+ out = self.hmr.predict(+ image_rgb,+ bbox=det["bbox"],+ confidence=det["confidence"],+ )+ outputs.append(out)+ hmr_ms = (time.perf_counter() - t1) * 1000.0++ if outputs:+ best = max(outputs, key=lambda o: o.confidence)+ self._last_bbox = best.bbox.copy()++ total_ms = (time.perf_counter() - t_total_start) * 1000.0++ return FrameResult(+ persons=outputs,+ detector_ms=detector_ms,+ hmr_ms=hmr_ms,+ total_ms=total_ms,+ )diff --git a/instanthmr/skeleton.py b/instanthmr/skeleton.pynew file mode 100644index 0000000..2345253--- /dev/null+++ b/instanthmr/skeleton.py@@ -0,0 +1,73 @@+"""MHR70 joint ordering and skeleton connectivity.++The student model regresses 70 keypoints in the MHR70 ordering, matching+the SAM3D teacher output (`pred_keypoints_3d`). Body landmarks first, then+hands, then olecranon / cubital / acromion / neck.+"""++from __future__ import annotations+++JOINT_NAMES: list[str] = [+ "nose", "left_eye", "right_eye", "left_ear", "right_ear", # 0-4+ "left_shoulder", "right_shoulder", # 5-6+ "left_elbow", "right_elbow", # 7-8+ "left_hip", "right_hip", # 9-10+ "left_knee", "right_knee", # 11-12+ "left_ankle", "right_ankle", # 13-14+ "left_big_toe_tip", "left_small_toe_tip", "left_heel", # 15-17+ "right_big_toe_tip", "right_small_toe_tip", "right_heel", # 18-20+ "right_thumb_tip", "right_thumb_first_joint",+ "right_thumb_second_joint", "right_thumb_third_joint", # 21-24+ "right_index_tip", "right_index_first_joint",+ "right_index_second_joint", "right_index_third_joint", # 25-28+ "right_middle_tip", "right_middle_first_joint",+ "right_middle_second_joint", "right_middle_third_joint", # 29-32+ "right_ring_tip", "right_ring_first_joint",+ "right_ring_second_joint", "right_ring_third_joint", # 33-36+ "right_pinky_tip", "right_pinky_first_joint",+ "right_pinky_second_joint", "right_pinky_third_joint", # 37-40+ "right_wrist", # 41+ "left_thumb_tip", "left_thumb_first_joint",+ "left_thumb_second_joint", "left_thumb_third_joint", # 42-45+ "left_index_tip", "left_index_first_joint",+ "left_index_second_joint", "left_index_third_joint", # 46-49+ "left_middle_tip", "left_middle_first_joint",+ "left_middle_second_joint", "left_middle_third_joint", # 50-53+ "left_ring_tip", "left_ring_first_joint",+ "left_ring_second_joint", "left_ring_third_joint", # 54-57+ "left_pinky_tip", "left_pinky_first_joint",+ "left_pinky_second_joint", "left_pinky_third_joint", # 58-61+ "left_wrist", # 62+ "left_olecranon", "right_olecranon", # 63-64+ "left_cubital_fossa", "right_cubital_fossa", # 65-66+ "left_acromion", "right_acromion", # 67-68+ "neck", # 69+]+++SKELETON_EDGES: list[tuple[int, int]] = [+ # Head <-> neck+ (0, 69),+ # Torso+ (69, 5), (69, 6),+ (5, 9), (6, 10), (9, 10),+ # Arms+ (5, 7), (7, 62),+ (6, 8), (8, 41),+ # Legs+ (9, 11), (11, 13), (13, 17),+ (10, 12), (12, 14), (14, 20),+ # Left hand fan+ (62, 42), (62, 46), (62, 50), (62, 54), (62, 58),+ # Right hand fan+ (41, 21), (41, 25), (41, 29), (41, 33), (41, 37),+]+++NUM_JOINTS = 70+++def edges_for(num_joints: int) -> list[tuple[int, int]]:+ """Return the subset of edges that index into the available joints."""+ return [(a, b) for a, b in SKELETON_EDGES if a < num_joints and b < num_joints]diff --git a/instanthmr/visualizer.py b/instanthmr/visualizer.pynew file mode 100644index 0000000..92f7ffb--- /dev/null+++ b/instanthmr/visualizer.py@@ -0,0 +1,276 @@+"""Rerun-based 3D visualization for the InstantHMR demo.++Logs, per frame:+ - ``camera/image`` : annotated RGB frame (2D skeleton + bbox + a+ text panel showing detector / HMR / total ms).+ - ``camera`` : pinhole intrinsics so the image becomes a+ proper frustum in the 3D scene.+ - ``world/persons/...`` : 3D joints + skeleton lines per person.+ - ``timing/detector_ms`` : scalar plot of RF-DETR latency.+ - ``timing/hmr_ms`` : scalar plot of InstantHMR latency.+ - ``timing/total_ms`` : scalar plot of full pipeline latency.+ - ``timing/fps`` : scalar plot of effective frames-per-second.++``rr.init(..., spawn=True)`` is called automatically so the viewer opens+on construction (pass ``spawn_viewer=False`` to suppress).+"""++from __future__ import annotations++from typing import Optional, Sequence++import cv2+import numpy as np++from .inference import HMRPrediction+from .skeleton import edges_for+++class RerunVisualizer:+ def __init__(+ self,+ application_id: str = "instanthmr_demo",+ spawn_viewer: bool = True,+ save_path: str | None = None,+ ):+ import rerun as rr++ self._rr = rr+ rr.init(application_id, spawn=spawn_viewer)++ # SAM3D / MHR convention: right-handed, Y-down camera frame.+ rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True)++ if save_path:+ rr.save(save_path)++ self._prev_num_persons = 0+ self._blueprint_sent = False++ # ------------------------------------------------------------------+ # Per-frame logging+ # ------------------------------------------------------------------++ def log_frame(+ self,+ image_rgb: np.ndarray,+ persons: Sequence[HMRPrediction],+ frame_idx: int,+ timestamp: float | None = None,+ detector_ms: Optional[float] = None,+ hmr_ms: Optional[float] = None,+ total_ms: Optional[float] = None,+ ) -> None:+ """Log one frame of the pipeline to Rerun.++ Args:+ image_rgb: full-frame RGB image, ``(H, W, 3)`` uint8.+ persons: per-person HMR predictions for this frame.+ frame_idx: monotonic frame counter (sets the Rerun timeline).+ timestamp: optional wall-clock timestamp (seconds).+ detector_ms: RF-DETR latency for this frame (ms).+ hmr_ms: InstantHMR latency for this frame, summed across+ persons (ms).+ total_ms: full pipeline latency for this frame (ms).+ """+ rr = self._rr+ rr.set_time("frame", sequence=frame_idx)+ if timestamp is not None:+ rr.set_time("timestamp", duration=timestamp)++ # Per-stage scalar plots+ if detector_ms is not None:+ rr.log("timing/detector_ms", rr.Scalars(float(detector_ms)))+ if hmr_ms is not None:+ rr.log("timing/hmr_ms", rr.Scalars(float(hmr_ms)))+ if total_ms is not None:+ rr.log("timing/total_ms", rr.Scalars(float(total_ms)))+ if total_ms > 0:+ rr.log("timing/fps", rr.Scalars(1000.0 / float(total_ms)))++ # Clear stale person entities from previous frames+ n = len(persons)+ for stale in range(n, self._prev_num_persons):+ rr.log(f"world/persons/person_{stale}", rr.Clear(recursive=True))+ self._prev_num_persons = n++ h, w = image_rgb.shape[:2]++ # Always log the image — even when no person was detected — so the+ # viewer keeps showing the camera feed and the timing HUD.+ if not persons:+ blank = self._draw_overlay(image_rgb, [], detector_ms, hmr_ms, total_ms)+ rr.log("camera/image", rr.Image(blank))+ return++ for idx, person in enumerate(persons):+ self._log_person(idx, person)++ annotated = self._draw_overlay(+ image_rgb, persons, detector_ms, hmr_ms, total_ms,+ )+ rr.log("camera/image", rr.Image(annotated))++ # Pinhole camera in the 3D scene — full-frame focal so the image+ # plane sits in front of the joints, not on top of them.+ first = persons[0]+ fx, fy = float(first.focal_length[0]), float(first.focal_length[1])+ cx, cy = w / 2.0, h / 2.0+ depth = float(first.joints_3d_cam[:, 2].mean())+ plane_dist = max(depth * 0.25, 0.2)++ rr.log(+ "camera",+ rr.Pinhole(+ width=w,+ height=h,+ focal_length=[fx, fy],+ principal_point=[cx, cy],+ image_plane_distance=plane_dist,+ ),+ )++ if not self._blueprint_sent:+ self._send_blueprint(first)+ self._blueprint_sent = True++ # ------------------------------------------------------------------+ # Helpers+ # ------------------------------------------------------------------++ def _log_person(self, idx: int, person: HMRPrediction) -> None:+ rr = self._rr+ path = f"world/persons/person_{idx}"++ joints = person.joints_3d_cam+ rr.log(+ f"{path}/joints",+ rr.Points3D(positions=joints, radii=0.012, colors=[0, 230, 0]),+ )++ edges = edges_for(joints.shape[0])+ if edges:+ lines = [+ [joints[i].tolist(), joints[j].tolist()]+ for i, j in edges+ ]+ rr.log(+ f"{path}/skeleton",+ rr.LineStrips3D(lines, colors=[255, 230, 0], radii=0.004),+ )++ @staticmethod+ def _draw_overlay(+ image_rgb: np.ndarray,+ persons: Sequence[HMRPrediction],+ detector_ms: Optional[float],+ hmr_ms: Optional[float],+ total_ms: Optional[float],+ ) -> np.ndarray:+ """Draw per-person bbox + 2D skeleton + a timing HUD onto the frame."""+ out = image_rgb.copy()+ h, w = out.shape[:2]++ for person in persons:+ x1, y1, x2, y2 = person.bbox.astype(int)+ cv2.rectangle(out, (x1, y1), (x2, y2), (0, 200, 255), 2)+ label = f"{person.confidence:.2f}"+ (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)+ cv2.rectangle(out, (x1, y1 - th - 6), (x1 + tw + 4, y1), (0, 200, 255), -1)+ cv2.putText(+ out, label, (x1 + 2, y1 - 4),+ cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1,+ )++ j2d = person.joints_2d+ valid = (+ (j2d[:, 0] >= 0) & (j2d[:, 0] < w)+ & (j2d[:, 1] >= 0) & (j2d[:, 1] < h)+ )++ for i, j in edges_for(j2d.shape[0]):+ if valid[i] and valid[j]:+ pt1 = tuple(j2d[i].astype(int))+ pt2 = tuple(j2d[j].astype(int))+ cv2.line(out, pt1, pt2, (255, 230, 0), 2)++ for k, (x, y) in enumerate(j2d):+ if valid[k]:+ cv2.circle(out, (int(x), int(y)), 3, (0, 230, 0), -1)++ # ---- Timing HUD (top-left, semi-transparent black panel) ----+ lines: list[str] = []+ if detector_ms is not None:+ lines.append(f"RF-DETR : {detector_ms:6.1f} ms")+ if hmr_ms is not None:+ lines.append(f"InstantHMR: {hmr_ms:6.1f} ms")+ if total_ms is not None:+ fps = 1000.0 / total_ms if total_ms > 0 else 0.0+ lines.append(f"Total : {total_ms:6.1f} ms ({fps:5.1f} fps)")+ if lines:+ font = cv2.FONT_HERSHEY_SIMPLEX+ scale = max(0.5, min(w, h) / 1500.0)+ thick = max(1, int(round(scale * 1.5)))+ sizes = [+ cv2.getTextSize(s, font, scale, thick)[0] for s in lines+ ]+ line_h = max(s[1] for s in sizes) + 8+ box_w = max(s[0] for s in sizes) + 16+ box_h = line_h * len(lines) + 8++ overlay = out.copy()+ cv2.rectangle(overlay, (8, 8), (8 + box_w, 8 + box_h), (0, 0, 0), -1)+ cv2.addWeighted(overlay, 0.55, out, 0.45, 0, out)++ y = 8 + line_h+ for s in lines:+ cv2.putText(+ out, s, (16, y),+ font, scale, (255, 255, 255), thick, cv2.LINE_AA,+ )+ y += line_h++ return out++ def _send_blueprint(self, first: HMRPrediction) -> None:+ """Send a 3-panel blueprint: camera image | 3D scene | timings."""+ import rerun.blueprint as rrb+ rr = self._rr++ joints = first.joints_3d_cam+ center = joints.mean(axis=0)+ extent = float(np.abs(joints - center).max())+ cam_distance = max(extent * 4.0, 1.5)+ # Y-down convention: place the orbit camera "in front" along -Z.+ eye_pos = [+ float(center[0]),+ float(center[1]),+ float(center[2] - cam_distance),+ ]++ blueprint = rrb.Blueprint(+ rrb.Horizontal(+ rrb.Vertical(+ rrb.Spatial2DView(origin="camera/image", name="Camera"),+ rrb.TimeSeriesView(+ origin="timing",+ name="Latency (ms)",+ ),+ row_shares=[3, 1],+ ),+ rrb.Spatial3DView(+ origin="/",+ name="3D Scene",+ eye_controls=rrb.EyeControls3D(+ position=eye_pos,+ look_target=[float(center[0]), float(center[1]), float(center[2])],+ eye_up=[0.0, -1.0, 0.0],+ kind=rrb.Eye3DKind.Orbital,+ tracking_entity="world/persons/person_0",+ ),+ background=rrb.Background(color=[25, 25, 25]),+ ),+ column_shares=[2, 3],+ ),+ )+ rr.send_blueprint(blueprint)diff --git a/models/README.md b/models/README.mdnew file mode 100644index 0000000..1434ee4--- /dev/null+++ b/models/README.md@@ -0,0 +1,119 @@+# InstantHMR — model weights++A lightweight, ONNX-exportable distillation of+[`facebook/sam-3d-body-dinov3`](https://huggingface.co/facebook/sam-3d-body-dinov3)+for **3D human pose estimation and mesh recovery**: a RepViT-M1.5+backbone + a 9-token cross-attention decoder + CLIFF camera conditioning.+Trained to mimic the SAM3D teacher's per-person 70-keypoint outputs from+a single 224×224 crop.++Demo, training, and inference code live at the main InstantHMR+repository.++## Files++| File | Size | Purpose |+|---|---|---|+| `instanthmr.onnx` | ~77 MB | fp16 ONNX export — what the demo and `InstantHMR(...)` wrapper load. |+| `instanthmr.pth` | ~461 MB | PyTorch checkpoint for fine-tuning / further distillation. |++Most users only need `instanthmr.onnx`. The `.pth` is included for people+who want to continue training, swap the head, or re-export with different+options — see [`docs/training.md`](https://github.com/mohamdev/InstantHMR/blob/main/docs/training.md)+in the main repo.++## ONNX I/O++- **Inputs**+ - `image` — `(N, 3, 224, 224)` float32, ImageNet-normalised RGB.+ - `cliff_cond` — `(N, 3)` float32, `[cx_norm, cy_norm, b_scale]` of the+ detector bbox in the **full** frame (`cx_norm, cy_norm ∈ [-1, 1]`,+ `b_scale = max(bw, bh) / max(W, H)`).+- **Outputs**+ - `mhr_params` — `(N, 204)` MHR pose + scale parameters.+ - `shape_params` — `(N, 45)` MHR identity blendshapes.+ - `cam_trans` — `(N, 3)` body translation in camera frame (metres).+ - `joints_2d` — `(N, 70, 2)` keypoints in normalised crop coords `[-1, 1]`.+ - `joints_3d` — `(N, 70, 3)` body-centred 3D keypoints (metres, Y-down).++The 70 keypoints follow the MHR70 ordering — see+[`instanthmr/skeleton.py`](https://github.com/mohamdev/InstantHMR/blob/main/instanthmr/skeleton.py)+for joint names and skeleton edges.++## Quick start (ONNX)++Minimal end-to-end inference with `onnxruntime`, no extra dependencies:++```python+import cv2+import numpy as np+import onnxruntime as ort++IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)++# 1. Load the ONNX session (CUDA if available, otherwise CPU).+sess = ort.InferenceSession(+ "instanthmr.onnx",+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],+)++# 2. Load an RGB image and a person bbox [x1, y1, x2, y2].+image = cv2.cvtColor(cv2.imread("person.jpg"), cv2.COLOR_BGR2RGB)+H, W = image.shape[:2]+x1, y1, x2, y2 = 120, 40, 380, 600 # replace with your detector output++# 3. Square 1.2x crop around the bbox, resized to 224x224.+cx, cy = (x1 + x2) / 2, (y1 + y2) / 2+size = max(x2 - x1, y2 - y1) * 1.2+sx1, sy1 = int(cx - size / 2), int(cy - size / 2)+sx2, sy2 = int(cx + size / 2), int(cy + size / 2)+pad = [max(0, -sy1), max(0, sy2 - H), max(0, -sx1), max(0, sx2 - W)]+patch = image[max(0, sy1):min(H, sy2), max(0, sx1):min(W, sx2)]+patch = cv2.copyMakeBorder(patch, *pad, cv2.BORDER_CONSTANT, value=0)+crop = cv2.resize(patch, (224, 224), interpolation=cv2.INTER_LINEAR)++# 4. Normalise + CHW.+img = (crop.astype(np.float32) / 255.0 - IMAGENET_MEAN) / IMAGENET_STD+img = np.transpose(img, (2, 0, 1))[None] # (1, 3, 224, 224)++# 5. CLIFF conditioning vector (full-frame coords).+cliff = np.array([+ 2.0 * cx / W - 1.0,+ 2.0 * cy / H - 1.0,+ max(x2 - x1, y2 - y1) / max(W, H),+], dtype=np.float32)[None] # (1, 3)++# 6. Run the model.+mhr_params, shape_params, cam_trans, joints_2d, joints_3d = sess.run(+ None, {"image": img, "cliff_cond": cliff}+)++print(joints_3d.shape) # (1, 70, 3) — body-centred metres, Y-down+print(joints_2d.shape) # (1, 70, 2) — normalised crop coords [-1, 1]+print(mhr_params.shape) # (1, 204) — feed into Meta's MHR module for a mesh+```++For a full pipeline (RF-DETR person detector + InstantHMR + Rerun+visualiser), use the [`PosePipeline`](https://github.com/mohamdev/InstantHMR/blob/main/instanthmr/pipeline.py)+helper from the main repository — `pip install -r requirements.txt` then+`python demo.py --image path/to/photo.jpg`.++## Body mesh++InstantHMR does **not** run the MHR mesh decoder — rendering the full+mesh is resource-heavy and the default pipeline stays lightweight. The+raw `mhr_params` and `shape_params` are exposed so you can feed them+into Meta's MHR TorchScript module from the original+[SAM 3D Body release](https://github.com/facebookresearch/sam-3d-body)+when you need vertices.++## License++These weights are released under the **SAM license**, since InstantHMR+is a distillation of `facebook/sam-3d-body-dinov3`. Refer to+<https://github.com/facebookresearch/sam-3d-body> for the full license+text and applicable use restrictions.++The accompanying source code in the InstantHMR repository is released+separately under the Apache License 2.0.diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynbnew file mode 100644index 0000000..2d43919--- /dev/null+++ b/notebooks/distill_transformer_decoder.ipynb@@ -0,0 +1,2365 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "ad5fd1be",+ "metadata": {},+ "outputs": [],+ "source": [+ "import torch\n",+ "import torch.nn as nn\n",+ "import torch.nn.functional as F\n",+ "import torch.optim as optim\n",+ "import torchvision.transforms as T\n",+ "import matplotlib.pyplot as plt\n",+ "import timm\n",+ "import cv2\n",+ "from pathlib import Path\n",+ "from tqdm.notebook import tqdm\n",+ "import onnxruntime as ort\n",+ "print(ort.get_available_providers())\n",+ "\n",+ "\n",+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "print(f\"Using device: {device}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "7e2ae437",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 1 — Configuration\n",+ "# ============================================================\n",+ "from dataclasses import dataclass \n",+ "from pathlib import Path\n",+ "\n",+ "PROJECT_ROOT = Path(\"\") #set your path\n",+ "\n",+ "@dataclass\n",+ "class DistillConfig:\n",+ " \"\"\"All hyperparameters in one place.\"\"\"\n",+ " # --- Paths ---\n",+ " images_dir: str = str(PROJECT_ROOT / \"dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",+ " annotations_dir: str = str(PROJECT_ROOT / \"dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",+ " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt\") #path to MHR model\n",+ " log_dir: str = str(PROJECT_ROOT / \"runs/distill_repvit_cliff_v1\") # Bumped to v1!\n",+ "\n",+ " # --- Data ---\n",+ " image_size: int = 224\n",+ " max_images: int = 100_000 \n",+ " val_split: float = 0.1\n",+ " num_workers: int = 4\n",+ " augment: bool = True\n",+ "\n",+ " # --- Model (Transformer + CLIFF) ---\n",+ " backbone: str = \"repvit_m2_3\" \n",+ " backbone_feat_dim: int = 640 # RepViT m2_3 usually outputs 640\n",+ " d_model: int = 512 \n",+ " n_heads: int = 8 \n",+ " n_decoder_layers: int = 4 \n",+ " dropout: float = 0.1\n",+ "\n",+ " # --- Output dimensions ---\n",+ " pose_dim: int = 136 \n",+ " scale_dim: int = 68 \n",+ " shape_dim: int = 45 \n",+ " cam_dim: int = 3 \n",+ " cliff_dim: int = 3 \n",+ " num_joints: int = 70 # We track 70 joints\n",+ "\n",+ " @property\n",+ " def model_params_dim(self) -> int:\n",+ " return self.pose_dim + self.scale_dim # 204\n",+ "\n",+ " # --- Training ---\n",+ " batch_size: int = 64\n",+ " lr: float = 3e-4\n",+ " weight_decay: float = 1e-4\n",+ " epochs: int = 300\n",+ " warmup_epochs: int = 3\n",+ " grad_clip: float = 5.0\n",+ " use_amp: bool = True \n",+ "\n",+ " # --- Loss weights (UPDATED for HMR Balance) ---\n",+ " w_pose: float = 1.0\n",+ " w_scale: float = 0.1\n",+ " w_shape: float = 1.0\n",+ " w_cam: float = 0.1 \n",+ " w_keypoints3d: float = 10.0 # Bumped up: 3D joints are the strongest spatial anchor\n",+ " w_keypoints2d: float = 10.0 # Bumped up: Forces pixel-perfect alignment\n",+ " w_3d_joints: float = 1e-3\n",+ " w_structural: float = 0.01\n",+ " w_reproj_3d: float = 0.01 # High weight to enforce strict scale/camera alignment\n",+ " w_reproj_mhr: float = 0.01 # High weight to enforce strict scale/camera alignment\n",+ "\n",+ " # ==========================================================\n",+ " # Feet Barycenter Anchors\n",+ " # ==========================================================\n",+ " w_feet = 0.01 # Soft prior weight, matching w_structural\n",+ " \n",+ " # Native 70-joint format indices\n",+ " native_left_foot = [15, 16, 17]\n",+ " native_right_foot = [18, 19, 20]\n",+ " \n",+ " # MHR 127-joint format indices\n",+ " mhr_left_foot = [6, 7, 8] \n",+ " mhr_right_foot = [22, 23, 24]\n",+ " \n",+ " # Data-Driven Topology Mapping (Native 70 <-> MHR 127)\n",+ " native_mapping_ids = [ 1, 2, 5, 6, 9, 10, 11, 12, 13, 14, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62 ]\n",+ " mhr_mapping_ids = [ 125, 123, 75, 39, 2, 18, 3, 19, 5, 21, 64, 63, 62, 61, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 41, 100, 99, 98, 97, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 77 ]\n",+ "\n",+ "cfg = DistillConfig()\n",+ "print(f\"Config loaded. Backbone: {cfg.backbone}. Ready for V3 Training.\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "561a65f3",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ==============================================================================\n",+ "# Cell 2 — X-Ray Dataset Visualizer (Math Verification)\n",+ "# ==============================================================================\n",+ "import numpy as np\n",+ "import cv2\n",+ "import torch\n",+ "import matplotlib.pyplot as plt\n",+ "from pathlib import Path\n",+ "import random\n",+ "\n",+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "\n",+ "\n",+ "# --- 1. Load Lightweight MHR Wrapper ---\n",+ "class MHRForwardPass:\n",+ " def __init__(self, mhr_path, device):\n",+ " self.device = device\n",+ " self.mhr = torch.jit.load(mhr_path, map_location=device).eval()\n",+ " for p in self.mhr.parameters():\n",+ " p.requires_grad_(False)\n",+ "\n",+ " @torch.no_grad()\n",+ " def get_joints(self, model_params, shape_params):\n",+ " B = model_params.shape[0]\n",+ " dummy_identity = torch.zeros(B, 45, device=self.device, dtype=model_params.dtype)\n",+ " concat_params = torch.cat([model_params, dummy_identity], dim=1)\n",+ " joint_params = self.mhr.character_torch.model_parameters_to_joint_parameters(concat_params)\n",+ " skel_state = self.mhr.character_torch.joint_parameters_to_skeleton_state(joint_params)\n",+ " return skel_state\n",+ "\n",+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",+ "\n",+ "# --- 2. Load a Random Sample ---\n",+ "npz_files = list(Path(cfg.annotations_dir).glob(\"*.npz\"))\n",+ "sample_npz = random.choice(npz_files)\n",+ "sample_stem = sample_npz.stem\n",+ "\n",+ "img_path = Path(cfg.images_dir) / f\"{sample_stem}.jpg\"\n",+ "if not img_path.exists():\n",+ " img_path = Path(cfg.images_dir) / f\"{sample_stem}.png\"\n",+ "\n",+ "ann = np.load(str(sample_npz))\n",+ "img_bgr = cv2.imread(str(img_path))\n",+ "img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n",+ "\n",+ "# --- 3. Extract Ground Truths ---\n",+ "gt_2d_full = ann[\"joints_2d\"] \n",+ "gt_3d_raw = ann[\"joints_3d\"] \n",+ "model_params = ann[\"mhr_model_params\"] \n",+ "shape_params = ann[\"shape_params\"] \n",+ "cam_trans = ann[\"cam_trans\"] \n",+ "focal_length = ann[\"cam_focal_length\"] \n",+ "orig_shape = ann[\"orig_shape\"] \n",+ "bbox_square = ann[\"bbox_square\"] \n",+ "\n",+ "# --- 4. Forward Pass through MHR with 6D Math Injection ---\n",+ "mp_tensor = torch.from_numpy(model_params).unsqueeze(0).to(device)\n",+ "sp_tensor = torch.from_numpy(shape_params).unsqueeze(0).to(device)\n",+ "raw_mhr_joints = mhr_module.get_joints(mp_tensor, sp_tensor)[0, :, :3].cpu().numpy()\n",+ "\n",+ "# ------------------------------------------------------------------------------\n",+ "# THE HOLY GRAIL MATH: Coordinate Transformations\n",+ "# ------------------------------------------------------------------------------\n",+ "# A. MHR -> Vision Coordinates\n",+ "raw_mhr_joints = raw_mhr_joints / 100.0 \n",+ "raw_mhr_joints[:, 1] = -raw_mhr_joints[:, 1] \n",+ "raw_mhr_joints[:, 2] = -raw_mhr_joints[:, 2] \n",+ "\n",+ "# B. SAM3D 70-Joints -> Camera Space\n",+ "j3d_70_cam = gt_3d_raw + cam_trans \n",+ "\n",+ "# --- 5. Project 3D to Full-Frame 2D ---\n",+ "orig_h, orig_w = orig_shape\n",+ "principal_point = np.array([orig_w / 2.0, orig_h / 2.0])\n",+ "\n",+ "mhr_cam = raw_mhr_joints + cam_trans\n",+ "z_mhr = np.maximum(mhr_cam[:, 2], 0.01)\n",+ "u_mhr_full = (focal_length[0] * mhr_cam[:, 0] / z_mhr) + principal_point[0]\n",+ "v_mhr_full = (focal_length[1] * mhr_cam[:, 1] / z_mhr) + principal_point[1]\n",+ "\n",+ "z_70 = np.maximum(j3d_70_cam[:, 2], 0.01)\n",+ "u_70_full = (focal_length[0] * j3d_70_cam[:, 0] / z_70) + principal_point[0]\n",+ "v_70_full = (focal_length[1] * j3d_70_cam[:, 1] / z_70) + principal_point[1]\n",+ "\n",+ "# --- 6. Map Full-Frame 2D to 224x224 Crop Space ---\n",+ "x1, y1, x2, y2 = bbox_square\n",+ "crop_w, crop_h = x2 - x1, y2 - y1\n",+ "\n",+ "u_mhr_crop = (u_mhr_full - x1) / max(crop_w, 1) * cfg.image_size\n",+ "v_mhr_crop = (v_mhr_full - y1) / max(crop_h, 1) * cfg.image_size\n",+ "\n",+ "u_70_crop = (u_70_full - x1) / max(crop_w, 1) * cfg.image_size\n",+ "v_70_crop = (v_70_full - y1) / max(crop_h, 1) * cfg.image_size\n",+ "\n",+ "gt_u_crop = (gt_2d_full[:, 0] - x1) / max(crop_w, 1) * cfg.image_size\n",+ "gt_v_crop = (gt_2d_full[:, 1] - y1) / max(crop_h, 1) * cfg.image_size\n",+ "\n",+ "# --- 7. Visualization ---\n",+ "fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n",+ "\n",+ "axes[0].imshow(img_rgb)\n",+ "axes[0].scatter(gt_u_crop, gt_v_crop, s=15, c='lime', edgecolors='black', label='Native 2D')\n",+ "axes[0].scatter(u_70_crop, v_70_crop, s=40, facecolors='none', edgecolors='cyan', linewidths=1.5, label='Projected 3D (70pts)')\n",+ "axes[0].set_title(f\"Target 1: Native 2D vs Projected 3D Head\\n{sample_stem}\", fontsize=10)\n",+ "axes[0].legend(loc='upper right')\n",+ "axes[0].axis('off')\n",+ "\n",+ "axes[1].imshow(img_rgb)\n",+ "axes[1].scatter(u_mhr_crop, v_mhr_crop, s=15, c='red', edgecolors='black', marker='X')\n",+ "axes[1].set_title(f\"Target 2: Projected MHR Params\\nMath verification (6D Cycle)\", fontsize=10)\n",+ "axes[1].axis('off')\n",+ "\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "1d5e32ef",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 3 — Dataset and Data Augmentation\n",+ "# ============================================================\n",+ "import os\n",+ "import glob\n",+ "import random\n",+ "import numpy as np\n",+ "from pathlib import Path\n",+ "from PIL import Image\n",+ "\n",+ "import torch\n",+ "from torch.utils.data import Dataset, DataLoader, random_split\n",+ "import torchvision.transforms as transforms\n",+ "import torchvision.transforms.functional as F_t\n",+ "from torchvision.transforms import InterpolationMode\n",+ "\n",+ "class RandomPixelate:\n",+ " \"\"\"Simulates low-quality webcams or low-bandwidth video.\"\"\"\n",+ " def __init__(self, p=0.2, min_res=32, max_res=112):\n",+ " self.p = p\n",+ " self.min_res = min_res \n",+ " self.max_res = max_res\n",+ "\n",+ " def __call__(self, img):\n",+ " if random.random() < self.p:\n",+ " orig_w, orig_h = img.size\n",+ " degraded_h = random.randint(self.min_res, self.max_res)\n",+ " degraded_w = int(orig_w * (degraded_h / orig_h))\n",+ " # Downsample then upsample to create pixelation effect\n",+ " img = F_t.resize(img, [degraded_h, degraded_w], interpolation=InterpolationMode.NEAREST)\n",+ " img = F_t.resize(img, [orig_h, orig_w], interpolation=InterpolationMode.NEAREST)\n",+ " return img\n",+ " \n",+ "\n",+ "class SAM3DStudentDataset(Dataset):\n",+ " def __init__(self, images_dir: str, annotations_dir: str,\n",+ " image_size: int = 224, max_images: int | None = None,\n",+ " augment: bool = True):\n",+ " super().__init__()\n",+ " self.image_size = image_size\n",+ " self.images_dir = Path(images_dir)\n",+ " self.annotations_dir = Path(annotations_dir)\n",+ "\n",+ " # --- 1. Build Transform Pipeline ---\n",+ " # The images on disk are already 224x224, but Resize guarantees safety.\n",+ " tfm_list = [transforms.Resize((image_size, image_size))]\n",+ " \n",+ " if augment:\n",+ " tfm_list.extend([\n",+ " transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n",+ " RandomPixelate(p=0.2, min_res=48, max_res=112), \n",+ " transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2), \n",+ " ])\n",+ " \n",+ " tfm_list.extend([\n",+ " transforms.ToTensor(),\n",+ " # Standard ImageNet Normalization\n",+ " transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",+ " std=[0.229, 0.224, 0.225]),\n",+ " ])\n",+ " \n",+ " if augment:\n",+ " tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))\n",+ " \n",+ " self.transform = transforms.Compose(tfm_list)\n",+ "\n",+ " # --- 2. Pair Discovery ---\n",+ " # Fast indexing of available NPZ files and corresponding images\n",+ " ann_files = list(self.annotations_dir.glob(\"*.npz\"))\n",+ " \n",+ " self.pairs = []\n",+ " for npz_path in sorted(ann_files):\n",+ " stem = npz_path.stem\n",+ " # Check for jpg or png\n",+ " img_path = self.images_dir / f\"{stem}.jpg\"\n",+ " if not img_path.exists():\n",+ " img_path = self.images_dir / f\"{stem}.png\"\n",+ " \n",+ " if img_path.exists():\n",+ " self.pairs.append((img_path, npz_path))\n",+ "\n",+ " if max_images is not None:\n",+ " self.pairs = self.pairs[:max_images]\n",+ "\n",+ " def __len__(self):\n",+ " return len(self.pairs)\n",+ "\n",+ " def __getitem__(self, idx):\n",+ " img_path, npz_path = self.pairs[idx]\n",+ "\n",+ " # --- 3. Load and Augment Image ---\n",+ " image = Image.open(img_path).convert(\"RGB\")\n",+ " image = self.transform(image) # Returns float tensor [3, 224, 224]\n",+ "\n",+ " # --- 4. Load Annotations ---\n",+ " ann = np.load(npz_path)\n",+ " \n",+ " orig_h, orig_w = ann[\"orig_shape\"]\n",+ " \n",+ " # --- 5. Correct Mathematical Rescaling for 2D Joints ---\n",+ " sq_bbox = ann[\"bbox_square\"] \n",+ " sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]\n",+ " orig_crop_size = sq_bbox[2] - sq_bbox[0]\n",+ " \n",+ " # Scale factor from the original crop square to the 224x224 tensor\n",+ " true_scale = self.image_size / max(orig_crop_size, 1.0)\n",+ "\n",+ " # Shift full-frame 2D joints relative to crop top-left, then scale\n",+ " joints_2d = ann[\"joints_2d\"].copy()\n",+ " joints_2d[:, 0] = (joints_2d[:, 0] - sq_x1) * true_scale\n",+ " joints_2d[:, 1] = (joints_2d[:, 1] - sq_y1) * true_scale\n",+ "\n",+ " # NEW: Normalize to [-1, 1] for neural network stability!\n",+ " joints_2d[:, 0] = (joints_2d[:, 0] / self.image_size) * 2.0 - 1.0\n",+ " joints_2d[:, 1] = (joints_2d[:, 1] / self.image_size) * 2.0 - 1.0\n",+ "\n",+ " joints_2d = torch.from_numpy(joints_2d).float()\n",+ "\n",+ " # --- 6. CLIFF Conditioning Vectors ---\n",+ " # Using the tighter YOLO bbox to tell the network where the person actually is\n",+ " tight_bbox = ann[\"bbox\"] \n",+ " cx = (tight_bbox[0] + tight_bbox[2]) / 2.0\n",+ " cy = (tight_bbox[1] + tight_bbox[3]) / 2.0\n",+ " \n",+ " # Normalize to [-1, 1] space for better network stability\n",+ " cx_norm = 2.0 * (cx / orig_w) - 1.0\n",+ " cy_norm = 2.0 * (cy / orig_h) - 1.0\n",+ " \n",+ " # Scale relative to the longest edge of the original image\n",+ " b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])\n",+ " b_scale = b_size / max(orig_w, orig_h)\n",+ " cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)\n",+ "\n",+ " return {\n",+ " \"image\": image, # [3, 224, 224]\n",+ " \"cliff_cond\": cliff_cond, # [3]\n",+ " \"mhr_model_params\": torch.from_numpy(ann[\"mhr_model_params\"]).float(), # [204]\n",+ " \"shape_params\": torch.from_numpy(ann[\"shape_params\"]).float(), # [45]\n",+ " \"cam_trans\": torch.from_numpy(ann[\"cam_trans\"]).float(), # [3]\n",+ " \"cam_focal\": torch.from_numpy(ann[\"cam_focal_length\"]).float(), # [2]\n",+ " \"joints_2d\": joints_2d, # [70, 2] Crop Space\n",+ " \"joints_3d\": torch.from_numpy(ann[\"joints_3d\"]).float(), # [70, 3] Root-centered\n",+ " \"orig_shape\": torch.tensor([orig_h, orig_w], dtype=torch.float32), # [2]\n",+ " \"bbox_square\": sq_bbox\n",+ " }\n",+ "\n",+ "# --- Initialization & Dataloaders ---\n",+ "full_dataset = SAM3DStudentDataset(\n",+ " cfg.images_dir, \n",+ " cfg.annotations_dir, \n",+ " augment=cfg.augment, \n",+ " max_images=cfg.max_images\n",+ ")\n",+ "\n",+ "val_dataset_clean = SAM3DStudentDataset(\n",+ " cfg.images_dir, \n",+ " cfg.annotations_dir, \n",+ " augment=False, \n",+ " max_images=cfg.max_images\n",+ ")\n",+ "\n",+ "n_val = int(len(full_dataset) * cfg.val_split)\n",+ "n_train = len(full_dataset) - n_val\n",+ "\n",+ "# Use a generator for reproducible splits\n",+ "generator = torch.Generator().manual_seed(42)\n",+ "train_dataset, _ = random_split(full_dataset, [n_train, n_val], generator=generator)\n",+ "\n",+ "generator = torch.Generator().manual_seed(42)\n",+ "_, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=generator)\n",+ "\n",+ "train_loader = DataLoader(\n",+ " train_dataset, \n",+ " batch_size=cfg.batch_size, \n",+ " shuffle=True, \n",+ " num_workers=cfg.num_workers, \n",+ " pin_memory=True, \n",+ " drop_last=True\n",+ ")\n",+ "\n",+ "val_loader = DataLoader(\n",+ " val_dataset, \n",+ " batch_size=cfg.batch_size, \n",+ " shuffle=False, \n",+ " num_workers=cfg.num_workers, \n",+ " pin_memory=True\n",+ ")\n",+ "\n",+ "\n",+ "print(f\"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "0f003098",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ==============================================================================\n",+ "# Cell 4 — CLIFF & Augmentation Sanity Check\n",+ "# ==============================================================================\n",+ "import matplotlib.pyplot as plt\n",+ "import cv2\n",+ "\n",+ "# 1. Grab a single batch from the DataLoader\n",+ "sample_batch = next(iter(train_loader))\n",+ "\n",+ "# Extract the first item in the batch\n",+ "idx = 0\n",+ "img_tensor = sample_batch[\"image\"][idx]\n",+ "# Un-normalize the 2D joints for visualization ---\n",+ "joints_2d_norm = sample_batch[\"joints_2d\"][idx].numpy() # Currently in [-1, 1] space\n",+ "joints_2d_crop = ((joints_2d_norm + 1.0) / 2.0) * cfg.image_size # Map back to [0, 224] pixel space\n",+ "\n",+ "# Get MHR parameters and Camera data\n",+ "mhr_params = sample_batch[\"mhr_model_params\"][idx:idx+1].to(device)\n",+ "shape_params = sample_batch[\"shape_params\"][idx:idx+1].to(device)\n",+ "cam_trans = sample_batch[\"cam_trans\"][idx].numpy()\n",+ "cam_focal = sample_batch[\"cam_focal\"][idx].numpy() # Original UNTOUCHED focal length\n",+ "\n",+ "# Get CLIFF Conditioning & Geometry data\n",+ "cliff_cond = sample_batch[\"cliff_cond\"][idx].numpy()\n",+ "orig_shape = sample_batch[\"orig_shape\"][idx].numpy() # [H, W]\n",+ "bbox_square = sample_batch[\"bbox_square\"][idx].numpy() # [x1, y1, x2, y2]\n",+ "\n",+ "orig_h, orig_w = orig_shape[0], orig_shape[1]\n",+ "\n",+ "# ------------------------------------------------------------------------------\n",+ "# SANITY CHECK 1: Reverse-Engineer CLIFF Conditioning\n",+ "# ------------------------------------------------------------------------------\n",+ "# cliff_cond = [cx_norm, cy_norm, scale]\n",+ "cx_full = ((cliff_cond[0] + 1.0) / 2.0) * orig_w\n",+ "cy_full = ((cliff_cond[1] + 1.0) / 2.0) * orig_h\n",+ "box_size_full = cliff_cond[2] * max(orig_h, orig_w)\n",+ "\n",+ "print(\"--- CLIFF Conditioning Verification ---\")\n",+ "print(f\"Original Image Size: {orig_w}x{orig_h}\")\n",+ "print(f\"BBox Center in Full Image: X={cx_full:.1f}, Y={cy_full:.1f}\")\n",+ "print(f\"BBox Max Dimension: {box_size_full:.1f} pixels\")\n",+ "print(f\"Does this look like a human bounding box? (Yes, if center is within image bounds)\")\n",+ "print(\"---------------------------------------\")\n",+ "\n",+ "# 2. Un-normalize the image tensor for viewing\n",+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",+ "img_unnorm = img_tensor * std + mean\n",+ "img_np = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",+ "\n",+ "# 3. Generate 3D Joints using MHR\n",+ "raw_3d_joints = mhr_module.get_joints(mhr_params, shape_params)[0, :, :3].cpu().numpy()\n",+ "raw_3d_joints = raw_3d_joints / 100.0 # cm to meters\n",+ "raw_3d_joints[:, 1] = -raw_3d_joints[:, 1] # Y-down convention\n",+ "raw_3d_joints[:, 2] = -raw_3d_joints[:, 2] # Z-forward convention\n",+ "\n",+ "# ------------------------------------------------------------------------------\n",+ "# SANITY CHECK 2: The CLIFF Projection Math\n",+ "# ------------------------------------------------------------------------------\n",+ "# A. Project natively into the FULL FRAME space\n",+ "j3d_cam = raw_3d_joints + cam_trans\n",+ "z = np.maximum(j3d_cam[:, 2], 0.01)\n",+ "\n",+ "principal_point_x = orig_w / 2.0\n",+ "principal_point_y = orig_h / 2.0\n",+ "\n",+ "u_full = (cam_focal[0] * j3d_cam[:, 0] / z) + principal_point_x\n",+ "v_full = (cam_focal[1] * j3d_cam[:, 1] / z) + principal_point_y\n",+ "\n",+ "# B. Map the full-frame 2D points into the 224x224 crop space\n",+ "sq_x1, sq_y1, sq_x2, sq_y2 = bbox_square\n",+ "crop_w = sq_x2 - sq_x1\n",+ "crop_h = sq_y2 - sq_y1\n",+ "true_scale = 224.0 / max(crop_w, 1.0)\n",+ "\n",+ "u_crop_proj = (u_full - sq_x1) * true_scale\n",+ "v_crop_proj = (v_full - sq_y1) * true_scale\n",+ "\n",+ "# 4. Visualize\n",+ "fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n",+ "\n",+ "# Plot 1: The Native 2D labels from the dataset (Checks augmentation/mapping)\n",+ "axes[0].imshow(img_np)\n",+ "axes[0].scatter(joints_2d_crop[:, 0], joints_2d_crop[:, 1], s=15, c='lime', edgecolors='black', label='Native 2D')\n",+ "axes[0].set_title(f\"Target 1: Native 2D Joints\\n(Verifies Augmentations & Crop mapping)\", fontsize=10)\n",+ "axes[0].legend(loc='upper right')\n",+ "axes[0].axis('off')\n",+ "\n",+ "# Plot 2: The 3D -> 2D CLIFF Projection (Checks focal length & cam_trans)\n",+ "axes[1].imshow(img_np)\n",+ "axes[1].scatter(u_crop_proj, v_crop_proj, s=15, c='red', edgecolors='black', marker='X')\n",+ "axes[1].set_title(f\"Target 2: CLIFF Projected 3D MHR\\n(Verifies Full-Frame Camera Math)\", fontsize=10)\n",+ "axes[1].axis('off')\n",+ "\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "47dea531",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ==============================================================================\n",+ "# Cell 5 — Data-Driven Topology Discovery (Pixel-Space Edition - FIXED)\n",+ "# ==============================================================================\n",+ "import torch\n",+ "import numpy as np\n",+ "import matplotlib.pyplot as plt\n",+ "import matplotlib.lines as mlines\n",+ "from tqdm import tqdm\n",+ "\n",+ "print(\"--- Starting 2D Pixel-Space Topology Discovery ---\")\n",+ "\n",+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",+ "\n",+ "# --- Configuration ---\n",+ "NUM_BATCHES_TO_SCAN = 100 \n",+ "PIXEL_THRESHOLD = 1.0 # 15 pixels is a great sweet spot for 224x224 anatomical matches\n",+ "\n",+ "cumulative_dist_matrix = torch.zeros((cfg.num_joints, 127), device=device)\n",+ "total_samples = 0\n",+ "\n",+ "print(f\"Scanning {NUM_BATCHES_TO_SCAN} batches to compute average 2D pixel distances...\")\n",+ "train_iter = iter(train_loader)\n",+ "\n",+ "for _ in tqdm(range(NUM_BATCHES_TO_SCAN)):\n",+ " batch = next(train_iter)\n",+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " B = batch[\"mhr_model_params\"].shape[0]\n",+ " \n",+ " # 1. Un-normalize Native 2D to Crop Space [0, 224]\n",+ " native_2d_norm = batch[\"joints_2d\"][..., :2] \n",+ " native_2d_crop = ((native_2d_norm + 1.0) / 2.0) * cfg.image_size \n",+ " \n",+ " # 2. Extract 3D MHR Joints\n",+ " with torch.no_grad():\n",+ " mhr_3d_raw = mhr_module.get_joints(batch[\"mhr_model_params\"], batch[\"shape_params\"])[..., :3]\n",+ " \n",+ " mhr_3d = mhr_3d_raw / 100.0 \n",+ " mhr_3d[..., 1] = -mhr_3d[..., 1] \n",+ " mhr_3d[..., 2] = -mhr_3d[..., 2] \n",+ " \n",+ " # 3. Apply CLIFF Projection Math (Batched properly!)\n",+ " cam_trans = batch[\"cam_trans\"].unsqueeze(1) # [B, 1, 3]\n",+ " j3d_cam = mhr_3d + cam_trans # [B, 127, 3]\n",+ " z = torch.clamp(j3d_cam[..., 2], min=0.01) # [B, 127]\n",+ " \n",+ " # FIXED: Removed the buggy .unsqueeze(1) so shapes match [B, 1] * [B, 127]\n",+ " orig_shape = batch[\"orig_shape\"]\n",+ " orig_w, orig_h = orig_shape[:, 1:2], orig_shape[:, 0:1] # [B, 1]\n",+ " cx, cy = orig_w / 2.0, orig_h / 2.0 # [B, 1]\n",+ " \n",+ " focal_key = \"cam_focal\" if \"cam_focal\" in batch else \"cam_focal_length\"\n",+ " cam_focal = batch[focal_key]\n",+ " fx, fy = cam_focal[:, 0:1], cam_focal[:, 1:2] # [B, 1]\n",+ " \n",+ " u_full = (fx * j3d_cam[..., 0] / z) + cx # [B, 127]\n",+ " v_full = (fy * j3d_cam[..., 1] / z) + cy # [B, 127]\n",+ " \n",+ " bbox = batch[\"bbox_square\"]\n",+ " sq_x1, sq_y1, sq_x2 = bbox[:, 0:1], bbox[:, 1:2], bbox[:, 2:3] # [B, 1]\n",+ " crop_w = torch.clamp(sq_x2 - sq_x1, min=1.0) # [B, 1]\n",+ " true_scale = cfg.image_size / crop_w # [B, 1]\n",+ " \n",+ " u_crop = (u_full - sq_x1) * true_scale # [B, 127]\n",+ " v_crop = (v_full - sq_y1) * true_scale # [B, 127]\n",+ " mhr_2d_crop = torch.stack([u_crop, v_crop], dim=-1) # [B, 127, 2]\n",+ " \n",+ " # 4. Compute Pairwise Distance in Pixel Space!\n",+ " dist_matrix = torch.cdist(native_2d_crop, mhr_2d_crop) # [B, 70, 127]\n",+ " cumulative_dist_matrix += dist_matrix.sum(dim=0)\n",+ " total_samples += B\n",+ "\n",+ "# Average the distance matrix\n",+ "avg_dist_matrix = cumulative_dist_matrix / total_samples\n",+ "\n",+ "# --- Extract the Mapping ---\n",+ "min_dists, closest_mhr_indices = avg_dist_matrix.min(dim=1)\n",+ "\n",+ "valid_mask = min_dists < PIXEL_THRESHOLD\n",+ "native_mapping_ids = torch.arange(70, device=device)[valid_mask].cpu().numpy()\n",+ "mhr_mapping_ids = closest_mhr_indices[valid_mask].cpu().numpy()\n",+ "matched_distances = min_dists[valid_mask].cpu().numpy()\n",+ "\n",+ "native_ids_2d, mhr_ids_2d = native_mapping_ids, mhr_mapping_ids \n",+ "print(\"\\n--- Discovery Complete ---\")\n",+ "print(f\"Found {len(native_mapping_ids)} valid anchor pairs with average distance < {PIXEL_THRESHOLD} pixels.\")\n",+ "\n",+ "# ==============================================================================\n",+ "# VISUALIZATION: Plotting the Anchors on a Sample\n",+ "# ==============================================================================\n",+ "sample_idx = 0\n",+ "nat_2d_viz = native_2d_crop[sample_idx].cpu().numpy()\n",+ "mhr_2d_viz = mhr_2d_crop[sample_idx].cpu().numpy()\n",+ "\n",+ "# Un-normalize image\n",+ "img_tensor = batch[\"image\"][sample_idx].cpu()\n",+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",+ "img_unnorm = img_tensor * std + mean\n",+ "img_rgb = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",+ "\n",+ "fig, ax = plt.subplots(figsize=(8, 8))\n",+ "ax.imshow(img_rgb)\n",+ "ax.set_title(f\"Data-Driven Anchor Mapping (Thresh: {PIXEL_THRESHOLD} px)\", fontsize=14)\n",+ "\n",+ "# Plot all joints faintly\n",+ "ax.scatter(nat_2d_viz[:, 0], nat_2d_viz[:, 1], s=20, c='cyan', alpha=0.3, label='All Native')\n",+ "ax.scatter(mhr_2d_viz[:, 0], mhr_2d_viz[:, 1], s=20, c='red', alpha=0.3, label='All MHR')\n",+ "\n",+ "# Draw connections for the valid mapped anchors\n",+ "for n_idx, m_idx in zip(native_mapping_ids, mhr_mapping_ids):\n",+ " ax.scatter(nat_2d_viz[n_idx, 0], nat_2d_viz[n_idx, 1], s=50, c='lime', edgecolors='black', zorder=5)\n",+ " ax.scatter(mhr_2d_viz[m_idx, 0], mhr_2d_viz[m_idx, 1], s=50, c='yellow', marker='X', edgecolors='black', zorder=5)\n",+ " \n",+ " line = mlines.Line2D([nat_2d_viz[n_idx, 0], mhr_2d_viz[m_idx, 0]], \n",+ " [nat_2d_viz[n_idx, 1], mhr_2d_viz[m_idx, 1]], \n",+ " color='magenta', linewidth=1.5, zorder=4)\n",+ " ax.add_line(line)\n",+ "\n",+ "# Clean up legend\n",+ "handles, labels = ax.get_legend_handles_labels()\n",+ "match_marker = mlines.Line2D([], [], color='magenta', marker='o', markersize=8, markerfacecolor='lime', label='Discovered Anchors')\n",+ "ax.legend(handles=[handles[0], handles[1], match_marker], loc='upper right')\n",+ "ax.axis('off')\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "af8c0277",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ==============================================================================\n",+ "# Cell 6 — Data-Driven Topology Discovery (3D Meter Space - FIXED ALIGNMENT)\n",+ "# ==============================================================================\n",+ "import torch\n",+ "import numpy as np\n",+ "import matplotlib.pyplot as plt\n",+ "from mpl_toolkits.mplot3d import Axes3D\n",+ "import matplotlib.lines as mlines\n",+ "from tqdm import tqdm\n",+ "\n",+ "print(\"--- Starting 3D Topology Discovery ---\")\n",+ "\n",+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",+ "\n",+ "# --- Configuration ---\n",+ "NUM_BATCHES_TO_SCAN = 100 \n",+ "# 4 cm is a rigorous but realistic threshold for overlapping anatomical joints\n",+ "DISTANCE_THRESHOLD_METERS = 0.01 \n",+ "\n",+ "cumulative_dist_matrix = torch.zeros((cfg.num_joints, 127), device=device)\n",+ "total_samples = 0\n",+ "\n",+ "print(f\"Scanning {NUM_BATCHES_TO_SCAN} batches to compute true 3D distances...\")\n",+ "train_iter = iter(train_loader)\n",+ "\n",+ "for _ in tqdm(range(NUM_BATCHES_TO_SCAN)):\n",+ " batch = next(train_iter)\n",+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " B = batch[\"mhr_model_params\"].shape[0]\n",+ " \n",+ " # 1. Extract 3D Native Joints (Body-Centered, Meters, Y-Down, Z-Forward)\n",+ " native_3d = batch[\"joints_3d\"][..., :3] \n",+ " \n",+ " # 2. Extract 3D MHR Joints (Internal coords, cm)\n",+ " with torch.no_grad():\n",+ " mhr_3d_raw = mhr_module.get_joints(batch[\"mhr_model_params\"], batch[\"shape_params\"])[..., :3]\n",+ " \n",+ " # Convert MHR to the exact same frame as Native 3D!\n",+ " mhr_3d = mhr_3d_raw / 100.0 \n",+ " mhr_3d[..., 1] = -mhr_3d[..., 1] \n",+ " mhr_3d[..., 2] = -mhr_3d[..., 2] \n",+ " \n",+ " # NO MEAN CENTERING! They are already in the same coordinate space!\n",+ " \n",+ " # 3. Compute Pairwise Distance in 3D METER Space [B, 70, 127]\n",+ " dist_matrix = torch.cdist(native_3d, mhr_3d) \n",+ " cumulative_dist_matrix += dist_matrix.sum(dim=0)\n",+ " total_samples += B\n",+ "\n",+ "# Average the distance matrix\n",+ "avg_dist_matrix = cumulative_dist_matrix / total_samples\n",+ "\n",+ "# --- Extract the Mapping ---\n",+ "min_dists, closest_mhr_indices = avg_dist_matrix.min(dim=1)\n",+ "\n",+ "valid_mask = min_dists < DISTANCE_THRESHOLD_METERS\n",+ "native_mapping_ids = torch.arange(70, device=device)[valid_mask].cpu().numpy()\n",+ "mhr_mapping_ids = closest_mhr_indices[valid_mask].cpu().numpy()\n",+ "matched_distances = min_dists[valid_mask].cpu().numpy()\n",+ "\n",+ "native_ids_3d, mhr_ids_3d = native_mapping_ids, mhr_mapping_ids \n",+ "print(\"\\n--- Discovery Complete ---\")\n",+ "print(f\"Found {len(native_mapping_ids)} valid anchor pairs with average distance < {DISTANCE_THRESHOLD_METERS} meters.\")\n",+ "\n",+ "# ==============================================================================\n",+ "# VISUALIZATION: Interactive 3D Plot with Plotly\n",+ "# ==============================================================================\n",+ "import plotly.graph_objects as go\n",+ "import numpy as np\n",+ "\n",+ "sample_idx = 0\n",+ "# Extract raw coordinates\n",+ "nat_3d_viz = native_3d[sample_idx].cpu().numpy()\n",+ "mhr_3d_viz = mhr_3d[sample_idx].cpu().numpy()\n",+ "\n",+ "# To make the person stand upright in Plotly (where Z is usually up):\n",+ "# Data X -> Plotly X\n",+ "# Data Z (Forward) -> Plotly Y (Depth)\n",+ "# Data -Y (Down) -> Plotly Z (Height)\n",+ "def map_coords(pts):\n",+ " return pts[:, 0], pts[:, 2], -pts[:, 1]\n",+ "\n",+ "nx, ny, nz = map_coords(nat_3d_viz)\n",+ "mx, my, mz = map_coords(mhr_3d_viz)\n",+ "\n",+ "fig = go.Figure()\n",+ "\n",+ "# 1. Plot Unmatched Native Joints (Cyan)\n",+ "fig.add_trace(go.Scatter3d(\n",+ " x=nx, y=ny, z=nz,\n",+ " mode='markers',\n",+ " marker=dict(size=4, color='cyan', opacity=0.4),\n",+ " name='All Native (70)',\n",+ " text=[f\"Native Joint: {i}\" for i in range(70)],\n",+ " hoverinfo='text'\n",+ "))\n",+ "\n",+ "# 2. Plot Unmatched MHR Joints (Red)\n",+ "fig.add_trace(go.Scatter3d(\n",+ " x=mx, y=my, z=mz,\n",+ " mode='markers',\n",+ " marker=dict(size=4, color='red', opacity=0.4),\n",+ " name='All MHR (127)',\n",+ " text=[f\"MHR Joint: {i}\" for i in range(127)],\n",+ " hoverinfo='text'\n",+ "))\n",+ "\n",+ "# 3. Plot Matched Anchors\n",+ "# Extract coordinates for the matched pairs\n",+ "nat_anchors_x, nat_anchors_y, nat_anchors_z = nx[native_mapping_ids], ny[native_mapping_ids], nz[native_mapping_ids]\n",+ "mhr_anchors_x, mhr_anchors_y, mhr_anchors_z = mx[mhr_mapping_ids], my[mhr_mapping_ids], mz[mhr_mapping_ids]\n",+ "\n",+ "# Native Anchors (Green)\n",+ "fig.add_trace(go.Scatter3d(\n",+ " x=nat_anchors_x, y=nat_anchors_y, z=nat_anchors_z,\n",+ " mode='markers',\n",+ " marker=dict(size=7, color='lime', line=dict(color='black', width=1)),\n",+ " name='Native Anchors',\n",+ " text=[f\"Matched Native: {i}\" for i in native_mapping_ids],\n",+ " hoverinfo='text'\n",+ "))\n",+ "\n",+ "# MHR Anchors (Yellow)\n",+ "fig.add_trace(go.Scatter3d(\n",+ " x=mhr_anchors_x, y=mhr_anchors_y, z=mhr_anchors_z,\n",+ " mode='markers',\n",+ " marker=dict(size=7, color='yellow', symbol='cross', line=dict(color='black', width=1)),\n",+ " name='MHR Anchors',\n",+ " text=[f\"Matched MHR: {i}\" for i in mhr_mapping_ids],\n",+ " hoverinfo='text'\n",+ "))\n",+ "\n",+ "# 4. Draw Connection Lines (Magenta)\n",+ "# To draw multiple disconnected lines efficiently in Plotly, we interleave with None\n",+ "line_x, line_y, line_z = [], [], []\n",+ "for i in range(len(native_mapping_ids)):\n",+ " line_x.extend([nat_anchors_x[i], mhr_anchors_x[i], None])\n",+ " line_y.extend([nat_anchors_y[i], mhr_anchors_y[i], None])\n",+ " line_z.extend([nat_anchors_z[i], mhr_anchors_z[i], None])\n",+ "\n",+ "fig.add_trace(go.Scatter3d(\n",+ " x=line_x, y=line_y, z=line_z,\n",+ " mode='lines',\n",+ " line=dict(color='magenta', width=4),\n",+ " name='Physical Distance',\n",+ " hoverinfo='none'\n",+ "))\n",+ "\n",+ "# 5. Layout and Camera Setup\n",+ "fig.update_layout(\n",+ " title=f\"Interactive 3D Anchor Mapping (Thresh: {DISTANCE_THRESHOLD_METERS*100:.1f} cm)\",\n",+ " scene=dict(\n",+ " xaxis_title='X (Meters)',\n",+ " yaxis_title='Z Depth (Meters)',\n",+ " zaxis_title='Y Height (Meters)',\n",+ " aspectmode='data' # THIS IS CRITICAL: Keeps the human proportions physically accurate!\n",+ " ),\n",+ " margin=dict(l=0, r=0, b=0, t=40),\n",+ " legend=dict(x=0.8, y=0.9)\n",+ ")\n",+ "\n",+ "fig.show()\n",+ "\n",+ "print(\"--- Anchor Verification: 2D vs 3D ---\")\n",+ "\n",+ "# 1. Create sets of tuples for easy comparison\n",+ "mapping_2d = set(zip(native_ids_2d, mhr_ids_2d))\n",+ "mapping_3d = set(zip(native_ids_3d, mhr_ids_3d))\n",+ "\n",+ "# 2. Check for perfect agreement\n",+ "if mapping_2d == mapping_3d:\n",+ " print(\"✅ PERFECT MATCH! Both 2D and 3D discovery found the exact same 52 anchor pairs.\")\n",+ "else:\n",+ " print(\"⚠️ MISMATCH DETECTED. The lists differ slightly.\")\n",+ " \n",+ " # Find what's unique to 2D\n",+ " only_in_2d = mapping_2d - mapping_3d\n",+ " if only_in_2d:\n",+ " print(f\"\\nPairs found ONLY in 2D ({len(only_in_2d)}):\")\n",+ " for nat, mhr in sorted(only_in_2d):\n",+ " print(f\" Native [{nat:02d}] <--> MHR [{mhr:03d}]\")\n",+ " \n",+ " # Find what's unique to 3D\n",+ " only_in_3d = mapping_3d - mapping_2d\n",+ " if only_in_3d:\n",+ " print(f\"\\nPairs found ONLY in 3D ({len(only_in_3d)}):\")\n",+ " for nat, mhr in sorted(only_in_3d):\n",+ " print(f\" Native [{nat:02d}] <--> MHR [{mhr:03d}]\")\n",+ "\n",+ "# 3. Print the final agreed-upon list to copy into your config!\n",+ "agreed_mapping = sorted(list(mapping_2d.intersection(mapping_3d)))\n",+ "\n",+ "print(\"\\n--- Final Verified Anchor Dictionary ---\")\n",+ "print(\"Copy this into your DistillConfig:\")\n",+ "print(\"native_mapping_ids = [\", \", \".join(str(pair[0]) for pair in agreed_mapping), \"]\")\n",+ "print(\"mhr_mapping_ids = [\", \", \".join(str(pair[1]) for pair in agreed_mapping), \"]\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "10f2586d",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 7 — Student Architecture (Mini-SAM3D with PE)\n",+ "# ============================================================\n",+ "import torch\n",+ "import torch.nn as nn\n",+ "import timm\n",+ "\n",+ "def get_2d_sincos_pos_embed(embed_dim, grid_size):\n",+ " \"\"\"Generate 2D sine-cosine positional embedding for preserving spatial topology.\"\"\"\n",+ " grid_h, grid_w = grid_size, grid_size\n",+ " grid_y, grid_x = torch.meshgrid(torch.arange(grid_h), torch.arange(grid_w), indexing='ij')\n",+ " \n",+ " omega = torch.arange(embed_dim // 4).float() / (embed_dim // 4)\n",+ " omega = 1.0 / (10000 ** omega)\n",+ " \n",+ " out_y = grid_y.flatten().unsqueeze(1) * omega.unsqueeze(0)\n",+ " out_x = grid_x.flatten().unsqueeze(1) * omega.unsqueeze(0)\n",+ " \n",+ " pe_y = torch.cat([torch.sin(out_y), torch.cos(out_y)], dim=1)\n",+ " pe_x = torch.cat([torch.sin(out_x), torch.cos(out_x)], dim=1)\n",+ " \n",+ " pos_embed = torch.cat([pe_y, pe_x], dim=1) # (H*W, embed_dim)\n",+ " return pos_embed.unsqueeze(0) # (1, H*W, embed_dim)\n",+ "\n",+ "\n",+ "class InstantHMRStudent(nn.Module):\n",+ " def __init__(self, cfg, pretrained=True):\n",+ " super().__init__()\n",+ " self.cfg = cfg\n",+ " \n",+ " # 1. The Vision Backbone (RepViT)\n",+ " self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)\n",+ " embed_dim = self.backbone.num_features \n",+ " self.feat_proj = nn.Linear(embed_dim, cfg.d_model)\n",+ " \n",+ " # --- NEW: Positional Embedding for Transformer Memory ---\n",+ " # RepViT naturally downsamples by 32x. For a 224x224 image, the grid is 7x7.\n",+ " self.grid_size = cfg.image_size // 32\n",+ " pos_embed = get_2d_sincos_pos_embed(cfg.d_model, self.grid_size)\n",+ " # register_buffer ensures it moves to GPU automatically but isn't updated by the optimizer\n",+ " self.register_buffer(\"mem_pos_embed\", pos_embed) \n",+ " \n",+ " # 2. Learnable Query Tokens (1 Global + 70 2D + 70 3D = 141 Queries)\n",+ " self.num_global = 1\n",+ " self.num_2d = cfg.num_joints\n",+ " self.num_3d = cfg.num_joints\n",+ " self.total_queries = self.num_global + self.num_2d + self.num_3d\n",+ " \n",+ " # self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model))\n",+ " self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model) * 0.02)\n",+ " \n",+ " # CLIFF Conditioning Injector\n",+ " self.cond_proj = nn.Sequential(\n",+ " nn.Linear(cfg.cliff_dim, 128),\n",+ " nn.GELU(),\n",+ " nn.Linear(128, cfg.d_model)\n",+ " )\n",+ " \n",+ " # 3. Transformer Cross-Attention Decoder\n",+ " decoder_layer = nn.TransformerDecoderLayer(\n",+ " d_model=cfg.d_model,\n",+ " nhead=cfg.n_heads,\n",+ " dim_feedforward=cfg.d_model * 4,\n",+ " dropout=cfg.dropout,\n",+ " activation=\"gelu\",\n",+ " batch_first=True,\n",+ " norm_first=True\n",+ " )\n",+ " self.transformer = nn.TransformerDecoder(decoder_layer, num_layers=cfg.n_decoder_layers)\n",+ " \n",+ " # 4. The Output Heads\n",+ " \n",+ " # Head A: Global Head (Token 0 -> MHR Params + Shape Params + Camera)\n",+ " self.global_out_dim = cfg.model_params_dim + cfg.shape_dim + cfg.cam_dim\n",+ " self.head_global = nn.Linear(cfg.d_model, self.global_out_dim)\n",+ " \n",+ " # --- Safely Split 2D Head ---\n",+ " self.head_2d_feat = nn.Sequential(\n",+ " nn.Linear(cfg.d_model, 256),\n",+ " nn.GELU()\n",+ " )\n",+ " self.head_2d_xy = nn.Sequential(\n",+ " nn.Linear(256, 2)\n",+ " # nn.Tanh() # Bounds predictions to [-1, 1]!\n",+ " )\n",+ " if getattr(cfg, 'learn_uncertainty', False):\n",+ " # Uncertainty must remain unbounded!\n",+ " self.head_2d_unc = nn.Linear(256, 1) \n",+ " \n",+ " # Head C: 3D Keypoint Head (No Tanh here, 3D meters can exceed [-1, 1])\n",+ " self.dim_3d = 4 if getattr(cfg, 'learn_uncertainty', False) else 3\n",+ " self.head_3d = nn.Sequential(\n",+ " nn.Linear(cfg.d_model, 256),\n",+ " nn.GELU(),\n",+ " nn.Linear(256, self.dim_3d)\n",+ " )\n",+ " \n",+ " # Initialize camera Z translation\n",+ " nn.init.constant_(self.head_global.bias[-1], 2.0)\n",+ " nn.init.constant_(self.head_global.bias[-2], 0.0)\n",+ " nn.init.constant_(self.head_global.bias[-3], 0.0)\n",+ "\n",+ " # ==========================================================\n",+ " # HMR-Standard Zero-Initialization for Stability\n",+ " # ==========================================================\n",+ " # 1. Global Head (MHR Params) - Must start near neutral pose!\n",+ " nn.init.normal_(self.head_global.weight, mean=0.0, std=1e-4)\n",+ " # Zero out the biases (except the last 3 camera params we just set)\n",+ " nn.init.constant_(self.head_global.bias[:-3], 0.0)\n",+ " \n",+ " # 2. 2D Spatial Head - Start with tiny weights\n",+ " nn.init.normal_(self.head_2d_xy[0].weight, mean=0.0, std=1e-4)\n",+ " nn.init.constant_(self.head_2d_xy[0].bias, 0.0)\n",+ " \n",+ " # 3. 3D Spatial Head - Start with tiny weights\n",+ " nn.init.normal_(self.head_3d[-1].weight, mean=0.0, std=1e-4)\n",+ " nn.init.constant_(self.head_3d[-1].bias, 0.0)\n",+ "\n",+ " def forward(self, images, cliff_cond):\n",+ " B = images.shape[0]\n",+ " \n",+ " # 1. Extract spatial image features\n",+ " img_feats = self.backbone.forward_features(images)\n",+ " \n",+ " # Defensive flattening \n",+ " if img_feats.dim() == 4:\n",+ " img_feats = img_feats.flatten(2).transpose(1, 2) # [B, H*W, C]\n",+ " \n",+ " memory = self.feat_proj(img_feats) \n",+ " \n",+ " # --- NEW: Inject Spatial Topology into Memory ---\n",+ " # Broadcasting [1, H*W, d_model] across Batch size\n",+ " memory = memory + self.mem_pos_embed \n",+ " \n",+ " # 2. Prepare Query Tokens [B, 141, d_model]\n",+ " queries = self.query_embed.expand(B, -1, -1)\n",+ " \n",+ " cond = self.cond_proj(cliff_cond).unsqueeze(1)\n",+ " queries = queries + cond\n",+ " \n",+ " # 3. Cross Attention\n",+ " shared_feats = self.transformer(tgt=queries, memory=memory)\n",+ " \n",+ " # 4. Route tokens to their dedicated heads\n",+ " feat_global = shared_feats[:, 0, :] \n",+ " feat_2d = shared_feats[:, 1 : 1 + self.num_2d, :] \n",+ " feat_3d = shared_feats[:, 1 + self.num_2d :, :] \n",+ " \n",+ " # --- Decode Global Token ---\n",+ " global_preds = self.head_global(feat_global) \n",+ " \n",+ " idx_mhr = self.cfg.model_params_dim\n",+ " idx_shape = idx_mhr + self.cfg.shape_dim\n",+ " \n",+ " pred_mhr_params = global_preds[:, :idx_mhr]\n",+ " pred_shape_params = global_preds[:, idx_mhr : idx_shape]\n",+ " pred_cam_trans = global_preds[:, idx_shape :]\n",+ " \n",+ " # --- Decode 2D Spatial Tokens (Safely!) ---\n",+ " feat_2d_processed = self.head_2d_feat(feat_2d)\n",+ " pred_xy = self.head_2d_xy(feat_2d_processed) # [B, 70, 2] bounded by Tanh\n",+ " \n",+ " if getattr(self.cfg, 'learn_uncertainty', False):\n",+ " pred_unc = self.head_2d_unc(feat_2d_processed) # [B, 70, 1] unbounded\n",+ " pred_joints_2d = torch.cat([pred_xy, pred_unc], dim=-1) # [B, 70, 3]\n",+ " else:\n",+ " pred_joints_2d = pred_xy\n",+ " \n",+ " # --- Decode 3D Spatial Tokens ---\n",+ " pred_joints_3d = self.head_3d(feat_3d) \n",+ " \n",+ " return {\n",+ " \"mhr_params\": pred_mhr_params,\n",+ " \"shape_params\": pred_shape_params,\n",+ " \"cam_trans\": pred_cam_trans,\n",+ " \"joints_2d\": pred_joints_2d,\n",+ " \"joints_3d\": pred_joints_3d\n",+ " }\n",+ "\n",+ "# --- Quick Architecture Sanity Check ---\n",+ "if __name__ == \"__main__\":\n",+ " test_model = InstantHMRStudent(cfg, pretrained=False)\n",+ " dummy_img = torch.randn(2, 3, 224, 224)\n",+ " dummy_cliff = torch.randn(2, 3)\n",+ " \n",+ " out = test_model(dummy_img, dummy_cliff)\n",+ " print(\"--- Architecture Output Shapes ---\")\n",+ " for k, v in out.items():\n",+ " print(f\"{k}: {v.shape}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "9239175a",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 8 — The Distillation Loss Module (Foolproof Edition)\n",+ "# ============================================================\n",+ "import torch\n",+ "import torch.nn as nn\n",+ "\n",+ "# --- Modified Forward Pass (Differentiable) ---\n",+ "class MHRForwardPass:\n",+ " def __init__(self, mhr_path, device):\n",+ " self.device = device\n",+ " self.mhr = torch.jit.load(mhr_path, map_location=device).eval()\n",+ " for p in self.mhr.parameters():\n",+ " p.requires_grad_(False)\n",+ "\n",+ " def get_joints(self, model_params, shape_params):\n",+ " B = model_params.shape[0]\n",+ " dummy_identity = torch.zeros(B, 45, device=model_params.device, dtype=model_params.dtype)\n",+ " concat_params = torch.cat([model_params, dummy_identity], dim=1)\n",+ " \n",+ " joint_params = self.mhr.character_torch.model_parameters_to_joint_parameters(concat_params.to(self.device))\n",+ " skel_state = self.mhr.character_torch.joint_parameters_to_skeleton_state(joint_params)\n",+ " \n",+ " return skel_state.to(model_params.device)\n",+ "\n",+ "class DistillationLoss(nn.Module):\n",+ " def __init__(self, cfg, mhr_module):\n",+ " super().__init__()\n",+ " self.cfg = cfg\n",+ " self.mhr_module = mhr_module \n",+ " self.mse = nn.MSELoss()\n",+ " self.l1 = nn.SmoothL1Loss()\n",+ " self.strict_l1 = nn.L1Loss()\n",+ "\n",+ " def _project_3d_to_norm_2d(self, j3d, pred_cam_trans, targets):\n",+ " \"\"\"Helper to project 3D points (in meters) to normalized [-1, 1] 2D crop space.\"\"\"\n",+ " # A. Move to Camera Space\n",+ " j3d_cam = j3d + pred_cam_trans.unsqueeze(1)\n",+ " \n",+ " # Prevent 1/Z^2 Gradient Explosions\n",+ " # 0.01 allows a gradient multiplier of 10,000. \n",+ " # 0.4 meters (40cm) limits the multiplier to a safe limit, protecting the kinematic tree.\n",+ " z_cam = torch.clamp(j3d_cam[..., 2], min=0.4)\n",+ " \n",+ " # B. Extract Intrinsics\n",+ " fx = targets[\"cam_focal\"][:, 0:1] # Shape: [B, 1]\n",+ " fy = targets[\"cam_focal\"][:, 1:2] # Shape: [B, 1]\n",+ " cx = targets[\"orig_shape\"][:, 1:2] / 2.0 # Shape: [B, 1]\n",+ " cy = targets[\"orig_shape\"][:, 0:1] / 2.0 # Shape: [B, 1]\n",+ " \n",+ " # C. Full-Frame Projection\n",+ " u_full = (fx * j3d_cam[..., 0] / z_cam) + cx\n",+ " v_full = (fy * j3d_cam[..., 1] / z_cam) + cy\n",+ " \n",+ " # D. Map to Crop Space\n",+ " bbox = targets[\"bbox_square\"]\n",+ " sq_x1 = bbox[:, 0:1] # [B, 1]\n",+ " sq_y1 = bbox[:, 1:2] # [B, 1]\n",+ " crop_w = torch.clamp(bbox[:, 2:3] - bbox[:, 0:1], min=1.0) # [B, 1]\n",+ " \n",+ " true_scale = self.cfg.image_size / crop_w\n",+ " u_crop = (u_full - sq_x1) * true_scale\n",+ " v_crop = (v_full - sq_y1) * true_scale\n",+ " \n",+ " # E. Normalize to [-1, 1]\n",+ " u_norm = (u_crop / self.cfg.image_size) * 2.0 - 1.0\n",+ " v_norm = (v_crop / self.cfg.image_size) * 2.0 - 1.0\n",+ " \n",+ " return torch.stack([u_norm, v_norm], dim=-1) # Shape: [B, J, 2]\n",+ "\n",+ " def forward(self, preds, targets):\n",+ " losses = {}\n",+ " idx_pose = self.cfg.pose_dim\n",+ " \n",+ " pred_pose = preds[\"mhr_params\"][:, :idx_pose]\n",+ " pred_scale = preds[\"mhr_params\"][:, idx_pose:]\n",+ " tgt_pose = targets[\"mhr_model_params\"][:, :idx_pose]\n",+ " tgt_scale = targets[\"mhr_model_params\"][:, idx_pose:]\n",+ " \n",+ " # --- 1. Base Parameters ---\n",+ " losses['loss_pose'] = self.l1(pred_pose, tgt_pose) * self.cfg.w_pose\n",+ " losses['loss_scale'] = self.mse(pred_scale, tgt_scale) * self.cfg.w_scale\n",+ " losses['loss_shape'] = self.mse(preds[\"shape_params\"], targets[\"shape_params\"]) * self.cfg.w_shape\n",+ " losses['loss_cam'] = self.mse(preds[\"cam_trans\"], targets[\"cam_trans\"]) * self.cfg.w_cam\n",+ " \n",+ " # --- 2. Differentiable 3D Joint Alignment (Centimeter space corrected!) ---\n",+ " pred_mhr_joints = self.mhr_module.get_joints(preds[\"mhr_params\"], preds[\"shape_params\"])[..., :3]\n",+ " with torch.no_grad():\n",+ " tgt_mhr_joints = self.mhr_module.get_joints(targets[\"mhr_model_params\"], targets[\"shape_params\"])[..., :3]\n",+ " \n",+ " losses['loss_mhr_joints'] = self.l1(pred_mhr_joints, tgt_mhr_joints.detach()) * self.cfg.w_3d_joints\n",+ "\n",+ " # --- Base MHR Vision Coordinates (Needed for Structural & Reprojection) ---\n",+ " pred_mhr_vision = pred_mhr_joints.clone() / 100.0\n",+ " pred_mhr_vision[..., 1] = -pred_mhr_vision[..., 1]\n",+ " pred_mhr_vision[..., 2] = -pred_mhr_vision[..., 2]\n",+ "\n",+ " # ==============================================================================\n",+ " # 2.5 Structural Consistency Loss (MHR mesh vs Native Keypoints)\n",+ " # ==============================================================================\n",+ " if hasattr(self.cfg, 'native_mapping_ids') and hasattr(self.cfg, 'mhr_mapping_ids'):\n",+ " pred_native_anchors = preds[\"joints_3d\"][:, self.cfg.native_mapping_ids, :3]\n",+ " pred_mhr_anchors = pred_mhr_vision[:, self.cfg.mhr_mapping_ids, :3]\n",+ " losses['loss_structural'] = self.l1(pred_native_anchors, pred_mhr_anchors) * self.cfg.w_structural\n",+ "\n",+ " # ==============================================================================\n",+ " # Feet Barycenter Alignment Loss\n",+ " # ==============================================================================\n",+ " if hasattr(self.cfg, 'native_left_foot') and hasattr(self.cfg, 'w_feet'):\n",+ " nat_lf_pts = preds[\"joints_3d\"][:, self.cfg.native_left_foot, :3]\n",+ " nat_rf_pts = preds[\"joints_3d\"][:, self.cfg.native_right_foot, :3]\n",+ " \n",+ " mhr_lf_pts = pred_mhr_vision[:, self.cfg.mhr_left_foot, :3]\n",+ " mhr_rf_pts = pred_mhr_vision[:, self.cfg.mhr_right_foot, :3]\n",+ " \n",+ " nat_lf_bary = nat_lf_pts.mean(dim=1)\n",+ " nat_rf_bary = nat_rf_pts.mean(dim=1)\n",+ " mhr_lf_bary = mhr_lf_pts.mean(dim=1)\n",+ " mhr_rf_bary = mhr_rf_pts.mean(dim=1)\n",+ " \n",+ " losses['loss_feet'] = (self.strict_l1(nat_lf_bary, mhr_lf_bary) + self.strict_l1(nat_rf_bary, mhr_rf_bary)) * self.cfg.w_feet\n",+ "\n",+ " # --- 3. DYNAMIC Native Keypoints ---\n",+ " pred_2d = preds[\"joints_2d\"]\n",+ " tgt_2d = targets[\"joints_2d\"]\n",+ " losses['loss_2d_native'] = self.l1(pred_2d[..., :2], tgt_2d) * self.cfg.w_keypoints2d\n",+ " \n",+ " pred_3d = preds[\"joints_3d\"]\n",+ " tgt_3d = targets[\"joints_3d\"]\n",+ " losses['loss_3d_native'] = self.l1(pred_3d[..., :3], tgt_3d) * self.cfg.w_keypoints3d\n",+ "\n",+ " # ==============================================================================\n",+ " # 4. 2D Reprojection Loss (Camera-Pose Consistency)\n",+ " # ==============================================================================\n",+ " # A. Native 3D Head Reprojection (All 70 Joints)\n",+ " pred_reproj_native = self._project_3d_to_norm_2d(preds[\"joints_3d\"][..., :3], preds[\"cam_trans\"], targets)\n",+ " losses['loss_reproj_native'] = self.strict_l1(pred_reproj_native, tgt_2d) * self.cfg.w_reproj_3d\n",+ "\n",+ " # B. MHR Mesh Anchors Reprojection (52 Anchors)\n",+ " if hasattr(self.cfg, 'native_mapping_ids') and hasattr(self.cfg, 'mhr_mapping_ids'):\n",+ " # Only project the 52 matched anchors from the MHR mesh\n",+ " pred_mhr_anchors_vision = pred_mhr_vision[:, self.cfg.mhr_mapping_ids, :3]\n",+ " pred_reproj_mhr = self._project_3d_to_norm_2d(pred_mhr_anchors_vision, preds[\"cam_trans\"], targets) #detach the MHR anchors to prevent gradients from flowing back through the projection during this loss calculation\n",+ " # pred_reproj_mhr = self._project_3d_to_norm_2d(pred_mhr_anchors_vision, preds[\"cam_trans\"], targets)\n",+ " \n",+ " # Compare against the corresponding 52 ground truth 2D targets\n",+ " tgt_2d_anchors = tgt_2d[:, self.cfg.native_mapping_ids, :]\n",+ " losses['loss_reproj_mhr'] = self.strict_l1(pred_reproj_mhr, tgt_2d_anchors) * self.cfg.w_reproj_mhr\n",+ " \n",+ " losses['total_loss'] = sum(losses.values())\n",+ " return losses\n",+ "\n",+ "# ==============================================================================\n",+ "# SANITY CHECK: The Perfect Student\n",+ "# ==============================================================================\n",+ "if __name__ == \"__main__\":\n",+ " print(\"--- Running Loss Function Sanity Check ---\")\n",+ " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ " mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",+ " criterion = DistillationLoss(cfg, mhr_module)\n",+ " \n",+ " sample_batch_cpu = next(iter(train_loader))\n",+ " sample_batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in sample_batch_cpu.items()}\n",+ " \n",+ " mock_preds = {\n",+ " \"mhr_params\": sample_batch[\"mhr_model_params\"].clone(),\n",+ " \"shape_params\": sample_batch[\"shape_params\"].clone(),\n",+ " \"cam_trans\": sample_batch[\"cam_trans\"].clone(),\n",+ " \"joints_2d\": sample_batch[\"joints_2d\"].clone(),\n",+ " \"joints_3d\": sample_batch[\"joints_3d\"].clone()\n",+ " }\n",+ " \n",+ " losses = criterion(mock_preds, sample_batch)\n",+ " \n",+ " print(\"Losses for the Perfect Student:\")\n",+ " for k, v in losses.items():\n",+ " print(f\" {k:<20}: {v.item():.6f}\")\n",+ " \n",+ " if abs(losses['total_loss'].item()) < 1e-4:\n",+ " print(\"\\n✅ SUCCESS: The Perfect Student achieved a loss of 0! Math is fully verified.\")\n",+ " else:\n",+ " print(\"\\n❌ ERROR: Loss is not zero. Check the math.\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "ba2129c1",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 9 — The 1-Batch Overfit Test (VRAM Optimized)\n",+ "# ============================================================\n",+ "import torch\n",+ "import torch.optim as optim\n",+ "import matplotlib.pyplot as plt\n",+ "import gc\n",+ "\n",+ "print(\"--- Starting 1-Batch Overfit Test ---\")\n",+ "\n",+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",+ "\n",+ "# 1. Flush Notebook Ghost Memory\n",+ "gc.collect()\n",+ "torch.cuda.empty_cache()\n",+ "\n",+ "# 2. Instantiate Model, Loss, and Optimizer\n",+ "overfit_model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",+ "criterion = DistillationLoss(cfg, mhr_module)\n",+ "\n",+ "optimizer = optim.AdamW(overfit_model.parameters(), lr=1e-3, weight_decay=0.0)\n",+ "\n",+ "# Setup Automatic Mixed Precision (AMP) Scaler to save VRAM\n",+ "scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",+ "\n",+ "# 3. Grab exactly ONE batch and SLICE it to fit in VRAM\n",+ "overfit_batch_cpu = next(iter(train_loader))\n",+ "\n",+ "# We only need 8 images to prove the network can learn!\n",+ "def slice_and_move(batch_dict, dev, subset_size=8):\n",+ " return {k: v[:subset_size].to(dev) if isinstance(v, torch.Tensor) else v for k, v in batch_dict.items()}\n",+ "\n",+ "overfit_batch = slice_and_move(overfit_batch_cpu, device, subset_size=8)\n",+ "\n",+ "# 4. Training Loop (150 epochs on the exact same 8 images)\n",+ "epochs = 1000\n",+ "loss_history = []\n",+ "\n",+ "overfit_model.train()\n",+ "for epoch in range(epochs):\n",+ " optimizer.zero_grad(set_to_none=True) # slightly more memory efficient than zero_grad()\n",+ " \n",+ " # Forward pass with Mixed Precision\n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " images = overfit_batch[\"image\"]\n",+ " cliff_cond = overfit_batch[\"cliff_cond\"]\n",+ " \n",+ " preds = overfit_model(images, cliff_cond)\n",+ " \n",+ " # Compute loss\n",+ " losses = criterion(preds, overfit_batch)\n",+ " total_loss = losses['total_loss']\n",+ " \n",+ " # Backward pass with scaled gradients\n",+ " scaler.scale(total_loss).backward()\n",+ " \n",+ " # Unscale before gradient clipping\n",+ " scaler.unscale_(optimizer)\n",+ " torch.nn.utils.clip_grad_norm_(overfit_model.parameters(), cfg.grad_clip)\n",+ " \n",+ " # Optimizer step\n",+ " scaler.step(optimizer)\n",+ " scaler.update()\n",+ " \n",+ " loss_history.append(total_loss.item())\n",+ " \n",+ " # Print progress every 10 epochs\n",+ " if epoch % 10 == 0 or epoch == epochs - 1:\n",+ " print(f\"Epoch {epoch:03d} | Total Loss: {total_loss.item():.4f} | \"\n",+ " f\"2D Loss: {losses['loss_2d_native'].item():.4f} | \"\n",+ " f\"3D Loss: {losses['loss_3d_native'].item():.4f} | \"\n",+ " f\"Loss pose: {losses['loss_pose'].item():.4f} | \"\n",+ " f\"Scale Loss: {losses['loss_scale'].item():.4f} | \"\n",+ " f\"Shape Loss: {losses['loss_shape'].item():.4f} | \"\n",+ " f\"Cam Loss: {losses['loss_cam'].item():.4f}\"\n",+ " f\"Mhr joints Loss: {losses['loss_mhr_joints'].item():.4f}\",\n",+ " f\"Structural Loss: {losses['loss_structural'].item():.4f}\")\n",+ "\n",+ "# 5. Plot the results\n",+ "plt.figure(figsize=(10, 5))\n",+ "plt.plot(loss_history, label='Total Loss', color='blue', linewidth=2)\n",+ "plt.title('1-Batch Overfit Curve (8 Images)', fontsize=14)\n",+ "plt.xlabel('Epoch', fontsize=12)\n",+ "plt.ylabel('Total Loss', fontsize=12)\n",+ "plt.yscale('log') # Log scale helps visualize the exponential decay\n",+ "plt.grid(True, which=\"both\", ls=\"--\", alpha=0.5)\n",+ "plt.legend()\n",+ "plt.tight_layout()\n",+ "plt.show()\n",+ "\n",+ "if loss_history[-1] < loss_history[0] * 0.05:\n",+ " print(\"\\n✅ SUCCESS: The model successfully overfit the batch! Gradients are flowing.\")\n",+ "else:\n",+ " print(\"\\n❌ WARNING: The loss did not decrease significantly. Check for detached tensors or learning rate issues.\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "27616862",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 10 — HMR Evaluation Metrics (MPJPE & PA-MPJPE)\n",+ "# ============================================================\n",+ "import torch\n",+ "import numpy as np\n",+ "\n",+ "def batched_procrustes_alignment(pred_pts, gt_pts):\n",+ " \"\"\"\n",+ " Computes the optimal Similarity Transform (Scale, Rotation, Translation) \n",+ " to align pred_pts to gt_pts using Batched SVD.\n",+ " \"\"\"\n",+ " B, N, D = pred_pts.shape\n",+ " \n",+ " # 1. Mean center the point clouds\n",+ " mu_pred = pred_pts.mean(dim=1, keepdim=True)\n",+ " mu_gt = gt_pts.mean(dim=1, keepdim=True)\n",+ " \n",+ " pred_centered = pred_pts - mu_pred\n",+ " gt_centered = gt_pts - mu_gt\n",+ " \n",+ " # 2. Calculate scale (Frobenius norm)\n",+ " norm_pred = torch.linalg.norm(pred_centered, dim=(1,2), keepdim=True)\n",+ " norm_gt = torch.linalg.norm(gt_centered, dim=(1,2), keepdim=True)\n",+ " \n",+ " pred_normalized = pred_centered / torch.clamp(norm_pred, min=1e-8)\n",+ " gt_normalized = gt_centered / torch.clamp(norm_gt, min=1e-8)\n",+ " \n",+ " # 3. Compute covariance matrix H (X^T Y)\n",+ " H = torch.bmm(pred_normalized.transpose(1, 2), gt_normalized)\n",+ " \n",+ " # 4. Singular Value Decomposition (SVD)\n",+ " # PyTorch returns U, S, and Vh (where Vh is V^T)\n",+ " U, S, Vh = torch.linalg.svd(H)\n",+ " \n",+ " # 5. Compute Rotation matrix R (U * V^T) - FIXED!\n",+ " R = torch.bmm(U, Vh)\n",+ " \n",+ " # 6. Fix reflection (Strict SO(3) rotation)\n",+ " det = torch.linalg.det(R)\n",+ " \n",+ " # If det < 0, we must flip the sign of the 3rd column of U\n",+ " # Shape of det_sign is [B, 1]\n",+ " det_sign = torch.where(det < 0, torch.tensor(-1.0, device=pred_pts.device), torch.tensor(1.0, device=pred_pts.device)).unsqueeze(-1)\n",+ " \n",+ " U_fixed = U.clone()\n",+ " U_fixed[:, :, 2] *= det_sign\n",+ " \n",+ " # Recompute R with the reflection fix\n",+ " R_fixed = torch.bmm(U_fixed, Vh)\n",+ " \n",+ " # 7. Optimal Scale\n",+ " # We must apply the same sign fix to the 3rd singular value!\n",+ " S_fixed = S.clone()\n",+ " S_fixed[:, 2] *= det_sign.squeeze(-1)\n",+ " \n",+ " scale = S_fixed.sum(dim=-1, keepdim=True).unsqueeze(-1) * (norm_gt / torch.clamp(norm_pred, min=1e-8))\n",+ " \n",+ " # 8. Apply the transformation\n",+ " pred_aligned = scale * torch.bmm(pred_centered, R_fixed) + mu_gt\n",+ " \n",+ " return pred_aligned\n",+ "\n",+ "@torch.no_grad()\n",+ "def evaluate_hmr_batch(preds, targets, cfg, mhr_module):\n",+ " \"\"\"\n",+ " Calculates MPJPE and PA-MPJPE for both the 3D Keypoints Head and the MHR Mesh.\n",+ " Returns errors in MILLIMETERS (mm).\n",+ " \"\"\"\n",+ " metrics = {}\n",+ " \n",+ " # ---------------------------------------------------------\n",+ " # 1. Evaluate the Native 3D Spatial Head (70 Joints)\n",+ " # ---------------------------------------------------------\n",+ " pred_3d_native = preds[\"joints_3d\"][..., :3]\n",+ " gt_3d_native = targets[\"joints_3d\"][..., :3]\n",+ " \n",+ " # MPJPE requires root-relative alignment (usually joint 0 is the root/pelvis)\n",+ " pred_3d_root_rel = pred_3d_native - pred_3d_native[:, 0:1, :]\n",+ " gt_3d_root_rel = gt_3d_native - gt_3d_native[:, 0:1, :]\n",+ " \n",+ " # MPJPE in mm (* 1000)\n",+ " err_native = torch.linalg.norm(pred_3d_root_rel - gt_3d_root_rel, dim=-1)\n",+ " metrics['Head_3D_MPJPE'] = err_native.mean().item() * 1000.0\n",+ " \n",+ " # PA-MPJPE in mm\n",+ " pred_3d_aligned = batched_procrustes_alignment(pred_3d_native, gt_3d_native)\n",+ " err_native_pa = torch.linalg.norm(pred_3d_aligned - gt_3d_native, dim=-1)\n",+ " metrics['Head_3D_PA_MPJPE'] = err_native_pa.mean().item() * 1000.0\n",+ "\n",+ " # ---------------------------------------------------------\n",+ " # 2. Evaluate the MHR Mesh Head (Using the 52 Anchors)\n",+ " # ---------------------------------------------------------\n",+ " if hasattr(cfg, 'native_mapping_ids') and hasattr(cfg, 'mhr_mapping_ids'):\n",+ " # Generate 3D mesh joints in cm\n",+ " pred_mhr_raw = mhr_module.get_joints(preds[\"mhr_params\"], preds[\"shape_params\"])[..., :3]\n",+ " \n",+ " # Convert to Meters and align axes\n",+ " pred_mhr_meters = pred_mhr_raw / 100.0\n",+ " pred_mhr_meters[..., 1] = -pred_mhr_meters[..., 1]\n",+ " pred_mhr_meters[..., 2] = -pred_mhr_meters[..., 2]\n",+ " \n",+ " # Extract the 52 matching anchors\n",+ " pred_mesh_anchors = pred_mhr_meters[:, cfg.mhr_mapping_ids, :]\n",+ " gt_mesh_anchors = gt_3d_native[:, cfg.native_mapping_ids, :]\n",+ " \n",+ " # Mean-Center for MPJPE (To bypass differing origin definitions between rigs)\n",+ " pred_mesh_centered = pred_mesh_anchors - pred_mesh_anchors.mean(dim=1, keepdim=True)\n",+ " gt_mesh_centered = gt_mesh_anchors - gt_mesh_anchors.mean(dim=1, keepdim=True)\n",+ " \n",+ " # Mesh MPJPE in mm\n",+ " err_mesh = torch.linalg.norm(pred_mesh_centered - gt_mesh_centered, dim=-1)\n",+ " metrics['Mesh_MPJPE'] = err_mesh.mean().item() * 1000.0\n",+ " \n",+ " # Mesh PA-MPJPE in mm (Procrustes naturally handles translation offsets!)\n",+ " pred_mesh_aligned = batched_procrustes_alignment(pred_mesh_anchors, gt_mesh_anchors)\n",+ " err_mesh_pa = torch.linalg.norm(pred_mesh_aligned - gt_mesh_anchors, dim=-1)\n",+ " metrics['Mesh_PA_MPJPE'] = err_mesh_pa.mean().item() * 1000.0\n",+ " \n",+ " return metrics\n",+ "\n",+ "# # ==============================================================================\n",+ "# # SANITY CHECK: Evaluating Your Overfit Model\n",+ "# # ==============================================================================\n",+ "# if __name__ == \"__main__\":\n",+ "# print(\"--- Running Evaluation on Overfit Model ---\")\n",+ "# overfit_model.eval()\n",+ " \n",+ "# with torch.no_grad():\n",+ "# with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ "# eval_preds = overfit_model(overfit_batch[\"image\"], overfit_batch[\"cliff_cond\"])\n",+ " \n",+ "# # Upcast to float32 for rigorous geometric math\n",+ "# eval_preds_fp32 = {k: v.float() for k, v in eval_preds.items()}\n",+ " \n",+ "# metrics = evaluate_hmr_batch(eval_preds_fp32, overfit_batch, cfg, mhr_module)\n",+ " \n",+ "# print(\"\\n--- Final Metrics on 1-Batch Overfit ---\")\n",+ "# print(f\"3D Keypoint Head MPJPE: {metrics['Head_3D_MPJPE']:.2f} mm\")\n",+ "# print(f\"3D Keypoint Head PA-MPJPE: {metrics['Head_3D_PA_MPJPE']:.2f} mm\")\n",+ "# print(\"-\" * 40)\n",+ "# print(f\"MHR Mesh Anchors MPJPE: {metrics['Mesh_MPJPE']:.2f} mm\")\n",+ "# print(f\"MHR Mesh Anchors PA-MPJPE: {metrics['Mesh_PA_MPJPE']:.2f} mm\")\n",+ " \n",+ "# if metrics['Mesh_PA_MPJPE'] < 10.0:\n",+ "# print(\"\\n✅ OUTSTANDING: Mesh PA-MPJPE is under 10mm! Your model has achieved sub-centimeter physical accuracy.\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "2ede5144",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 11 — The Full Distillation Training Loop (Phase 2/Fresh)\n",+ "# ============================================================\n",+ "import torch\n",+ "import torch.optim as optim\n",+ "import torch.optim.lr_scheduler as lr_scheduler\n",+ "import os\n",+ "import math\n",+ "from tqdm.notebook import tqdm\n",+ "\n",+ "def train_instant_hmr():\n",+ " print(f\"--- Starting Full Training Pipeline: {cfg.backbone} ---\")\n",+ " os.makedirs(cfg.log_dir, exist_ok=True)\n",+ " \n",+ " # --- CONFIGURATION ---\n",+ " RESUME_FROM_CHECKPOINT = False # Set to False to train from scratch\n",+ " \n",+ " # 1. Initialize Model & Loss\n",+ " model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",+ " criterion = DistillationLoss(cfg, mhr_module)\n",+ " \n",+ " # 2. Optimizer & AMP Scaler\n",+ " optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n",+ " scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",+ " \n",+ " # --- RESUME / FRESH START LOGIC ---\n",+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model.pth\")\n",+ " start_epoch = 0 \n",+ " best_val_metric = float('inf') \n",+ " \n",+ " if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n",+ " print(f\"🔄 Resuming from checkpoint: {best_ckpt_path}\")\n",+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",+ " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False) \n",+ " else:\n",+ " print(\"🚀 Starting training from scratch with fresh weights.\")\n",+ "\n",+ " # Disable torch.compile() to prevent the LLVM pthread_join crash \n",+ " # and to stop BatchNorm running stats from corrupting during validation.\n",+ " # print(\"⏳ Compiling model with torch.compile()...\")\n",+ " # model = torch.compile(model, mode=\"default\") \n",+ "\n",+ " # 3. Learning Rate Scheduler\n",+ " steps_per_epoch = len(train_loader)\n",+ " scheduler = lr_scheduler.OneCycleLR(\n",+ " optimizer, \n",+ " max_lr=cfg.lr, \n",+ " epochs=cfg.epochs, \n",+ " steps_per_epoch=steps_per_epoch,\n",+ " pct_start=0.1\n",+ " )\n",+ " \n",+ " # --- MAIN EPOCH LOOP ---\n",+ " for epoch in range(start_epoch, cfg.epochs):\n",+ " model.train()\n",+ " \n",+ " train_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",+ " valid_train_batches = 0\n",+ " \n",+ " pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Train]\")\n",+ " for batch in pbar:\n",+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " \n",+ " optimizer.zero_grad(set_to_none=True)\n",+ " \n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",+ " \n",+ " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",+ " losses = criterion(preds_fp32, batch)\n",+ " loss = losses['total_loss']\n",+ " \n",+ " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",+ " print(\"⚠️ WARNING: NaN/Inf loss detected! Skipping step.\")\n",+ " continue\n",+ " \n",+ " scaler.scale(loss).backward()\n",+ " scaler.unscale_(optimizer)\n",+ " torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)\n",+ " \n",+ " # 🛑 Only step the scheduler if the optimizer actually stepped\n",+ " scale_before = scaler.get_scale()\n",+ " scaler.step(optimizer)\n",+ " scaler.update()\n",+ " scale_after = scaler.get_scale()\n",+ " \n",+ " # If the scale didn't drop (meaning no NaNs were found in gradients), step the LR scheduler\n",+ " if scale_before <= scale_after:\n",+ " scheduler.step()\n",+ " \n",+ " # Accumulate metrics\n",+ " train_metrics['total'] += loss.item()\n",+ " train_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",+ " train_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",+ " train_metrics['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",+ " train_metrics['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",+ " train_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",+ " train_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",+ " valid_train_batches += 1\n",+ " \n",+ " pbar.set_postfix({\n",+ " 'Tot': f\"{loss.item():.3f}\", \n",+ " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\",\n",+ " 'Str': f\"{losses.get('loss_structural', torch.tensor(0)).item():.3f}\"\n",+ " })\n",+ " \n",+ " pbar.close() \n",+ " avg_train = {k: v / max(valid_train_batches, 1) for k, v in train_metrics.items()}\n",+ " \n",+ " # --- VALIDATION LOOP ---\n",+ " model.eval()\n",+ " val_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",+ " \n",+ " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",+ " valid_val_batches = 0\n",+ " \n",+ " with torch.no_grad():\n",+ " for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val]\", leave=False):\n",+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " \n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",+ " \n",+ " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",+ " losses = criterion(preds_fp32, batch)\n",+ " loss = losses['total_loss']\n",+ " \n",+ " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",+ " continue\n",+ " \n",+ " val_metrics['total'] += loss.item()\n",+ " val_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",+ " val_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",+ " val_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",+ " val_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",+ " \n",+ " batch_hmr_metrics = evaluate_hmr_batch(preds_fp32, batch, cfg, mhr_module)\n",+ " for k in hmr_trackers.keys():\n",+ " hmr_trackers[k] += batch_hmr_metrics.get(k, 0.0)\n",+ " \n",+ " valid_val_batches += 1\n",+ " \n",+ " avg_val = {k: v / max(valid_val_batches, 1) for k, v in val_metrics.items()}\n",+ " avg_hmr = {k: v / max(valid_val_batches, 1) for k, v in hmr_trackers.items()}\n",+ " \n",+ " # --- DETAILED EPOCH SUMMARY ---\n",+ " print(f\"\\n📈 Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}\")\n",+ " print(f\" [Train Loss] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | Struct: {avg_train['struct']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",+ " print(f\" [Val Loss] Tot: {avg_val['total']:.4f} | MHR: {avg_val['mhr']:.4f} | Struct: {avg_val['struct']:.4f} | 2D: {avg_val['2d']:.4f} | 3D: {avg_val['3d']:.4f}\")\n",+ " print(f\" 📏 [Val Metrics] Mesh PA-MPJPE: {avg_hmr['Mesh_PA_MPJPE']:.1f} mm | Mesh MPJPE: {avg_hmr['Mesh_MPJPE']:.1f} mm\")\n",+ " print(f\" 📏 [Val Metrics] Head PA-MPJPE: {avg_hmr['Head_3D_PA_MPJPE']:.1f} mm | Head MPJPE: {avg_hmr['Head_3D_MPJPE']:.1f} mm\")\n",+ " \n",+ " # --- CHECKPOINTING (Now based on Mesh PA-MPJPE!) ---\n",+ " current_metric = avg_hmr['Mesh_PA_MPJPE']\n",+ " if current_metric < best_val_metric and valid_val_batches > 0:\n",+ " best_val_metric = current_metric\n",+ " save_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",+ " torch.save({\n",+ " 'epoch': epoch,\n",+ " 'model_state_dict': model.state_dict(),\n",+ " 'optimizer_state_dict': optimizer.state_dict(),\n",+ " 'val_pa_mpjpe': best_val_metric,\n",+ " }, save_path)\n",+ " print(f\"💾 New Best Model saved! Mesh PA-MPJPE dropped to {best_val_metric:.1f} mm.\")\n",+ "\n",+ " print(\"\\n🎉 Training Complete!\")\n",+ "\n",+ "# Kick off the training!\n",+ "if __name__ == \"__main__\":\n",+ " import gc\n",+ " gc.collect()\n",+ " torch.cuda.empty_cache()\n",+ " \n",+ " train_instant_hmr()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "e12c985b",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 13 — Validation Visualization (The Reveal!)\n",+ "# ============================================================\n",+ "import torch\n",+ "import matplotlib.pyplot as plt\n",+ "import os\n",+ "\n",+ "print(\"--- Running Inference Visualization ---\")\n",+ "\n",+ "# 1. Load the Best Saved Model\n",+ "best_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",+ "checkpoint_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",+ "\n",+ "if os.path.exists(checkpoint_path):\n",+ " checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=True)\n",+ " \n",+ " # --- THE FIX: Clean the torch.compile() prefix ---\n",+ " clean_state_dict = {}\n",+ " for k, v in checkpoint['model_state_dict'].items():\n",+ " # Remove '_orig_mod.' prefix if it exists\n",+ " new_key = k.replace('_orig_mod.', '')\n",+ " clean_state_dict[new_key] = v\n",+ " \n",+ " # Load the cleaned dictionary\n",+ " best_model.load_state_dict(clean_state_dict, strict=False)\n",+ " print(f\"✅ Loaded Best Model from Epoch {checkpoint.get('epoch', 0)+1} with PA-MPJPE: {checkpoint.get('val_pa_mpjpe', 0):.1f} mm\")\n",+ "else:\n",+ " print(\"⚠️ No checkpoint found! Using current model in memory.\")\n",+ " best_model = model\n",+ "\n",+ "best_model.eval()\n",+ "\n",+ "# 2. Grab a Validation Batch\n",+ "val_batch = next(iter(val_loader))\n",+ "images = val_batch[\"image\"].to(device)\n",+ "cliff_cond = val_batch[\"cliff_cond\"].to(device)\n",+ "gt_2d = val_batch[\"joints_2d\"].cpu()\n",+ "\n",+ "# 3. Run Inference\n",+ "with torch.no_grad():\n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " preds = best_model(images, cliff_cond)\n",+ "\n",+ "# Extract only the X,Y coordinates\n",+ "pred_2d = preds[\"joints_2d\"][..., :2].cpu()\n",+ "\n",+ "# 4. Math Helper: Un-normalize coordinates from [-1, 1] back to [0, 224] pixels\n",+ "def unnorm_coords(coords, size=224.0):\n",+ " return ((coords + 1.0) / 2.0) * size\n",+ "\n",+ "gt_2d_px = unnorm_coords(gt_2d[..., :2], cfg.image_size)\n",+ "pred_2d_px = unnorm_coords(pred_2d, cfg.image_size)\n",+ "\n",+ "# 5. Image Helper: Un-normalize ImageNet pixels for matplotlib\n",+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",+ "\n",+ "# 6. Plot a 2x2 Grid of Predictions\n",+ "fig, axes = plt.subplots(2, 2, figsize=(10, 10))\n",+ "axes = axes.flatten()\n",+ "\n",+ "for i in range(4): # Plot the first 4 images in the batch\n",+ " # Prepare image\n",+ " img_unnorm = images[i].cpu() * std + mean\n",+ " img_np = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",+ " \n",+ " # Plot Image\n",+ " axes[i].imshow(img_np)\n",+ " \n",+ " # Plot Ground Truth (Teacher) - Lime Green Dots\n",+ " axes[i].scatter(gt_2d_px[i, :, 0], gt_2d_px[i, :, 1], \n",+ " c='lime', s=25, edgecolors='black', label='Teacher (GT)' if i==0 else \"\")\n",+ " \n",+ " # Plot Predictions (Student) - Red Crosses\n",+ " axes[i].scatter(pred_2d_px[i, :, 0], pred_2d_px[i, :, 1], \n",+ " c='red', s=35, marker='X', edgecolors='black', linewidths=0.5, label='Student' if i==0 else \"\")\n",+ " \n",+ " axes[i].axis('off')\n",+ " axes[i].set_title(f\"Validation Sample {i+1}\", fontsize=12)\n",+ "\n",+ "fig.legend(loc='upper center', ncol=2, fontsize=14, bbox_to_anchor=(0.5, 1.05))\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "2cd9f6d4",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 14 — Inference Speed Benchmark (Fixed Loader)\n",+ "# ============================================================\n",+ "import time\n",+ "import os\n",+ "import torch\n",+ "\n",+ "print(\"--- Running Inference Speed Benchmark ---\")\n",+ "\n",+ "@torch.no_grad()\n",+ "def benchmark_fps(model, image_size=224, batch_size=1, n_warmup=50, n_iters=200):\n",+ " \"\"\"Benchmark model throughput on GPU.\"\"\"\n",+ " model.eval()\n",+ " # Dummy inputs matching our architecture\n",+ " dummy_img = torch.randn(batch_size, 3, image_size, image_size, device=device)\n",+ " dummy_cliff = torch.randn(batch_size, 3, device=device) # [cx_norm, cy_norm, b_scale]\n",+ "\n",+ " # Warmup (Wraps the forward pass in AMP for real-world float16 speed)\n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " for _ in range(n_warmup):\n",+ " _ = model(dummy_img, dummy_cliff)\n",+ " \n",+ " if torch.cuda.is_available():\n",+ " torch.cuda.synchronize()\n",+ "\n",+ " # Timed run\n",+ " t0 = time.perf_counter()\n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " for _ in range(n_iters):\n",+ " _ = model(dummy_img, dummy_cliff)\n",+ " \n",+ " if torch.cuda.is_available():\n",+ " torch.cuda.synchronize()\n",+ " \n",+ " elapsed = time.perf_counter() - t0\n",+ "\n",+ " fps = n_iters * batch_size / elapsed\n",+ " ms_per_image = elapsed / (n_iters * batch_size) * 1000\n",+ " return fps, ms_per_image\n",+ "\n",+ "# 1. Instantiate the Model\n",+ "benchmark_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",+ "\n",+ "# 2. Load best checkpoint with the compile fix!\n",+ "best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",+ "if os.path.exists(best_ckpt_path):\n",+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",+ " \n",+ " # --- Clean the torch.compile() prefix ---\n",+ " clean_state_dict = {}\n",+ " for k, v in ckpt['model_state_dict'].items():\n",+ " new_key = k.replace('_orig_mod.', '')\n",+ " clean_state_dict[new_key] = v\n",+ " \n",+ " benchmark_model.load_state_dict(clean_state_dict, strict=False)\n",+ " print(f\"✅ Loaded best model from epoch {ckpt.get('epoch', 0)+1} (PA-MPJPE={ckpt.get('val_pa_mpjpe', 0):.1f} mm)\")\n",+ "else:\n",+ " print(\"⚠️ No checkpoint found! Benchmarking with random weights.\")\n",+ "\n",+ "# 3. Benchmark at different batch sizes\n",+ "fps_b1, ms_b1 = benchmark_fps(benchmark_model, batch_size=1)\n",+ "fps_b16, ms_b16 = benchmark_fps(benchmark_model, batch_size=16)\n",+ "fps_b64, ms_b64 = benchmark_fps(benchmark_model, batch_size=64)\n",+ "\n",+ "# 4. Compute total parameters\n",+ "total_params = sum(p.numel() for p in benchmark_model.parameters())\n",+ "\n",+ "print(f\"\\n🚀 Inference Speed (RTX {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}):\")\n",+ "print(f\" Batch=1: {fps_b1:.0f} FPS ({ms_b1:.1f} ms/image)\")\n",+ "print(f\" Batch=16: {fps_b16:.0f} FPS ({ms_b16:.1f} ms/image)\")\n",+ "print(f\" Batch=64: {fps_b64:.0f} FPS ({ms_b64:.1f} ms/image)\")\n",+ "print(f\"\\n📊 Comparison:\")\n",+ "print(f\" SAM3D Teacher: ~3 FPS (670M params)\")\n",+ "print(f\" Student: {fps_b1:.0f} FPS ({total_params/1e6:.1f}M params) → {fps_b1/3:.0f}x faster\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "334859a9",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 15 — Export & Static Quantization (FP32, FP16, INT8)\n",+ "# ============================================================\n",+ "import os\n",+ "import torch\n",+ "import torch.nn as nn\n",+ "import numpy as np\n",+ "from pathlib import Path\n",+ "\n",+ "print(\"--- Exporting and Quantizing Full Model for Deployment ---\")\n",+ "\n",+ "# --- 1. Deploy Wrapper (full outputs) ---\n",+ "class HMRDeployWrapper(nn.Module):\n",+ " def __init__(self, model):\n",+ " super().__init__()\n",+ " self.model = model\n",+ " def forward(self, image, cliff_cond):\n",+ " out = self.model(image, cliff_cond)\n",+ " return (out[\"mhr_params\"], out[\"shape_params\"], out[\"cam_trans\"], \n",+ " out[\"joints_2d\"], out[\"joints_3d\"])\n",+ "\n",+ "# --- 2. INT8 Calibration Data Reader ---\n",+ "try:\n",+ " from onnxruntime.quantization import (\n",+ " CalibrationDataReader, quantize_static,\n",+ " QuantType, QuantFormat, CalibrationMethod,\n",+ " )\n",+ " \n",+ " class HMRCalibrationDataReader(CalibrationDataReader):\n",+ " def __init__(self, dataloader, max_samples=100):\n",+ " self.data_list = []\n",+ " \n",+ " print(f\"Extracting {max_samples} individual images for INT8 calibration...\")\n",+ " for batch in dataloader:\n",+ " images = batch[\"image\"].numpy().astype(np.float32)\n",+ " cliffs = batch[\"cliff_cond\"].numpy().astype(np.float32)\n",+ " \n",+ " for b_idx in range(images.shape[0]):\n",+ " if len(self.data_list) >= max_samples:\n",+ " break\n",+ " self.data_list.append({\n",+ " \"image\": images[b_idx:b_idx+1],\n",+ " \"cliff_cond\": cliffs[b_idx:b_idx+1]\n",+ " })\n",+ " if len(self.data_list) >= max_samples:\n",+ " break\n",+ " \n",+ " self.enum_data = iter(self.data_list)\n",+ "\n",+ " def get_next(self):\n",+ " return next(self.enum_data, None)\n",+ "\n",+ "except ImportError:\n",+ " print(\"⚠️ 'onnxruntime' is not installed. INT8 quantization will fail.\")\n",+ "\n",+ "\n",+ "def topological_sort_onnx(model):\n",+ " \"\"\"Sort ONNX graph nodes in topological order.\n",+ " \n",+ " The ORT float16 converter with keep_io_types=True inserts Cast nodes at graph\n",+ " inputs but doesn't re-sort the graph, breaking topological order. This fixes it.\n",+ " \"\"\"\n",+ " available = set()\n",+ " for inp in model.graph.input:\n",+ " available.add(inp.name)\n",+ " for init in model.graph.initializer:\n",+ " available.add(init.name)\n",+ " \n",+ " remaining = list(model.graph.node)\n",+ " sorted_nodes = []\n",+ " \n",+ " while remaining:\n",+ " progress = False\n",+ " for node in remaining:\n",+ " if all(inp in available or inp == '' for inp in node.input):\n",+ " sorted_nodes.append(node)\n",+ " remaining.remove(node)\n",+ " for out in node.output:\n",+ " available.add(out)\n",+ " progress = True\n",+ " break\n",+ " if not progress:\n",+ " sorted_nodes.extend(remaining)\n",+ " break\n",+ " \n",+ " del model.graph.node[:]\n",+ " model.graph.node.extend(sorted_nodes)\n",+ " return model\n",+ "\n",+ "\n",+ "# --- 3. Export & Quantize ---\n",+ "export_dir = Path(cfg.log_dir) / \"export\"\n",+ "export_dir.mkdir(parents=True, exist_ok=True)\n",+ "\n",+ "onnx_fp32_path = export_dir / \"sam3d_full_fp32.onnx\"\n",+ "onnx_fp16_path = export_dir / \"sam3d_full_fp16.onnx\"\n",+ "onnx_int8_path = export_dir / \"sam3d_full_int8.onnx\"\n",+ "\n",+ "output_keys = [\"mhr_params\", \"shape_params\", \"cam_trans\", \"joints_2d\", \"joints_3d\"]\n",+ "\n",+ "export_target_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",+ "\n",+ "best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",+ "\n",+ "if os.path.exists(best_ckpt_path):\n",+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",+ " clean_state_dict = {k.replace('_orig_mod.', ''): v for k, v in ckpt['model_state_dict'].items()}\n",+ " export_target_model.load_state_dict(clean_state_dict, strict=False)\n",+ " print(f\"✅ Loaded best model from epoch {ckpt.get('epoch', 0)+1}.\\n\")\n",+ " \n",+ " deploy_model = HMRDeployWrapper(export_target_model)\n",+ " deploy_model.eval()\n",+ " \n",+ " dummy_img = torch.randn(1, 3, cfg.image_size, cfg.image_size, device=device)\n",+ " dummy_cliff = torch.randn(1, 3, device=device)\n",+ " \n",+ " dynamic_axes = {\"image\": {0: \"batch\"}, \"cliff_cond\": {0: \"batch\"}}\n",+ " for k in output_keys:\n",+ " dynamic_axes[k] = {0: \"batch\"}\n",+ "\n",+ " # A. Export FP32\n",+ " try:\n",+ " import onnx\n",+ " torch.onnx.export(\n",+ " deploy_model, (dummy_img, dummy_cliff), str(onnx_fp32_path),\n",+ " input_names=[\"image\", \"cliff_cond\"], output_names=output_keys,\n",+ " dynamic_axes=dynamic_axes, opset_version=17\n",+ " )\n",+ " print(f\"✅ FP32 saved: {onnx_fp32_path.name} | Size: {onnx_fp32_path.stat().st_size / 1e6:.1f} MB\")\n",+ " except Exception as e:\n",+ " print(f\"❌ FP32 export failed: {e}\")\n",+ "\n",+ " # B. Export FP16 — uses ORT's converter which correctly updates Cast node `to` attributes\n",+ " # + topological sort to fix node ordering after keep_io_types Cast insertion\n",+ " try:\n",+ " from onnxruntime.transformers.float16 import convert_float_to_float16\n",+ " import warnings\n",+ " with warnings.catch_warnings():\n",+ " warnings.simplefilter(\"ignore\")\n",+ " model_fp32 = onnx.load(str(onnx_fp32_path))\n",+ " model_fp16 = convert_float_to_float16(model_fp32, keep_io_types=True)\n",+ " model_fp16 = topological_sort_onnx(model_fp16)\n",+ " onnx.save(model_fp16, str(onnx_fp16_path))\n",+ " onnx.checker.check_model(onnx.load(str(onnx_fp16_path)), full_check=True)\n",+ " print(f\"✅ FP16 saved: {onnx_fp16_path.name} | Size: {onnx_fp16_path.stat().st_size / 1e6:.1f} MB\")\n",+ " except ImportError:\n",+ " print(\"⚠️ FP16 skipped (onnxruntime.transformers not available)\")\n",+ " except Exception as e:\n",+ " print(f\"❌ FP16 conversion failed: {e}\")\n",+ "\n",+ " # C. Export INT8 — selective: only quantize Conv ops (backbone), skip transformer/heads\n",+ " # Naive quantization of all ops destroys accuracy (Softmax, LayerNorm, attention MatMuls\n",+ " # need FP32 precision) and is slower due to unfused QDQ overhead blocking NCHWc optimization.\n",+ " try:\n",+ " calib_reader = HMRCalibrationDataReader(train_loader, max_samples=100)\n",+ " calib_reader.enum_data = iter(calib_reader.data_list)\n",+ " \n",+ " # Collect transformer + head + projection node names to exclude from quantization\n",+ " # (defense-in-depth: op_types_to_quantize=['Conv'] already excludes them since they\n",+ " # have zero Conv ops, but explicit exclusion prevents issues if the model changes)\n",+ " fp32_model = onnx.load(str(onnx_fp32_path))\n",+ " nodes_to_exclude = []\n",+ " for node in fp32_model.graph.node:\n",+ " name = node.name or ''\n",+ " if any(kw in name for kw in ['/model/transformer/', '/model/head_',\n",+ " '/model/feat_proj', '/model/cond_proj']):\n",+ " nodes_to_exclude.append(name)\n",+ " del fp32_model\n",+ " print(f\" Excluding {len(nodes_to_exclude)} transformer/head nodes from quantization\")\n",+ " \n",+ " quantize_static(\n",+ " model_input=str(onnx_fp32_path),\n",+ " model_output=str(onnx_int8_path),\n",+ " calibration_data_reader=calib_reader,\n",+ " quant_format=QuantFormat.QDQ,\n",+ " op_types_to_quantize=['Conv'], # Only quantize Conv (backbone)\n",+ " per_channel=True, # Better accuracy for Conv weights\n",+ " reduce_range=False, # Not needed with VNNI\n",+ " activation_type=QuantType.QInt8,\n",+ " weight_type=QuantType.QInt8,\n",+ " nodes_to_exclude=nodes_to_exclude,\n",+ " calibrate_method=CalibrationMethod.Percentile,\n",+ " extra_options={\n",+ " 'CalibTensorRangeSymmetric': True,\n",+ " },\n",+ " )\n",+ " print(f\"✅ INT8 saved: {onnx_int8_path.name} | Size: {onnx_int8_path.stat().st_size / 1e6:.1f} MB\")\n",+ " except NameError:\n",+ " print(\"⚠️ INT8 skipped (train_loader not available)\")\n",+ " except Exception as e:\n",+ " import traceback\n",+ " print(f\"❌ INT8 conversion failed: {e}\")\n",+ " traceback.print_exc()\n",+ "\n",+ " # D. Save PyTorch state dict\n",+ " sd_path = export_dir / \"sam3d_student_state_dict.pth\"\n",+ " torch.save(export_target_model.state_dict(), sd_path)\n",+ " print(f\"\\n✅ PyTorch state dict saved: {sd_path.name}\")\n",+ " \n",+ "else:\n",+ " print(\"⚠️ No checkpoint found at:\", best_ckpt_path)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "11d916bf",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 17 — ONNX Profiler (FP32, FP16, INT8) x (CPU, GPU)\n",+ "# ============================================================\n",+ "import time\n",+ "import numpy as np\n",+ "from pathlib import Path\n",+ "\n",+ "print(\"--- ONNX Hardware Profiler ---\")\n",+ "\n",+ "try:\n",+ " import onnxruntime as ort\n",+ " ONNX_AVAILABLE = True\n",+ "except ImportError:\n",+ " ONNX_AVAILABLE = False\n",+ " print(\"⚠️ Missing: pip install onnxruntime-gpu\")\n",+ "\n",+ "if ONNX_AVAILABLE:\n",+ " def profile_onnx_model(onnx_path, provider, batch_size=1, warmup_iters=20, benchmark_iters=100):\n",+ " \"\"\"Profile latency for a specific ONNX model and execution provider.\"\"\"\n",+ " \n",+ " sess_options = ort.SessionOptions()\n",+ " sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\n",+ " sess_options.log_severity_level = 3\n",+ " \n",+ " try:\n",+ " session = ort.InferenceSession(str(onnx_path), sess_options=sess_options, providers=[provider])\n",+ " except Exception as e:\n",+ " return None, None, f\"Error: {e}\"\n",+ " \n",+ " active_provider = session.get_providers()[0]\n",+ " if active_provider != provider:\n",+ " return None, None, f\"Fallback! Requested {provider} but got {active_provider}\"\n",+ "\n",+ " ort_inputs = {}\n",+ " dtype_map = {'tensor(float)': np.float32, 'tensor(float16)': np.float16}\n",+ " \n",+ " for inp in session.get_inputs():\n",+ " shape = [batch_size if isinstance(dim, str) else dim for dim in inp.shape]\n",+ " np_dtype = dtype_map.get(inp.type, np.float32)\n",+ " ort_inputs[inp.name] = np.random.randn(*shape).astype(np_dtype)\n",+ "\n",+ " for _ in range(warmup_iters):\n",+ " session.run(None, ort_inputs)\n",+ "\n",+ " t0 = time.perf_counter()\n",+ " for _ in range(benchmark_iters):\n",+ " session.run(None, ort_inputs)\n",+ " t1 = time.perf_counter()\n",+ "\n",+ " total_time = t1 - t0\n",+ " avg_ms = (total_time / benchmark_iters) * 1000.0\n",+ " fps = (benchmark_iters * batch_size) / total_time\n",+ " return avg_ms, fps, \"OK\"\n",+ "\n",+ " # --- Setup ---\n",+ " export_dir = Path(cfg.log_dir) / \"export\"\n",+ " precisions = [\"fp32\", \"fp16\", \"int8\"]\n",+ "\n",+ " available_providers = ort.get_available_providers()\n",+ " providers_to_test = ['CPUExecutionProvider']\n",+ " if 'CUDAExecutionProvider' in available_providers:\n",+ " providers_to_test.append('CUDAExecutionProvider')\n",+ " else:\n",+ " print(\"⚠️ CUDAExecutionProvider not found. Profiling CPU only.\")\n",+ " print(\" Fix: pip uninstall onnxruntime onnxruntime-gpu -y && pip install onnxruntime-gpu\")\n",+ "\n",+ " print(f\"\\n{'Precision':<6} | {'Size':<7} | {'HW':<4} | {'Latency':<12} | {'FPS':<8} | Notes\")\n",+ " print(\"-\" * 75)\n",+ "\n",+ " for prec in precisions:\n",+ " model_path = export_dir / f\"sam3d_full_{prec}.onnx\"\n",+ " \n",+ " if not model_path.exists():\n",+ " print(f\"{prec:<6} | --- | --- | {'Not Found':<12} | ---\")\n",+ " continue\n",+ " \n",+ " size_mb = f\"{model_path.stat().st_size / 1e6:.1f}MB\"\n",+ " \n",+ " for provider in providers_to_test:\n",+ " hw_tag = \"GPU\" if \"CUDA\" in provider else \"CPU\"\n",+ " \n",+ " # FP16 on CPU: not supported (no native x86 FP16 compute)\n",+ " if prec == \"fp16\" and provider == \"CPUExecutionProvider\":\n",+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {'N/A':<12} | --- | x86 has no FP16 ALU\")\n",+ " continue\n",+ " \n",+ " note = \"\"\n",+ " if prec == \"fp16\" and \"CUDA\" in provider:\n",+ " note = \"Best for GPU deploy\"\n",+ " elif prec == \"int8\" and \"CPU\" in provider:\n",+ " note = \"Conv-only quant, for edge/NPU\"\n",+ " elif prec == \"int8\" and \"CUDA\" in provider:\n",+ " note = \"Needs TensorRT EP for gains\"\n",+ " \n",+ " ms, fps, status = profile_onnx_model(model_path, provider)\n",+ " \n",+ " if ms is not None:\n",+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {ms:>7.2f} ms | {fps:>6.1f} | {note}\")\n",+ " else:\n",+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {'Failed':<12} | --- | {status}\")\n",+ "\n",+ " print(\"-\" * 75)\n",+ " print(\"\\nDeployment guide:\")\n",+ " print(\" GPU (CUDA EP) → FP16 (best latency + half memory)\")\n",+ " print(\" GPU (TensorRT EP) → INT8 (if TRT available, even faster)\")\n",+ " print(\" CPU (x86 VNNI) → FP32 or INT8 (Conv-only quant, test both)\")\n",+ " print(\" Mobile NPU → INT8 with NNAPI/QNN EP (native INT8 HW)\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "eikwijqvez7",+ "metadata": {},+ "outputs": [],+ "source": [+ "# ============================================================\n",+ "# Cell 18 — ONNX Accuracy Comparison (FP32 vs FP16 vs INT8)\n",+ "# ============================================================\n",+ "import numpy as np\n",+ "import torch\n",+ "from pathlib import Path\n",+ "from tqdm.notebook import tqdm\n",+ "\n",+ "print(\"--- ONNX Quantization Accuracy Comparison ---\")\n",+ "\n",+ "try:\n",+ " import onnxruntime as ort\n",+ " ONNX_AVAILABLE = True\n",+ "except ImportError:\n",+ " ONNX_AVAILABLE = False\n",+ " print(\"⚠️ Missing: pip install onnxruntime-gpu\")\n",+ "\n",+ "if ONNX_AVAILABLE:\n",+ " export_dir = Path(cfg.log_dir) / \"export\"\n",+ " output_names = [\"mhr_params\", \"shape_params\", \"cam_trans\", \"joints_2d\", \"joints_3d\"]\n",+ " \n",+ " models_to_eval = {\n",+ " \"FP32\": export_dir / \"sam3d_full_fp32.onnx\",\n",+ " \"FP16\": export_dir / \"sam3d_full_fp16.onnx\",\n",+ " \"INT8\": export_dir / \"sam3d_full_int8.onnx\",\n",+ " }\n",+ " \n",+ " # Filter to existing models only\n",+ " models_to_eval = {k: v for k, v in models_to_eval.items() if v.exists()}\n",+ " print(f\"Found models: {', '.join(models_to_eval.keys())}\")\n",+ " \n",+ " # Use GPU if available, otherwise CPU (FP16 needs GPU)\n",+ " available_providers = ort.get_available_providers()\n",+ " providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if 'CUDAExecutionProvider' in available_providers else ['CPUExecutionProvider']\n",+ " \n",+ " results = {}\n",+ " \n",+ " for model_name, model_path in models_to_eval.items():\n",+ " print(f\"\\nEvaluating {model_name} ({model_path.stat().st_size / 1e6:.1f} MB)...\")\n",+ " \n",+ " sess_options = ort.SessionOptions()\n",+ " sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\n",+ " sess_options.log_severity_level = 3\n",+ " \n",+ " try:\n",+ " session = ort.InferenceSession(str(model_path), sess_options=sess_options, providers=providers)\n",+ " except Exception as e:\n",+ " print(f\" ❌ Failed to load: {e}\")\n",+ " continue\n",+ " \n",+ " active_provider = session.get_providers()[0]\n",+ " print(f\" Running on: {active_provider}\")\n",+ " \n",+ " # Accumulate metrics across all val batches\n",+ " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",+ " valid_batches = 0\n",+ " \n",+ " for batch in tqdm(val_loader, desc=f\" {model_name}\", leave=False):\n",+ " images_np = batch[\"image\"].numpy().astype(np.float32)\n",+ " cliff_np = batch[\"cliff_cond\"].numpy().astype(np.float32)\n",+ " \n",+ " ort_inputs = {\"image\": images_np, \"cliff_cond\": cliff_np}\n",+ " ort_outputs = session.run(None, ort_inputs)\n",+ " \n",+ " # Convert ONNX outputs to PyTorch tensor dict\n",+ " preds = {}\n",+ " for name, arr in zip(output_names, ort_outputs):\n",+ " preds[name] = torch.from_numpy(arr.astype(np.float32)).to(device)\n",+ " \n",+ " # Move GT targets to device\n",+ " targets = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " \n",+ " batch_metrics = evaluate_hmr_batch(preds, targets, cfg, mhr_module)\n",+ " for k in hmr_trackers:\n",+ " hmr_trackers[k] += batch_metrics.get(k, 0.0)\n",+ " valid_batches += 1\n",+ " \n",+ " avg_metrics = {k: v / max(valid_batches, 1) for k, v in hmr_trackers.items()}\n",+ " results[model_name] = avg_metrics\n",+ " print(f\" ✅ {valid_batches} batches evaluated\")\n",+ " \n",+ " # --- Print Comparison Table ---\n",+ " print(f\"\\n{'='*72}\")\n",+ " print(f\"{'Metric':<22} |\", end=\"\")\n",+ " for name in results:\n",+ " print(f\" {name:>10}\", end=\" |\")\n",+ " \n",+ " # Show degradation vs FP32\n",+ " if \"FP32\" in results and len(results) > 1:\n",+ " print(f\" {'vs FP32':>10}\", end=\"\")\n",+ " print()\n",+ " print(\"-\" * 72)\n",+ " \n",+ " metric_labels = {\n",+ " 'Head_3D_MPJPE': 'Head 3D MPJPE (mm)',\n",+ " 'Head_3D_PA_MPJPE': 'Head 3D PA-MPJPE (mm)',\n",+ " 'Mesh_MPJPE': 'Mesh MPJPE (mm)',\n",+ " 'Mesh_PA_MPJPE': 'Mesh PA-MPJPE (mm)',\n",+ " }\n",+ " \n",+ " for key, label in metric_labels.items():\n",+ " print(f\"{label:<22} |\", end=\"\")\n",+ " for name in results:\n",+ " print(f\" {results[name][key]:>10.2f}\", end=\" |\")\n",+ " \n",+ " # Show delta vs FP32 for non-FP32 models\n",+ " if \"FP32\" in results and len(results) > 1:\n",+ " deltas = []\n",+ " for name in results:\n",+ " if name != \"FP32\":\n",+ " delta = results[name][key] - results[\"FP32\"][key]\n",+ " deltas.append(f\"{delta:+.2f}\")\n",+ " print(f\" {', '.join(deltas):>10}\", end=\"\")\n",+ " print()\n",+ " \n",+ " print(f\"{'='*72}\")"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "sam3d_video_312",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.12.12"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 5+}diff --git a/requirements.txt b/requirements.txtnew file mode 100644index 0000000..5bb1471--- /dev/null+++ b/requirements.txt@@ -0,0 +1,24 @@+# ----- inference + viz (required by demo.py) -----+numpy>=1.24+opencv-python>=4.8+pillow>=10+onnxruntime-gpu>=1.19 ; platform_system != "Darwin"+onnxruntime>=1.19 ; platform_system == "Darwin"+rfdetr>=1.0+rerun-sdk>=0.21++# ----- CUDA runtime libs for onnxruntime-gpu -----+# onnxruntime-gpu wheels are built against CUDA 12 + cuDNN 9 but do NOT+# bundle them. These pip packages provide the .so files so you don't need+# a system CUDA install. Instaharm calls onnxruntime.preload_dlls() at+# session creation, which finds them automatically (ORT >= 1.19).+nvidia-cudnn-cu12>=9.0,<10 ; platform_system != "Darwin"+nvidia-cuda-runtime-cu12 ; platform_system != "Darwin"+nvidia-cublas-cu12 ; platform_system != "Darwin"++# ----- training / annotation (optional, for reproduction) -----+# torch>=2.2+# timm>=1.0+# tqdm+# loguru+# (For annotation: install Meta's sam-3d-body separately — see docs/annotation.md)diff --git a/tools/annotate_dataset.py b/tools/annotate_dataset.pynew file mode 100755index 0000000..7298948--- /dev/null+++ b/tools/annotate_dataset.py@@ -0,0 +1,876 @@+#!/usr/bin/env python3+"""+Cloud-grade annotation pipeline for CLIFF-style student distillation.++Scans a root directory tree for images, runs YOLO person detection + SAM3D+teacher inference, and saves tight 224×224 crops alongside full-frame-aware+annotations (.npz) ready for CLIFF-style training.++Output layout:+ <output_root>/+ images/ <datasetName>_<imageStem>_p<personID>.jpg (224×224 crop)+ annotations/ <datasetName>_<imageStem>_p<personID>.npz++NPZ payload (per crop):+ orig_shape (2,) [H, W] of the full uncropped image+ bbox (4,) [x1, y1, x2, y2] raw YOLO detection in original image coords+ (used by CLIFF student for bbox_center / bbox_scale conditioning)+ bbox_square (4,) [x1, y1, x2, y2] expanded square crop (1.2×) used for 224×224 resize+ cam_focal_length (2,) [fx, fy] where f = sqrt(H² + W²)+ cam_trans (3,) [tx, ty, tz] from SAM3D (computed with raw bbox, full-frame perspective)+ mhr_model_params (204,) pose + scale params+ shape_params (45,) identity blendshapes+ joints_3d (70, 3) regressed 3D joints (Y-down camera convention, meters)+ joints_2d (70, 2) projected into ORIGINAL full-frame pixel space++Resume-safe: existing annotations are skipped automatically.++Usage:+ python tools/annotate_dataset_sam3d_cloud.py \\+ --root_dir /data/training_images \\+ --output_dir /data/cliff_annotations \\+ --confidence 0.7++ # Quick diverse subset (50k crops sampled across all datasets):+ python tools/annotate_dataset_sam3d_cloud.py \\+ --root_dir /data/training_images \\+ --output_dir /data/cliff_annotations \\+ --shuffle --max_samples 50000++ # Structure of --root_dir:+ # /data/training_images/+ # Harmony4D/ *.jpg *.png ...+ # COCO/ *.jpg ...+ # Synthetic/ *.png ...+"""++from __future__ import annotations++import argparse+import gc+import math+import os+import random+import signal+import sys+import time+from contextlib import contextmanager+from pathlib import Path++# Ensure project root is importable when running from repo root.+_PROJECT_ROOT = Path(__file__).resolve().parent.parent+if str(_PROJECT_ROOT) not in sys.path:+ sys.path.insert(0, str(_PROJECT_ROOT))++import cv2+import numpy as np+import torch+from loguru import logger+from tqdm import tqdm+++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------+IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}+CROP_SIZE = 224+++# ---------------------------------------------------------------------------+# Rerun helpers+# ---------------------------------------------------------------------------+def setup_rerun_blueprint():+ """Send a 3-panel Rerun blueprint: raw image | overlay | 3D mesh."""+ import rerun as rr+ import rerun.blueprint as rrb++ blueprint = rrb.Blueprint(+ rrb.Horizontal(+ rrb.Spatial2DView(+ origin="raw_image",+ name="Raw Image",+ ),+ rrb.Spatial2DView(+ origin="overlay/image",+ name="Overlay",+ ),+ rrb.Spatial3DView(+ origin="scene_3d",+ name="3D Mesh",+ background=rrb.Background(color=[25, 25, 25]),+ ),+ ),+ )+ rr.send_blueprint(blueprint)+++_MAX_RERUN_PERSONS = 10 # upper bound for clearing stale entities+++def log_to_rerun(+ image_rgb: np.ndarray,+ detections: list[dict],+ sam3d_outputs: list[dict | None],+ faces: np.ndarray | None,+) -> None:+ """Log one image with all its person detections to the Rerun viewer.++ All entities are logged as static so Rerun only ever holds one frame in+ memory — previous data is overwritten, not accumulated.++ Args:+ image_rgb: Full uncropped image (H, W, 3), uint8 RGB.+ detections: YOLO detection dicts with ``"bbox"`` keys.+ sam3d_outputs: Parallel list of raw SAM3D output dicts (may contain None).+ faces: Mesh triangle indices from the SAM3D teacher.+ """+ import rerun as rr++ h, w = image_rgb.shape[:2]++ # Clear all person entities from the previous frame to avoid stale meshes+ for i in range(_MAX_RERUN_PERSONS):+ rr.log(f"scene_3d/person_{i}", rr.Clear(recursive=True), static=True)+ rr.log("scene_3d/camera", rr.Clear(recursive=True), static=True)++ # Panel 1: raw image+ rr.log("raw_image", rr.Image(image_rgb), static=True)++ overlay = image_rgb.copy()+ first_cam_t = None+ first_fx, first_fy = None, None+ first_joints_cam = None++ for person_id, (det, sam3d_out) in enumerate(zip(detections, sam3d_outputs)):+ if sam3d_out is None:+ continue++ # Extract outputs+ cam_t = sam3d_out.get("pred_cam_t")+ if cam_t is not None and hasattr(cam_t, "cpu"):+ cam_t = cam_t.cpu().numpy()+ cam_t = np.asarray(cam_t, dtype=np.float32).ravel()[:3] if cam_t is not None else np.zeros(3, dtype=np.float32)++ focal = sam3d_out.get("focal_length")+ if focal is not None and hasattr(focal, "cpu"):+ focal = focal.cpu().numpy()+ if focal is not None:+ focal = np.asarray(focal, dtype=np.float32).ravel()+ fx = float(focal[0])+ fy = float(focal[-1]) # handles both (1,) and (2,)+ else:+ f = math.sqrt(h * h + w * w)+ fx, fy = f, f+ cx, cy = w / 2.0, h / 2.0++ if first_cam_t is None:+ first_cam_t = cam_t+ first_fx, first_fy = fx, fy++ # Vertices+ verts = sam3d_out.get("pred_vertices", sam3d_out.get("vertices", sam3d_out.get("verts")))+ if verts is not None:+ if hasattr(verts, "cpu"):+ verts = verts.cpu().numpy()+ verts = np.asarray(verts, dtype=np.float32)+ if verts.ndim == 3 and verts.shape[0] == 1:+ verts = verts[0]++ verts_cam = verts + cam_t++ # Panel 2: project mesh onto overlay+ z = verts_cam[:, 2]+ valid = z > 0.01+ u = (fx * verts_cam[valid, 0] / z[valid] + cx).astype(int)+ v = (fy * verts_cam[valid, 1] / z[valid] + cy).astype(int)+ mask = (u >= 0) & (u < w) & (v >= 0) & (v < h)+ for px, py in zip(u[mask], v[mask]):+ cv2.circle(overlay, (px, py), 1, (0, 255, 128), -1)++ # Panel 3: 3D mesh+ person_path = f"scene_3d/person_{person_id}"+ if faces is not None:+ rr.log(+ f"{person_path}/mesh",+ rr.Mesh3D(+ vertex_positions=verts_cam,+ triangle_indices=faces,+ albedo_factor=[180, 200, 220, 200],+ ),+ static=True,+ )+ else:+ rr.log(+ f"{person_path}/mesh",+ rr.Points3D(positions=verts_cam, radii=0.003, colors=[180, 200, 220]),+ static=True,+ )++ # 3D joints+ joints_3d = sam3d_out.get("pred_keypoints_3d")+ if joints_3d is not None:+ if hasattr(joints_3d, "cpu"):+ joints_3d = joints_3d.cpu().numpy()+ joints_3d = np.asarray(joints_3d, dtype=np.float32)+ if joints_3d.ndim == 3 and joints_3d.shape[0] == 1:+ joints_3d = joints_3d[0]+ joints_cam = joints_3d + cam_t+ if first_joints_cam is None:+ first_joints_cam = joints_cam+ rr.log(+ f"scene_3d/person_{person_id}/joints",+ rr.Points3D(positions=joints_cam, radii=0.008, colors=[0, 255, 0]),+ static=True,+ )++ rr.log("overlay/image", rr.Image(overlay), static=True)++ # Log Pinhole camera in the 3D scene so the image appears as a properly+ # sized frustum. Resolution must match the full frame (not the 224 crop)+ # because cam_focal_length is computed from sqrt(H² + W²).+ if first_cam_t is not None:+ mesh_depth = float(first_joints_cam[:, 2].mean()) if first_joints_cam is not None else None+ image_plane_dist = max(mesh_depth * 0.25, 0.2) if mesh_depth else 0.5++ rr.log(+ "scene_3d/camera",+ rr.Pinhole(+ resolution=[w, h],+ focal_length=[first_fx, first_fy],+ principal_point=[w / 2.0, h / 2.0],+ image_plane_distance=image_plane_dist,+ ),+ static=True,+ )+ rr.log("scene_3d/camera/image", rr.Image(image_rgb), static=True)+++# ---------------------------------------------------------------------------+# Timeout helper (POSIX — uses SIGALRM)+# ---------------------------------------------------------------------------+class InferenceTimeout(Exception):+ pass+++@contextmanager+def time_limit(seconds: float):+ """Raise InferenceTimeout if the block takes longer than *seconds*."""+ def _handler(signum, frame):+ raise InferenceTimeout(f"Inference exceeded {seconds}s timeout")++ old = signal.signal(signal.SIGALRM, _handler)+ signal.setitimer(signal.ITIMER_REAL, seconds)+ try:+ yield+ finally:+ signal.setitimer(signal.ITIMER_REAL, 0)+ signal.signal(signal.SIGALRM, old)+++# ---------------------------------------------------------------------------+# Image discovery+# ---------------------------------------------------------------------------+def discover_images(root_dir: Path) -> list[tuple[str, Path]]:+ """Walk *root_dir* and return ``(dataset_name, image_path)`` pairs.++ ``dataset_name`` is the name of the first-level subdirectory under+ *root_dir*. Images directly in *root_dir* get dataset_name ``"root"``.+ """+ entries: list[tuple[str, Path]] = []+ root_dir = root_dir.resolve()++ for path in sorted(root_dir.rglob("*")):+ if not path.is_file():+ continue+ if path.suffix.lower() not in IMAGE_EXTENSIONS:+ continue++ # Derive dataset name from the first child directory+ try:+ relative = path.relative_to(root_dir)+ parts = relative.parts+ dataset_name = parts[0] if len(parts) > 1 else "root"+ except ValueError:+ dataset_name = "root"++ entries.append((dataset_name, path))++ return entries+++# ---------------------------------------------------------------------------+# YOLO detector (standalone, no pipeline dependency)+# ---------------------------------------------------------------------------+class YOLOPersonDetector:+ """Thin wrapper around ultralytics YOLO for person detection."""++ def __init__(+ self,+ model_name: str = "yolo11n.pt",+ device: str = "cuda:0",+ confidence: float = 0.7,+ ):+ from ultralytics import YOLO++ self._model = YOLO(model_name)+ self._device = device+ self._confidence = confidence++ def detect(self, image_rgb: np.ndarray) -> list[dict]:+ """Return list of ``{"bbox": [x1,y1,x2,y2], "confidence": float}``."""+ results = self._model(+ image_rgb,+ device=self._device,+ classes=[0], # COCO person+ conf=self._confidence,+ verbose=False,+ )+ detections = []+ for r in results:+ if r.boxes is None:+ continue+ for box, conf in zip(+ r.boxes.xyxy.cpu().numpy(),+ r.boxes.conf.cpu().numpy(),+ ):+ detections.append({+ "bbox": box.astype(np.float32),+ "confidence": float(conf),+ })+ detections.sort(key=lambda d: d["confidence"], reverse=True)+ return detections+++# ---------------------------------------------------------------------------+# SAM3D teacher (uses the estimator directly for max control)+# ---------------------------------------------------------------------------+class SAM3DTeacher:+ """Loads the SAM3D teacher and exposes a simple per-crop inference API.++ We call the ``SAM3DBodyEstimator`` directly (not via the pipeline wrapper)+ so that we have access to the raw output dict with ``pred_cam_t``,+ ``focal_length``, ``mhr_model_params``, etc.+ """++ def __init__(+ self,+ checkpoint_path: str,+ mhr_path: str,+ hf_repo_id: str = "facebook/sam-3d-body-dinov3",+ device: str = "cuda:0",+ ):+ self.device = device++ # Add sam-3d-body to path+ for candidate in [+ Path.cwd() / "sam-3d-body",+ Path.cwd().parent / "sam-3d-body",+ Path.home() / "sam-3d-body",+ ]:+ if candidate.exists() and (candidate / "sam_3d_body").exists():+ if str(candidate) not in sys.path:+ sys.path.insert(0, str(candidate))+ break++ from sam_3d_body import SAM3DBodyEstimator++ ckpt = Path(checkpoint_path)+ if ckpt.exists():+ from sam_3d_body import load_sam_3d_body+ logger.info(f"Loading SAM3D from local checkpoint: {ckpt}")+ model, cfg = load_sam_3d_body(+ checkpoint_path=str(ckpt),+ mhr_path=mhr_path,+ )+ else:+ from sam_3d_body import load_sam_3d_body_hf+ logger.info(f"Loading SAM3D from HuggingFace: {hf_repo_id}")+ model, cfg = load_sam_3d_body_hf(hf_repo_id=hf_repo_id)++ model = model.to(device).eval()+ self._estimator = SAM3DBodyEstimator(model, cfg)+ self._faces = self._estimator.faces++ logger.info(f"SAM3D teacher ready on {device}")++ @property+ def faces(self) -> np.ndarray:+ return self._faces++ @torch.inference_mode()+ def predict_person(+ self,+ full_image_rgb: np.ndarray,+ bbox: np.ndarray,+ ) -> dict | None:+ """Run SAM3D on one person in *full_image_rgb*.++ We pass the **full image** with the bbox in full-frame coordinates.+ SAM3D's CLIFF camera head then uses the bbox position, bbox scale,+ and full-image dimensions to compute ``pred_cam_t`` and+ ``focal_length`` natively in full-frame perspective space — no+ manual coordinate conversion needed.++ Args:+ full_image_rgb: The original uncropped image (H, W, 3), uint8 RGB.+ bbox: ``[x1, y1, x2, y2]`` in full-image pixel coords.++ Returns:+ Raw SAM3D output dict for the single person, or ``None`` on+ failure.+ """+ bbox_arr = np.array(bbox, dtype=np.float32).reshape(1, 4)++ output_list = self._estimator.process_one_image(+ full_image_rgb,+ bboxes=bbox_arr,+ inference_type="body",+ )++ if not output_list:+ return None++ return output_list[0]+++# ---------------------------------------------------------------------------+# Crop & annotation writer+# ---------------------------------------------------------------------------+def make_annotation_name(+ dataset_name: str,+ image_stem: str,+ person_id: int,+) -> str:+ """Deterministic filename: ``<dataset>_<stem>_p<id>``."""+ # Sanitise: replace path separators / spaces that may leak from subfolders+ safe_stem = image_stem.replace(os.sep, "_").replace(" ", "_")+ return f"{dataset_name}_{safe_stem}_p{person_id}"+++def get_square_crop_padded(image: np.ndarray, bbox: np.ndarray, expand: float = 1.2) -> tuple[np.ndarray, np.ndarray]:+ """+ Extracts a perfectly square crop from the image, padding with black pixels+ if the crop goes outside the image boundaries.+ """+ h, w = image.shape[:2]+ x1, y1, x2, y2 = bbox.astype(float)++ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0+ size = max(x2 - x1, y2 - y1) * expand+ half = size / 2.0++ # The perfect square coordinates (may fall outside the image)+ sq_x1, sq_y1 = int(cx - half), int(cy - half)+ sq_x2, sq_y2 = int(cx + half), int(cy + half)++ # Calculate padding needed if the square goes out of bounds+ pad_top = max(0, -sq_y1)+ pad_bottom = max(0, sq_y2 - h)+ pad_left = max(0, -sq_x1)+ pad_right = max(0, sq_x2 - w)++ # Extract the valid portion of the image+ valid_x1, valid_y1 = max(0, sq_x1), max(0, sq_y1)+ valid_x2, valid_y2 = min(w, sq_x2), min(h, sq_y2)+ crop_valid = image[valid_y1:valid_y2, valid_x1:valid_x2]++ # Pad the valid crop to make it perfectly square+ crop_square = cv2.copyMakeBorder(+ crop_valid, pad_top, pad_bottom, pad_left, pad_right,+ cv2.BORDER_CONSTANT, value=(0, 0, 0)+ )++ sq_bbox = np.array([sq_x1, sq_y1, sq_x2, sq_y2], dtype=np.float32)+ return crop_square, sq_bbox+++def save_crop_and_annotation(+ full_image_rgb: np.ndarray,+ bbox: np.ndarray,+ sam3d_output: dict,+ name: str,+ images_dir: Path,+ annotations_dir: Path,+) -> None:+ """Resize the person crop to 224x224, save as JPEG, and write the .npz."""+ h, w = full_image_rgb.shape[:2]++ # Extract perfect square crop (padded if out of bounds)+ crop_square, sq_bbox = get_square_crop_padded(full_image_rgb, bbox, expand=1.2)++ # Resize to 224x224 and save as lossless PNG (RGB -> BGR for cv2)+ crop_224 = cv2.resize(crop_square, (CROP_SIZE, CROP_SIZE), interpolation=cv2.INTER_LINEAR)+ crop_bgr = cv2.cvtColor(crop_224, cv2.COLOR_RGB2BGR)+ cv2.imwrite(str(images_dir / f"{name}.png"), crop_bgr)++ # ... (The rest of Claude's function below this line stays exactly the same!) ...+ f = math.sqrt(h * h + w * w)++ # Extract SAM3D outputs+ cam_trans = sam3d_output.get("pred_cam_t")+ if cam_trans is not None and hasattr(cam_trans, "numpy"):+ cam_trans = cam_trans.numpy()+ cam_trans = np.asarray(cam_trans, dtype=np.float32).ravel()[:3]++ focal = sam3d_output.get("focal_length")+ if focal is not None and hasattr(focal, "numpy"):+ focal = focal.numpy()+ if focal is not None:+ focal = np.asarray(focal, dtype=np.float32).ravel()+ if focal.size == 1:+ focal = np.array([focal[0], focal[0]], dtype=np.float32)+ else:+ focal = np.array([f, f], dtype=np.float32)++ joints_3d = _to_np(sam3d_output.get("pred_keypoints_3d")) # (70, 3)+ joints_2d = _to_np(sam3d_output.get("pred_keypoints_2d")) # (70, 2)+ mhr_model_params = _to_np(sam3d_output.get("mhr_model_params")) # (204,)+ shape_params = _to_np(sam3d_output.get("shape_params")) # (45,)++ # Build the annotation payload+ data: dict[str, np.ndarray] = {+ "orig_shape": np.array([h, w], dtype=np.int32),+ "bbox": bbox.astype(np.float32), # raw YOLO detection — CLIFF student needs this+ "bbox_square": sq_bbox, # expanded square crop used for the 224×224 resize+ "cam_focal_length": focal.astype(np.float32),+ "cam_trans": cam_trans,+ }+ if mhr_model_params is not None:+ data["mhr_model_params"] = mhr_model_params.astype(np.float32)+ if shape_params is not None:+ data["shape_params"] = shape_params.astype(np.float32)+ if joints_3d is not None:+ data["joints_3d"] = joints_3d.astype(np.float32)+ if joints_2d is not None:+ data["joints_2d"] = joints_2d.astype(np.float32)++ np.savez(annotations_dir / f"{name}.npz", **data)+++def _to_np(x) -> np.ndarray | None:+ """Convert torch.Tensor / np.ndarray to a squeezed numpy array."""+ if x is None:+ return None+ if hasattr(x, "cpu"):+ x = x.cpu().numpy()+ x = np.asarray(x, dtype=np.float32)+ # Squeeze leading batch dim if present: (1, ...) → (...)+ if x.ndim >= 2 and x.shape[0] == 1:+ x = x[0]+ return x+++# ---------------------------------------------------------------------------+# CLI+# ---------------------------------------------------------------------------+def parse_args() -> argparse.Namespace:+ p = argparse.ArgumentParser(+ description="Cloud-grade CLIFF annotation pipeline (YOLO + SAM3D)",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog=__doc__,+ )++ # I/O+ p.add_argument(+ "--root_dir", type=str, required=True,+ help="Root directory containing dataset subdirectories of images.",+ )+ p.add_argument(+ "--output_dir", type=str, required=True,+ help="Output directory (images/ and annotations/ created inside).",+ )++ # Detection+ p.add_argument(+ "--yolo_model", type=str, default="yolo11n.pt",+ help="YOLO model name or path (default: yolo11n.pt).",+ )+ p.add_argument(+ "--confidence", type=float, default=0.7,+ help="YOLO person detection confidence threshold (default: 0.7).",+ )+ p.add_argument(+ "--max_persons", type=int, default=10,+ help="Max persons to annotate per image (default: 10).",+ )++ # SAM3D teacher+ p.add_argument(+ "--checkpoint_path", type=str,+ default="checkpoints/sam-3d-body-dinov3/model.ckpt",+ )+ p.add_argument(+ "--mhr_path", type=str,+ default="checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt",+ )+ p.add_argument(+ "--hf_repo", type=str, default="facebook/sam-3d-body-dinov3",+ )++ # Performance+ p.add_argument(+ "--timeout", type=float, default=5.0,+ help="Per-image timeout in seconds (default: 5.0).",+ )+ p.add_argument(+ "--gc_interval", type=int, default=200,+ help="Force GC + CUDA cache clear every N images (default: 200).",+ )+ p.add_argument(+ "--max_samples", type=int, default=None,+ help="Stop after producing N annotation crops (not N images).",+ )++ # Misc+ p.add_argument("--device", type=str, default="cuda:0")+ p.add_argument("--debug", action="store_true")+ p.add_argument(+ "--shuffle", action="store_true",+ help="Shuffle image order for diverse sampling across datasets.",+ )+ p.add_argument(+ "--use_rerun", action="store_true",+ help="Open Rerun viewer with 3 panels: raw image, overlay, 3D mesh.",+ )++ return p.parse_args()+++# ---------------------------------------------------------------------------+# Main+# ---------------------------------------------------------------------------+def main() -> None:+ args = parse_args()++ # Logging+ logger.remove()+ fmt = (+ "<green>{time:HH:mm:ss}</green> | <level>{level:<8}</level> | "+ "<cyan>{function}</cyan> - <level>{message}</level>"+ )+ logger.add(sys.stderr, level="DEBUG" if args.debug else "INFO", format=fmt)++ root_dir = Path(args.root_dir)+ output_dir = Path(args.output_dir)+ images_dir = output_dir / "images"+ annotations_dir = output_dir / "annotations"+ images_dir.mkdir(parents=True, exist_ok=True)+ annotations_dir.mkdir(parents=True, exist_ok=True)++ # ------------------------------------------------------------------+ # 1. Discover images+ # ------------------------------------------------------------------+ logger.info(f"Scanning {root_dir} for images ...")+ all_images = discover_images(root_dir)+ if not all_images:+ logger.error(f"No images found under {root_dir}")+ return++ if args.shuffle:+ random.shuffle(all_images)+ logger.info("Image order shuffled for diverse cross-dataset sampling")++ logger.info(f"Found {len(all_images)} images across {len(set(d for d, _ in all_images))} dataset(s)")+ if args.max_samples:+ logger.info(f"Will stop after {args.max_samples} annotation crops")++ # ------------------------------------------------------------------+ # 2. Build resume set — collect existing annotation stems+ # ------------------------------------------------------------------+ existing_stems: set[str] = set()+ for p in annotations_dir.glob("*.npz"):+ existing_stems.add(p.stem)++ if existing_stems:+ logger.info(f"Resume: {len(existing_stems)} annotations already on disk")++ # Count existing annotations toward the budget so resumed runs stop correctly+ total_crops = len(existing_stems)++ # ------------------------------------------------------------------+ # 3. Load models+ # ------------------------------------------------------------------+ logger.info("Loading YOLO detector ...")+ detector = YOLOPersonDetector(+ model_name=args.yolo_model,+ device=args.device,+ confidence=args.confidence,+ )++ logger.info("Loading SAM3D teacher ...")+ teacher = SAM3DTeacher(+ checkpoint_path=args.checkpoint_path,+ mhr_path=args.mhr_path,+ hf_repo_id=args.hf_repo,+ device=args.device,+ )++ # ------------------------------------------------------------------+ # 3b. Rerun setup+ # ------------------------------------------------------------------+ use_rerun = args.use_rerun+ if use_rerun:+ try:+ import rerun as rr+ rr.init("sam3d_cloud_annotation", spawn=True)+ rr.disable_timeline("frame")+ rr.log("scene_3d", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True)+ setup_rerun_blueprint()+ logger.info("Rerun viewer launched (3-panel layout, no history)")+ except ImportError:+ logger.warning("rerun-sdk not installed, disabling --use_rerun")+ use_rerun = False++ # ------------------------------------------------------------------+ # 4. Processing loop+ # ------------------------------------------------------------------+ stats = {"processed": 0, "skipped": 0, "failed": 0, "timed_out": 0, "crops": 0}+ t0 = time.perf_counter()++ pbar = tqdm(all_images, desc="Annotating", unit="img", dynamic_ncols=True)+ for dataset_name, img_path in pbar:+ # Use relative path from root so subfolder structure prevents collisions+ # e.g. COCO/train/000001.jpg → "train_000001" (dataset_name = "COCO")+ rel_path = img_path.relative_to(root_dir)+ image_stem = str(rel_path.with_suffix("")).replace(os.sep, "_").replace(" ", "_")+ # Strip the dataset prefix since make_annotation_name re-adds it+ if image_stem.startswith(dataset_name + "_"):+ image_stem = image_stem[len(dataset_name) + 1:]++ # Quick check: can we skip the entire image?+ # We check for person 0; if it exists the image was processed before.+ first_name = make_annotation_name(dataset_name, image_stem, 0)+ if first_name in existing_stems:+ stats["skipped"] += 1+ continue++ # Load image+ img_bgr = cv2.imread(str(img_path))+ if img_bgr is None:+ logger.warning(f"Cannot read {img_path}")+ stats["failed"] += 1+ continue+ img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)+ H, W = img_rgb.shape[:2]++ # Detect persons+ try:+ with time_limit(args.timeout):+ detections = detector.detect(img_rgb)+ except InferenceTimeout:+ logger.warning(f"YOLO timeout on {img_path.name}")+ stats["timed_out"] += 1+ continue+ except Exception as e:+ logger.warning(f"YOLO failed on {img_path.name}: {e}")+ stats["failed"] += 1+ continue++ if not detections:+ stats["processed"] += 1+ continue++ detections = detections[: args.max_persons]++ # Process each detected person — pass the FULL image so SAM3D's+ # CLIFF camera head uses the correct bbox_center, bbox_size, and+ # img_size to produce cam_trans in full-frame perspective space.+ rerun_sam3d_outputs: list[dict | None] = []+ for person_id, det in enumerate(detections):+ name = make_annotation_name(dataset_name, image_stem, person_id)++ # Skip if this specific crop already exists+ if name in existing_stems:+ rerun_sam3d_outputs.append(None)+ continue++ bbox = det["bbox"]++ try:+ with time_limit(args.timeout):+ sam3d_out = teacher.predict_person(img_rgb, bbox)+ except InferenceTimeout:+ logger.warning(f"SAM3D timeout on {name}")+ stats["timed_out"] += 1+ rerun_sam3d_outputs.append(None)+ continue+ except Exception as e:+ logger.warning(f"SAM3D failed on {name}: {e}")+ torch.cuda.empty_cache()+ if args.debug:+ raise+ stats["failed"] += 1+ rerun_sam3d_outputs.append(None)+ continue++ if sam3d_out is None:+ stats["failed"] += 1+ rerun_sam3d_outputs.append(None)+ continue++ # Validate essential outputs+ if sam3d_out.get("mhr_model_params") is None:+ logger.warning(f"No mhr_model_params for {name} (keys: {list(sam3d_out.keys())})")+ stats["failed"] += 1+ rerun_sam3d_outputs.append(None)+ continue++ try:+ save_crop_and_annotation(+ img_rgb, bbox, sam3d_out, name,+ images_dir, annotations_dir,+ )+ stats["crops"] += 1+ total_crops += 1+ except Exception as e:+ logger.warning(f"Failed to save {name}: {e}")+ if args.debug:+ raise+ stats["failed"] += 1+ rerun_sam3d_outputs.append(None)+ continue++ rerun_sam3d_outputs.append(sam3d_out)++ # Rerun visualization (after all persons for this image)+ if use_rerun and any(o is not None for o in rerun_sam3d_outputs):+ log_to_rerun(img_rgb, detections, rerun_sam3d_outputs, teacher.faces)++ stats["processed"] += 1++ # Stop once we have enough crops+ if args.max_samples and total_crops >= args.max_samples:+ logger.info(f"Reached {total_crops} total crops (--max_samples {args.max_samples}), stopping")+ break++ # Periodic cleanup+ if stats["processed"] % args.gc_interval == 0:+ torch.cuda.empty_cache()+ gc.collect()++ # Update progress bar+ pbar.set_postfix(+ crops=stats["crops"],+ fail=stats["failed"],+ skip=stats["skipped"],+ )++ elapsed = time.perf_counter() - t0+ hours = elapsed / 3600++ logger.info("=" * 60)+ logger.info("ANNOTATION COMPLETE")+ logger.info("=" * 60)+ logger.info(f" Images scanned: {len(all_images)}")+ logger.info(f" Images processed: {stats['processed']}")+ logger.info(f" Images skipped: {stats['skipped']} (already annotated)")+ logger.info(f" Crops saved: {stats['crops']} (total on disk: {total_crops})")+ logger.info(f" Failures: {stats['failed']}")+ logger.info(f" Timeouts: {stats['timed_out']}")+ logger.info(f" Wall time: {hours:.2f}h ({elapsed / max(stats['processed'], 1):.2f}s/img)")+ logger.info(f" Output: {output_dir}")+ logger.info("=" * 60)+++if __name__ == "__main__":+ main()