Skip to content
Python

first commit

mohamdevec1bc434458e
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..39052df
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,40 @@
+# Byte-compiled / cache
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# Editor / OS
+.vscode/
+.idea/
+.DS_Store
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# RF-DETR auto-downloaded weights
+rf-detr*.pth
+*.pth.lock
+*.onnx
+*.pth
+
+# Rerun recordings
+*.rrd
+
+# Annotation outputs
+data/
+annotations/
+images/
+
+# Model weights — too large for git, use Git LFS or release artifacts
+models/*.onnx
+models/*.pt
+models/*.pth
+!models/README.md
+
+# Notebook checkpoints
+.ipynb_checkpoints/
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..0b55932
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,216 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for describing the origin of the Work and
+ reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may accept and charge a
+ fee for, the acceptance of support, warranty, indemnity, or other
+ liability obligations and/or rights consistent with this License.
+ However, in accepting such obligations, You may act only on Your
+ own behalf and on Your sole responsibility, not on behalf of any
+ other Contributor, and only if You agree to indemnify, defend, and
+ hold each Contributor harmless for any liability incurred by, or
+ claims asserted against, such Contributor by reason of your accepting
+ any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2026 InstantHMR authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied. See the License for the specific language governing permissions
+ and limitations under the License.
+
+================================================================================
+
+THIRD-PARTY NOTICES
+-------------------
+
+The InstantHMR model weights distributed at
+https://huggingface.co/momolesang/InstantHMR are released under the SAM
+license, since the model is a distillation of Meta's SAM 3D Body
+(https://github.com/facebookresearch/sam-3d-body). Refer to that
+repository for the full SAM license text and applicable use restrictions.
+
+The demo uses Roboflow's RF-DETR for person detection, which is
+distributed under its own license. See https://github.com/roboflow/rf-detr
+for terms.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5b8dafb
--- /dev/null
+++ b/README.md
@@ -0,0 +1,114 @@
+# InstantHMR
+
+A lightweight, ONNX-exportable distillation of
+[`facebook/sam-3d-body-dinov3`](https://huggingface.co/facebook/sam-3d-body-dinov3)
+for **3D human pose estimation and mesh recovery**: a RepViT-M1.5 backbone + a 9-token
+cross-attention decoder + CLIFF camera conditioning. Trained to mimic the
+SAM3D teacher's per-person 70-keypoint outputs from a single 224×224 crop.
+
+InstantHMR ships as a single `.onnx` file. The demo pipeline pairs it with
+**RF-DETR** for person detection — both stages are timed independently so
+you always know where the latency is going.
+
+- **Inputs:** `image (N, 3, 224, 224)`, `cliff_cond (N, 3)`.
+- **Outputs:** `mhr_params (204)`, `shape_params (45)`, `cam_trans (3)`,
+ `joints_2d (70, 2)` (normalised crop coords),
+ `joints_3d (70, 3)` (body-centred metres, Y-down).
+- **Speed:** ~5 ms / frame (~200 FPS) on a single RTX 4070 with the fp16
+ ONNX (CUDA EP); CPU works too (~25 FPS, depending on hardware).
+
+> **Note on the body mesh.** InstantHMR does **not** run the MHR mesh
+> decoder — rendering the full body mesh is expensive and we want the
+> default pipeline to stay lightweight. The raw `mhr_params` and
+> `shape_params` are still exposed on every `HMRPrediction` so you can
+> feed them into Meta's MHR TorchScript module from the original
+> [SAM 3D Body release](https://github.com/facebookresearch/sam-3d-body)
+> if you need a mesh.
+
+## Install
+
+```bash
+git clone <this-repo> instanthmr
+cd instanthmr
+
+conda create -n instanthmr python=3.11 -y
+conda activate instanthmr
+
+pip install -r requirements.txt
+```
+
+The first run downloads the RF-DETR (medium) checkpoint into the rfdetr
+cache directory automatically. The InstantHMR ONNX weights are released
+separately on HuggingFace — see [Model weights](#model-weights) below.
+
+## Run the demo
+
+```bash
+# Single image
+python demo.py --image path/to/photo.jpg
+
+# Video file
+python demo.py --video path/to/clip.mp4
+
+# Live webcam (index 0)
+python demo.py --camera 0
+```
+
+`python demo.py --help` lists every flag (model path, device, detector
+variant, confidence, max persons, frame skip, `.rrd` recording, …).
+
+The demo opens a Rerun viewer with the source image + 2D skeleton, a live
+detector / HMR / total-latency plot, and the 3D scene with the predicted
+camera frustum.
+
+## Use it from Python
+
+```python
+from instanthmr import PosePipeline
+
+pipeline = PosePipeline(
+ onnx_path="models/instanthmr.onnx",
+ device="cuda",
+ detector_variant="medium",
+)
+
+result = pipeline.predict(image_rgb)
+for r in result.persons:
+ print(r.joints_3d_cam.shape) # (70, 3)
+ print(r.joints_2d.shape) # (70, 2)
+ print(r.mhr_params.shape) # (204,) — for downstream MHR mesh decoding
+ print(r.shape_params.shape) # (45,)
+```
+
+The 70 keypoints follow the MHR70 ordering — see
+[`instanthmr/skeleton.py`](instanthmr/skeleton.py) for joint names and the
+canonical skeleton edge list.
+
+## Model weights
+
+The InstantHMR ONNX weights are hosted on HuggingFace:
+
+> **<https://huggingface.co/momolesang/InstantHMR>**
+
+Download `instanthmr.onnx` and place it under `models/`, or pass any path
+via `--model`.
+
+## Documentation
+
+- **[`docs/architecture.md`](docs/architecture.md)** — what the network
+ looks like (RepViT backbone, 9-query decoder, CLIFF condition).
+- **[`docs/annotation.md`](docs/annotation.md)** — how the training data
+ was generated with the SAM3D teacher.
+- **[`docs/training.md`](docs/training.md)** — how the released checkpoint
+ was distilled and exported to ONNX.
+
+## License
+
+The code in this repository is released under the **Apache License 2.0**
+(see [LICENSE](LICENSE)).
+
+The model weights distributed at
+<https://huggingface.co/momolesang/InstantHMR> are released under the
+[SAM license](https://github.com/facebookresearch/sam-3d-body), since
+InstantHMR is a distillation of `facebook/sam-3d-body-dinov3`. Please
+review the SAM and RF-DETR licenses before downstream use.
diff --git a/demo.py b/demo.py
new file mode 100755
index 0000000..6b91441
--- /dev/null
+++ b/demo.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""InstantHMR — demo runner.
+
+Detects persons in a video / image / live camera stream with RF-DETR, runs
+InstantHMR on each, and visualises the results in Rerun: 3D joints, camera
+pose as a pinhole frustum, the source image with the projected 2D skeleton
+drawn on top, and a live latency plot showing RF-DETR vs InstantHMR cost
+separately.
+
+Usage
+-----
+ # Single image
+ python demo.py --image path/to/photo.jpg
+
+ # Video file
+ python demo.py --video path/to/clip.mp4
+
+ # Live camera (index 0)
+ python demo.py --camera 0
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+from instanthmr import PosePipeline
+from instanthmr.visualizer import RerunVisualizer
+
+
+DEFAULT_MODEL = "models/instanthmr.onnx"
+
+
+def parse_args() -> argparse.Namespace:
+ p = argparse.ArgumentParser(
+ description="Run InstantHMR (RF-DETR + ONNX HMR) on a video, image, or camera.",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+
+ src = p.add_mutually_exclusive_group(required=True)
+ src.add_argument("--video", type=str, help="Path to an input video file.")
+ src.add_argument("--image", type=str, help="Path to an input image.")
+ src.add_argument(
+ "--camera", type=int, metavar="IDX",
+ help="Live camera index (0, 1, ...).",
+ )
+
+ p.add_argument(
+ "--model", type=str, default=DEFAULT_MODEL,
+ help=f"Path to the InstantHMR ONNX file (default: {DEFAULT_MODEL}).",
+ )
+ p.add_argument(
+ "--device", type=str, default="cuda",
+ help="Inference device for InstantHMR: 'cuda' or 'cpu' (default: cuda).",
+ )
+ p.add_argument(
+ "--detector-variant", type=str, default="medium",
+ choices=["nano", "small", "medium", "base", "large"],
+ help="RF-DETR size variant (default: medium).",
+ )
+ p.add_argument(
+ "--det-confidence", type=float, default=0.5,
+ help="RF-DETR detection confidence threshold (default: 0.5).",
+ )
+ p.add_argument(
+ "--max-persons", type=int, default=5,
+ help="Max persons processed per frame (default: 5).",
+ )
+ p.add_argument(
+ "--frame-skip", type=int, default=1,
+ help="Process every Nth video frame (default: 1).",
+ )
+ p.add_argument(
+ "--save-rrd", type=str, default=None,
+ help="Optional path to save the Rerun recording as a .rrd file.",
+ )
+ p.add_argument(
+ "--no-spawn", action="store_true",
+ help="Don't spawn the Rerun viewer GUI (useful when only saving .rrd).",
+ )
+
+ return p.parse_args()
+
+
+def build_pipeline(args: argparse.Namespace) -> PosePipeline:
+ model_path = Path(args.model)
+ if not model_path.exists():
+ sys.stderr.write(
+ f"\n[error] InstantHMR ONNX not found: {model_path}\n"
+ " Place 'instanthmr.onnx' under models/, or pass\n"
+ " --model /path/to/instanthmr.onnx\n\n"
+ )
+ sys.exit(1)
+
+ return PosePipeline(
+ onnx_path=model_path,
+ device=args.device,
+ detector_variant=args.detector_variant,
+ det_confidence=args.det_confidence,
+ max_persons=args.max_persons,
+ )
+
+
+def build_visualizer(args: argparse.Namespace) -> RerunVisualizer:
+ return RerunVisualizer(
+ application_id="instanthmr_demo",
+ spawn_viewer=not args.no_spawn,
+ save_path=args.save_rrd,
+ )
+
+
+def _print_timings(
+ prefix: str,
+ detector_ms: float,
+ hmr_ms: float,
+ total_ms: float,
+ n_persons: int,
+) -> None:
+ fps = 1000.0 / total_ms if total_ms > 0 else 0.0
+ print(
+ f"{prefix} "
+ f"persons={n_persons:>2d} "
+ f"RF-DETR={detector_ms:6.1f} ms "
+ f"InstantHMR={hmr_ms:6.1f} ms "
+ f"total={total_ms:6.1f} ms ({fps:5.1f} fps)"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Source-specific runners
+# ---------------------------------------------------------------------------
+
+
+def run_image(args: argparse.Namespace) -> None:
+ image_path = Path(args.image)
+ if not image_path.exists():
+ sys.exit(f"[error] image not found: {image_path}")
+
+ bgr = cv2.imread(str(image_path))
+ if bgr is None:
+ sys.exit(f"[error] cv2.imread failed for {image_path}")
+ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+
+ pipeline = build_pipeline(args)
+ viz = build_visualizer(args)
+
+ # Warm-up: first call includes lazy CUDA / ORT initialisation, so a real
+ # latency reading is much more useful from the second call onwards.
+ _ = pipeline.predict(rgb)
+ result = pipeline.predict(rgb)
+
+ viz.log_frame(
+ rgb, result.persons,
+ frame_idx=0, timestamp=0.0,
+ detector_ms=result.detector_ms,
+ hmr_ms=result.hmr_ms,
+ total_ms=result.total_ms,
+ )
+ _print_timings("image:", result.detector_ms, result.hmr_ms, result.total_ms, len(result.persons))
+
+
+def run_video(args: argparse.Namespace) -> None:
+ video_path = Path(args.video)
+ if not video_path.exists():
+ sys.exit(f"[error] video not found: {video_path}")
+
+ cap = cv2.VideoCapture(str(video_path))
+ if not cap.isOpened():
+ sys.exit(f"[error] cv2.VideoCapture failed for {video_path}")
+
+ fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
+ total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ print(f"video: {video_path.name}, {total} frames @ {fps:.1f} fps")
+
+ pipeline = build_pipeline(args)
+ viz = build_visualizer(args)
+
+ frame_idx = 0
+ sent = 0
+ det_times: list[float] = []
+ hmr_times: list[float] = []
+ tot_times: list[float] = []
+ try:
+ while True:
+ ret, bgr = cap.read()
+ if not ret:
+ break
+
+ if frame_idx % args.frame_skip != 0:
+ frame_idx += 1
+ continue
+
+ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+
+ result = pipeline.predict(rgb)
+ det_times.append(result.detector_ms)
+ hmr_times.append(result.hmr_ms)
+ tot_times.append(result.total_ms)
+
+ viz.log_frame(
+ rgb, result.persons,
+ frame_idx=frame_idx,
+ timestamp=frame_idx / fps,
+ detector_ms=result.detector_ms,
+ hmr_ms=result.hmr_ms,
+ total_ms=result.total_ms,
+ )
+
+ sent += 1
+ frame_idx += 1
+
+ if sent % 30 == 0:
+ _print_timings(
+ f" frame {frame_idx:>5d}/{total:<5d}",
+ float(np.mean(det_times[-30:])),
+ float(np.mean(hmr_times[-30:])),
+ float(np.mean(tot_times[-30:])),
+ len(result.persons),
+ )
+ except KeyboardInterrupt:
+ print("\ninterrupted")
+ finally:
+ cap.release()
+
+ if tot_times:
+ # Skip the first frame from the average — it bears the warm-up cost.
+ warm = slice(1, None) if len(tot_times) > 1 else slice(None)
+ print(
+ "\nsummary "
+ f"frames={sent:d} "
+ f"RF-DETR={float(np.mean(det_times[warm])):.1f} ms "
+ f"InstantHMR={float(np.mean(hmr_times[warm])):.1f} ms "
+ f"total={float(np.mean(tot_times[warm])):.1f} ms "
+ f"({1000.0 / float(np.mean(tot_times[warm])):.1f} fps)"
+ )
+
+
+def run_camera(args: argparse.Namespace) -> None:
+ cap = cv2.VideoCapture(args.camera)
+ if not cap.isOpened():
+ sys.exit(f"[error] cannot open camera index {args.camera}")
+
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
+ print(f"camera {args.camera}: {width}x{height} @ {fps:.1f} fps — Ctrl+C to stop")
+
+ pipeline = build_pipeline(args)
+ viz = build_visualizer(args)
+
+ frame_idx = 0
+ det_times: list[float] = []
+ hmr_times: list[float] = []
+ tot_times: list[float] = []
+ try:
+ while True:
+ ret, bgr = cap.read()
+ if not ret:
+ print("[warn] camera read failed")
+ break
+
+ rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+ result = pipeline.predict(rgb)
+ det_times.append(result.detector_ms)
+ hmr_times.append(result.hmr_ms)
+ tot_times.append(result.total_ms)
+
+ viz.log_frame(
+ rgb, result.persons,
+ frame_idx=frame_idx,
+ timestamp=frame_idx / fps,
+ detector_ms=result.detector_ms,
+ hmr_ms=result.hmr_ms,
+ total_ms=result.total_ms,
+ )
+
+ if frame_idx and frame_idx % 30 == 0:
+ _print_timings(
+ f" frame {frame_idx:>5d}",
+ float(np.mean(det_times[-30:])),
+ float(np.mean(hmr_times[-30:])),
+ float(np.mean(tot_times[-30:])),
+ len(result.persons),
+ )
+ frame_idx += 1
+ except KeyboardInterrupt:
+ print("\nstopped")
+ finally:
+ cap.release()
+
+
+def main() -> None:
+ args = parse_args()
+ if args.image is not None:
+ run_image(args)
+ elif args.video is not None:
+ run_video(args)
+ else:
+ run_camera(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docs/annotation.md b/docs/annotation.md
new file mode 100644
index 0000000..7496f5b
--- /dev/null
+++ b/docs/annotation.md
@@ -0,0 +1,109 @@
+# Building the training dataset
+
+InstantHMR is supervised entirely by the **SAM3D teacher**
+(`facebook/sam-3d-body-dinov3`): for every detector-found person we run
+the teacher and store its outputs alongside a tight 224×224 crop. The
+training loop in `notebooks/distill_transformer_decoder.ipynb` then learns
+to mimic those teacher outputs from the crop alone.
+
+The bundled `tools/annotate_dataset.py` uses YOLO for the
+detector during annotation. That choice is independent of the runtime
+detector — InstantHMR sees the same 1.2× square crop either way, and the
+demo uses RF-DETR at inference time for better recall. If you re-run the
+annotation with a different detector, just preserve the crop geometry.
+
+The annotation pipeline is `tools/annotate_dataset.py`.
+
+## What it does
+
+```
+root_dir/ output_dir/
+├── COCO/ ─┐ ├── images/
+├── Harmony4D/ │ YOLO + SAM3D │ COCO_<stem>_p0.png (224×224)
+├── Synthetic/ │ ──────────────► │ COCO_<stem>_p0.npz
+└── … ─┘ │ …
+ └── annotations/
+ COCO_<stem>_p0.npz
+ …
+```
+
+For each detected person (capped at `--max_persons` per image):
+
+1. **YOLO 11n** detects persons (`conf ≥ --confidence`, default 0.7).
+2. The **full-resolution** image is passed to SAM3D with the raw bbox.
+ SAM3D's CLIFF camera head sees the full frame, so `pred_cam_t` and
+ `focal_length` are produced in real perspective coordinates — no manual
+ camera math.
+3. A 1.2× **square** crop around the bbox is resized to 224×224 and stored
+ as a PNG.
+4. The teacher outputs are stored as a `.npz` next to the image:
+
+ | Key | Shape | Description |
+ |-----|-------|-------------|
+ | `orig_shape` | `(2,)` | `[H, W]` of the original frame |
+ | `bbox` | `(4,)` | raw YOLO `[x1, y1, x2, y2]` |
+ | `bbox_square` | `(4,)` | the 1.2× expanded square crop bounds |
+ | `cam_focal_length` | `(2,)` | `[fx, fy]` (≈ √(H²+W²)) |
+ | `cam_trans` | `(3,)` | SAM3D translation in camera frame |
+ | `mhr_model_params` | `(204,)` | MHR pose + scale |
+ | `shape_params` | `(45,)` | MHR identity blendshapes |
+ | `joints_3d` | `(70, 3)` | SAM3D `pred_keypoints_3d` |
+ | `joints_2d` | `(70, 2)` | SAM3D `pred_keypoints_2d`, in **full-frame** pixels |
+
+The script is **resume-safe**: it scans existing `.npz` filenames at startup
+and skips any image whose first crop is already on disk.
+
+## Running it
+
+```bash
+# Setup: clone Meta's sam-3d-body and download the dinov3 teacher checkpoint.
+git clone https://github.com/facebookresearch/sam-3d-body
+huggingface-cli download facebook/sam-3d-body-dinov3 \
+ --local-dir checkpoints/sam-3d-body-dinov3
+
+# Annotate a tree of images
+python tools/annotate_dataset.py \
+ --root_dir /data/training_images \
+ --output_dir /data/cliff_annotations \
+ --confidence 0.7
+
+# Quick balanced subset (50k crops, shuffled across datasets)
+python tools/annotate_dataset.py \
+ --root_dir /data/training_images \
+ --output_dir /data/cliff_annotations \
+ --shuffle --max_samples 50000
+
+# Visually inspect what's getting saved
+python tools/annotate_dataset.py \
+ --root_dir /data/training_images \
+ --output_dir /data/cliff_annotations \
+ --use_rerun
+```
+
+Useful options:
+
+- `--shuffle` — process images in random order so a partial run still spans
+ the full dataset diversity.
+- `--max_samples N` — stop after N **crops** are saved (not images), counting
+ whatever was already on disk.
+- `--timeout 5.0` — per-image SIGALRM; rare GPU hangs don't kill the run.
+- `--gc_interval 200` — periodic CUDA cache + GC reset to keep VRAM stable
+ across long runs.
+
+## Why a separate cloud script?
+
+This is the same per-person preprocessing the demo uses at inference time
+(`InstantHMR._preprocess`). Aligning the training data with the runtime
+preprocessing was the single biggest factor in getting InstantHMR to match
+SAM3D's accuracy: any drift between annotation crops and inference crops
+shows up directly as test-time error.
+
+## Datasets we used
+
+InstantHMR was trained on a mix of:
+
+- **COCO** (in-the-wild humans),
+- **Harmony4D** (multi-person studio captures),
+- a **synthetic dataset** of rendered humans, and
+- domain-specific footage (rugby, aikido) that the model is meant to handle.
+
diff --git a/docs/architecture.md b/docs/architecture.md
new file mode 100644
index 0000000..4017009
--- /dev/null
+++ b/docs/architecture.md
@@ -0,0 +1,100 @@
+# Model architecture
+
+InstantHMR is a 17 M-parameter network that mimics
+`facebook/sam-3d-body-dinov3` on the **per-person 3D pose estimation** task,
+running ~80× faster on a single GPU and exporting cleanly to ONNX / TFLite /
+QNN for mobile and edge deployment.
+
+## High level
+
+```
+ RF-DETR ──► square 1.2× crop ──► RepViT backbone ──► cross-attn decoder
+ │
+ CLIFF cond [cx, cy, scale] ──┘
+ ▼
+ mhr_params (204), shape (45), cam_trans (3),
+ joints_2d (70, 2), joints_3d (70, 3)
+```
+
+## Backbone — RepViT-M1.5
+
+`timm.create_model("repvit_m1_5", pretrained=False, features_only=True)`
+gives a `(B, 512, 7, 7)` feature map at 224 input. Re-parameterised
+inverted-residual blocks make this fast on both desktop and mobile CPUs.
+
+## Decoder — 9-query cross-attention
+
+Two `nn.TransformerDecoderLayer` blocks (`d_model=256`, 4 heads, GELU,
+pre-LN) attend the flattened backbone features (49 tokens) with 9 learnable
+query tokens:
+
+| Query | Output | Head shape |
+|-------|------------------------------|-----------:|
+| 0 | camera translation (CLIFF) | (3,) |
+| 1 | pose (axis-angle joints) | (136,) |
+| 2 | scales | (68,) |
+| 3 | identity shape blendshapes | (45,) |
+| 4–8 | torso / L arm / R arm / L leg / R leg → joints | (5×256 → 70×3) |
+
+A fixed 2D sin-cos positional embedding is added to the 7×7 token grid.
+
+## CLIFF camera conditioning
+
+The camera-translation head is the only one that consumes the CLIFF
+conditioning vector — `[cx_norm, cy_norm, b_scale]` of the detector bbox
+(RF-DETR at runtime, YOLO during annotation) in **full-frame** coordinates.
+This is what lets InstantHMR predict translation consistent with the
+original perspective of the source image, without ever seeing pixels
+outside the 224×224 crop.
+
+```
+cam_token (B, 256)
+cliff_cond (B, 3)
+ │ │
+ └─────cat─────┘
+ │
+ LayerNorm + MLP
+ │
+ ▼
+ cam_trans (B, 3)
+```
+
+## Output coordinate frame
+
+Both `joints_3d` and `cam_trans` are in metres in the SAM3D camera frame:
+
+- **X right**, **Y down**, **Z forward** (right-handed, Y-down).
+- `joints_3d` are body-centred — add `cam_trans` to land in camera space.
+- `joints_2d` come from a dedicated 2D head, in **normalised crop coords**
+ `[-1, 1]`; the demo's `instanthmr/inference.py` re-projects them to
+ full-frame pixels using the bbox-derived crop transform.
+
+## Why distill?
+
+The DINOv3-based teacher is excellent but heavy: its tokenizer + decoder runs
+at <20 FPS on a desktop GPU and doesn't fit comfortably on edge hardware.
+Distillation gives us InstantHMR — a model that:
+
+- runs at ~150 FPS on a single RTX 4070,
+- exports to ONNX / TFLite / QNN, and
+- preserves the SAM3D 70-keypoint & MHR outputs, so any downstream code that
+ consumed `pred_keypoints_3d` keeps working.
+
+## Detection: RF-DETR
+
+The demo pipeline uses **RF-DETR** (Roboflow's transformer-based detector)
+for the upstream person detection step. We picked it over YOLO because:
+
+- it has **stronger recall on small / partially-occluded persons**, which
+ matters a lot once the bbox is tight enough to drive a pose model;
+- its single-stage transformer head produces calibrated confidence scores,
+ so the `--det-confidence` knob behaves predictably; and
+- it ships with a `optimize_for_inference()` pass (TorchScript +
+ fused ops) that reliably halves detector latency on a desktop GPU.
+
+At inference the demo reports detector and HMR latency **separately**, both
+in the console and as scalar plots in the Rerun viewer, so it's always
+clear whether RF-DETR or InstantHMR is the bottleneck on a given machine.
+The default `--detector-variant medium` is tuned for ~480p–1080p input.
+Drop to `nano` / `small` for higher FPS at lower resolution, or step up to
+`base` / `large` for crowded scenes.
diff --git a/docs/training.md b/docs/training.md
new file mode 100644
index 0000000..3aeeaf1
--- /dev/null
+++ b/docs/training.md
@@ -0,0 +1,112 @@
+# Training
+
+The InstantHMR training notebook is
+[`notebooks/distill_transformer_decoder.ipynb`](../notebooks/distill_transformer_decoder.ipynb).
+It is the source of truth — this page is a tour, not a re-implementation.
+
+## Setup
+
+```bash
+pip install torch timm onnx tqdm loguru
+jupyter notebook notebooks/distill_transformer_decoder.ipynb
+```
+
+You'll need the annotations produced by
+[`docs/annotation.md`](annotation.md). The notebook expects the directory
+layout:
+
+```
+<dataset_root>/
+├── images/ *.png (224×224 crops)
+└── annotations/ *.npz (teacher outputs)
+```
+
+## What the notebook does
+
+1. **Dataset** — loads the `(image, annotation)` pairs, applies light
+ augmentation (colour jitter, horizontal flip with paired-joint remapping),
+ and yields the 4-tuple `(image, cliff_cond, gt_targets, mask)`.
+2. **Architecture** — defines InstantHMR (RepViT-M1.5 backbone + 9-token
+ cross-attention decoder) exactly as it ships in this repo's
+ `instanthmr/inference.py` (the runtime path is parameter-compatible
+ with the trained `.pth`).
+3. **Loss** — weighted L1 on each output head:
+ - mhr_params, shape_params, cam_trans (parameter regression),
+ - joints_3d (mean per-joint position error in camera frame),
+ - joints_2d (after re-projection back to full-frame pixels).
+4. **Training loop** — AdamW with cosine schedule + warm-up, mixed precision
+ on CUDA, gradient clipping at 1.0.
+5. **Validation** — held-out split, MPJPE on `joints_3d` reported.
+6. **Export** — at the end of training, the model is exported to
+ `instanthmr.onnx` with dynamic batch axis using the same input schema
+ the demo expects (see [`models/README.md`](../models/README.md)).
+
+## Hyperparameters at a glance
+
+| | |
+|---|---|
+| Backbone | RepViT-M1.5 (`timm`) |
+| Decoder layers | 2 |
+| `d_model` | 256 |
+| Heads | 4 |
+| Dropout | 0.1 |
+| Image size | 224 |
+| Optimiser | AdamW (β=0.9/0.999, wd=1e-2) |
+| LR schedule | cosine + linear warm-up |
+| Precision | mixed (fp16 forward, fp32 master) |
+
+The exact values for the released checkpoint live in the notebook's config
+cell.
+
+## Re-exporting to ONNX
+
+The notebook's export cell is reproduced below for reference:
+
+```python
+dummy_image = torch.randn(1, 3, 224, 224, device=device)
+dummy_cliff = torch.randn(1, 3, device=device)
+
+torch.onnx.export(
+ model.eval(),
+ (dummy_image, dummy_cliff),
+ "instanthmr.onnx",
+ input_names=["image", "cliff_cond"],
+ output_names=[
+ "mhr_params", "shape_params", "cam_trans",
+ "joints_2d", "joints_3d",
+ ],
+ dynamic_axes={
+ "image": {0: "batch"},
+ "cliff_cond": {0: "batch"},
+ "mhr_params": {0: "batch"},
+ "shape_params": {0: "batch"},
+ "cam_trans": {0: "batch"},
+ "joints_2d": {0: "batch"},
+ "joints_3d": {0: "batch"},
+ },
+ opset_version=17,
+)
+```
+
+Verify with `onnxruntime`:
+
+```python
+import onnxruntime as ort
+s = ort.InferenceSession("instanthmr.onnx",
+ providers=["CPUExecutionProvider"])
+print([(i.name, i.shape) for i in s.get_inputs()])
+print([(o.name, o.shape) for o in s.get_outputs()])
+```
+
+## Tips
+
+- **Don't drift the preprocessing.** The demo's
+ `InstantHMR._preprocess` (`instanthmr/inference.py`) is the exact
+ inverse of the annotation crop: 1.2× square padded crop → 224 → ImageNet
+ norm. Any change to either side has to be mirrored on the other.
+- **Joints are body-centred.** Don't add `cam_trans` to `joints_3d` before
+ computing the loss; the teacher stores them as body-centred, and
+ InstantHMR's last layer is trained against that.
+- **CLIFF conditioning lives in full-frame coords**, not crop coords.
+ `cx_norm = 2·cx/W − 1` etc. Recomputing this from the crop will silently
+ degrade the camera head.
diff --git a/instanthmr/__init__.py b/instanthmr/__init__.py
new file mode 100644
index 0000000..1194ccc
--- /dev/null
+++ b/instanthmr/__init__.py
@@ -0,0 +1,20 @@
+"""InstantHMR — standalone 3D human pose inference + Rerun visualization."""
+
+from .inference import InstantHMR, HMRPrediction
+from .detector import RFDETRDetector
+from .pipeline import PosePipeline, FrameResult
+from .skeleton import JOINT_NAMES, SKELETON_EDGES, NUM_JOINTS, edges_for
+
+__all__ = [
+ "InstantHMR",
+ "HMRPrediction",
+ "RFDETRDetector",
+ "PosePipeline",
+ "FrameResult",
+ "JOINT_NAMES",
+ "SKELETON_EDGES",
+ "NUM_JOINTS",
+ "edges_for",
+]
+
+__version__ = "0.1.0"
diff --git a/instanthmr/detector.py b/instanthmr/detector.py
new file mode 100644
index 0000000..c828eb4
--- /dev/null
+++ b/instanthmr/detector.py
@@ -0,0 +1,99 @@
+"""RF-DETR person detector.
+
+Wraps Roboflow's RF-DETR (medium variant by default) with a minimal API:
+``detect(image_rgb)`` returns a list of
+``{"bbox": [x1, y1, x2, y2], "confidence": float}`` dicts sorted by
+descending confidence.
+
+RF-DETR is a transformer-based detector with strong COCO recall on small
+and partially-occluded persons — a good match for upstream pose
+estimation, which is sensitive to bbox localisation.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+COCO_PERSON_CLASS_ID = 1 # RF-DETR exposes COCO 1-indexed class ids
+
+
+class RFDETRDetector:
+ """Person detector backed by RF-DETR.
+
+ Args:
+ variant: Which RF-DETR variant to load. One of
+ ``"nano"``, ``"small"``, ``"medium"``, ``"base"``, ``"large"``.
+ Default ``"medium"`` balances speed and accuracy.
+ confidence: Minimum detection score.
+ max_persons: Maximum number of persons returned per frame.
+ optimize_for_inference: Call RF-DETR's TorchScript / fused-op
+ optimisation pass on construction. Adds a few seconds of
+ startup but speeds up subsequent ``detect`` calls noticeably.
+ """
+
+ _VARIANTS = {
+ "nano": "RFDETRNano",
+ "small": "RFDETRSmall",
+ "medium": "RFDETRMedium",
+ "base": "RFDETRBase",
+ "large": "RFDETRLarge",
+ }
+
+ def __init__(
+ self,
+ variant: str = "medium",
+ confidence: float = 0.5,
+ max_persons: int = 5,
+ optimize_for_inference: bool = True,
+ ):
+ import rfdetr
+
+ if variant not in self._VARIANTS:
+ raise ValueError(
+ f"Unknown RF-DETR variant {variant!r}. "
+ f"Choose one of {list(self._VARIANTS)}."
+ )
+
+ cls = getattr(rfdetr, self._VARIANTS[variant])
+ self._model = cls()
+ if optimize_for_inference:
+ self._model.optimize_for_inference()
+
+ self._confidence = confidence
+ self._max_persons = max_persons
+ self._variant = variant
+
+ @property
+ def variant(self) -> str:
+ return self._variant
+
+ def detect(self, image_rgb: np.ndarray) -> list[dict]:
+ """Detect persons in an RGB image.
+
+ Args:
+ image_rgb: ``(H, W, 3)`` uint8 RGB array.
+
+ Returns:
+ List of detection dicts sorted by descending confidence,
+ truncated to ``max_persons``.
+ """
+ from PIL import Image as PILImage
+
+ pil_image = PILImage.fromarray(image_rgb)
+ sv_detections = self._model.predict(pil_image, threshold=self._confidence)
+
+ detections: list[dict] = []
+ if len(sv_detections) > 0 and sv_detections.class_id is not None:
+ mask = sv_detections.class_id == COCO_PERSON_CLASS_ID
+ for bbox, conf in zip(
+ sv_detections.xyxy[mask],
+ sv_detections.confidence[mask],
+ ):
+ detections.append({
+ "bbox": np.asarray(bbox, dtype=np.float32),
+ "confidence": float(conf),
+ })
+
+ detections.sort(key=lambda d: d["confidence"], reverse=True)
+ return detections[: self._max_persons]
diff --git a/instanthmr/inference.py b/instanthmr/inference.py
new file mode 100644
index 0000000..13e21a5
--- /dev/null
+++ b/instanthmr/inference.py
@@ -0,0 +1,267 @@
+"""ONNX inference for InstantHMR.
+
+InstantHMR takes a 224x224 person crop and a 3-vector CLIFF condition
+(bbox center / scale in full-frame coords) and returns mhr_params, shape,
+camera translation, and 70 joints in 2D (crop space) and 3D (camera coords,
+body-centred, metres, Y-down).
+
+This module wraps the ONNX session and the per-person preprocessing — square
+crop, CLIFF cond, ImageNet normalisation, and re-projection of joints back
+into full-frame pixels.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import cv2
+import numpy as np
+
+
+IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+INPUT_SIZE = 224
+CROP_EXPAND = 1.2 # square crop around the detector bbox, matching training
+
+
+@dataclass
+class HMRPrediction:
+ """Per-person outputs from one InstantHMR forward pass.
+
+ All numpy, all in the FULL-frame coordinate system.
+
+ Attributes:
+ bbox: (4,) raw detector bbox [x1, y1, x2, y2].
+ confidence: detection confidence in [0, 1].
+ joints_3d_local: (70, 3) body-centred joints, metres, Y-down.
+ joints_3d_cam: (70, 3) joints in camera space (= local + cam_trans).
+ joints_2d: (70, 2) joints projected into full-frame pixel coords.
+ cam_trans: (3,) camera translation [tx, ty, tz], metres.
+ focal_length: (2,) [fx, fy] full-frame virtual focal = sqrt(H^2+W^2).
+ principal_point: (2,) [cx, cy] = full-frame centre.
+ image_shape: (H, W) of the source frame.
+ mhr_params: (204,) raw MHR pose+scale parameters. InstantHMR does
+ not run the MHR mesh decoder itself (it is resource-heavy and
+ kept out of the default pipeline); these params are exposed so
+ you can feed them into Meta's MHR TorchScript module from the
+ original SAM 3D Body release if you want a body mesh —
+ https://github.com/facebookresearch/sam-3d-body.
+ shape_params: (45,) MHR identity shape blendshapes. Same use as
+ ``mhr_params`` — pair them and run them through MHR.
+ """
+
+ bbox: np.ndarray
+ confidence: float
+ joints_3d_local: np.ndarray
+ joints_3d_cam: np.ndarray
+ joints_2d: np.ndarray
+ cam_trans: np.ndarray
+ focal_length: np.ndarray
+ principal_point: np.ndarray
+ image_shape: tuple[int, int]
+ mhr_params: np.ndarray
+ shape_params: np.ndarray
+
+
+class InstantHMR:
+ """ONNX wrapper for the InstantHMR model.
+
+ Example
+ -------
+ hmr = InstantHMR("models/instanthmr.onnx")
+ out = hmr.predict(image_rgb, bbox=[x1, y1, x2, y2])
+ print(out.joints_3d_cam.shape) # (70, 3)
+ """
+
+ def __init__(
+ self,
+ onnx_path: str | Path,
+ device: str = "cuda",
+ providers: Optional[list[str]] = None,
+ ):
+ import onnxruntime as ort
+
+ onnx_path = Path(onnx_path)
+ if not onnx_path.exists():
+ raise FileNotFoundError(f"ONNX model not found: {onnx_path}")
+
+ # ORT >= 1.19 ships preload_dlls() which discovers CUDA / cuDNN
+ # bundled by the nvidia-* pip wheels (or by torch) and adds them to
+ # the loader's search path. Calling it before session creation is
+ # the simplest fix for "libcudnn.so.9: cannot open shared object
+ # file" on machines that don't ship system CUDA.
+ if hasattr(ort, "preload_dlls"):
+ try:
+ ort.preload_dlls()
+ except Exception:
+ # preload_dlls is best-effort — never let it block init.
+ pass
+
+ if providers is None:
+ providers = self._default_providers(device)
+
+ self.session = ort.InferenceSession(str(onnx_path), providers=providers)
+ self.active_provider = self.session.get_providers()[0]
+
+ in_names = [i.name for i in self.session.get_inputs()]
+ out_names = [o.name for o in self.session.get_outputs()]
+ self._in_image = in_names[0] # "image"
+ self._in_cliff = in_names[1] # "cliff_cond"
+ # Output order from export: mhr_params, shape_params, cam_trans, joints_2d, joints_3d
+ self._out_names = out_names
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def predict(
+ self,
+ image_rgb: np.ndarray,
+ bbox: np.ndarray | list[float],
+ confidence: float = 1.0,
+ ) -> HMRPrediction:
+ """Run InstantHMR on a single person.
+
+ Args:
+ image_rgb: (H, W, 3) uint8 RGB full-frame image.
+ bbox: tight person bbox [x1, y1, x2, y2] in pixel coords.
+ confidence: detection confidence to attach to the output.
+
+ Returns:
+ ``HMRPrediction`` with joints in full-frame coordinates.
+ """
+ image_rgb = np.ascontiguousarray(image_rgb)
+ h, w = image_rgb.shape[:2]
+ bbox_arr = np.asarray(bbox, dtype=np.float32).reshape(4)
+
+ crop, sq_x1, sq_y1, sq_size, cliff_cond = self._preprocess(
+ image_rgb, bbox_arr, h, w
+ )
+
+ outs = self.session.run(
+ None,
+ {
+ self._in_image: crop[np.newaxis], # (1, 3, 224, 224)
+ self._in_cliff: cliff_cond[np.newaxis], # (1, 3)
+ },
+ )
+ # Order: mhr_params, shape_params, cam_trans, joints_2d, joints_3d
+ mhr_params = outs[0][0].astype(np.float32)
+ shape_params = outs[1][0].astype(np.float32)
+ cam_trans = outs[2][0].astype(np.float32)
+ joints_2d_norm = outs[3][0].astype(np.float32) # (70, 2) in [-1, 1]
+ joints_3d_local = outs[4][0].astype(np.float32) # (70, 3) body-centred
+
+ # Re-project the 2D head from normalised crop space → full-frame pixels.
+ crop_px = (joints_2d_norm + 1.0) * 0.5 * INPUT_SIZE
+ scale = sq_size / INPUT_SIZE
+ joints_2d = np.stack(
+ [crop_px[:, 0] * scale + sq_x1, crop_px[:, 1] * scale + sq_y1],
+ axis=-1,
+ ).astype(np.float32)
+
+ joints_3d_cam = joints_3d_local + cam_trans
+
+ # Full-frame virtual pinhole: focal = sqrt(H^2 + W^2), principal = centre.
+ f = math.sqrt(h * h + w * w)
+ focal_length = np.array([f, f], dtype=np.float32)
+ principal_point = np.array([w / 2.0, h / 2.0], dtype=np.float32)
+
+ return HMRPrediction(
+ bbox=bbox_arr,
+ confidence=float(confidence),
+ joints_3d_local=joints_3d_local,
+ joints_3d_cam=joints_3d_cam,
+ joints_2d=joints_2d,
+ cam_trans=cam_trans,
+ focal_length=focal_length,
+ principal_point=principal_point,
+ image_shape=(h, w),
+ mhr_params=mhr_params,
+ shape_params=shape_params,
+ )
+
+ # ------------------------------------------------------------------
+ # Preprocessing
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _preprocess(
+ image_rgb: np.ndarray,
+ bbox: np.ndarray,
+ h: int,
+ w: int,
+ ) -> tuple[np.ndarray, float, float, float, np.ndarray]:
+ """Square 1.2x crop, ImageNet normalise, and CLIFF conditioning."""
+ x1, y1, x2, y2 = bbox.astype(float)
+ bw = x2 - x1
+ bh = y2 - y1
+ cx = (x1 + x2) / 2.0
+ cy = (y1 + y2) / 2.0
+
+ # CLIFF conditioning vector (full-frame coords)
+ cx_norm = 2.0 * (cx / w) - 1.0
+ cy_norm = 2.0 * (cy / h) - 1.0
+ b_scale = max(bw, bh) / max(w, h)
+ cliff_cond = np.array([cx_norm, cy_norm, b_scale], dtype=np.float32)
+
+ sq_size = max(bw, bh) * CROP_EXPAND
+ half = sq_size / 2.0
+ sq_x1 = cx - half
+ sq_y1 = cy - half
+
+ ix1 = int(math.floor(sq_x1))
+ iy1 = int(math.floor(sq_y1))
+ ix2 = int(math.ceil(sq_x1 + sq_size))
+ iy2 = int(math.ceil(sq_y1 + sq_size))
+
+ pad_left = max(0, -ix1)
+ pad_top = max(0, -iy1)
+ pad_right = max(0, ix2 - w)
+ pad_bottom = max(0, iy2 - h)
+
+ src_x1 = max(0, ix1)
+ src_y1 = max(0, iy1)
+ src_x2 = min(w, ix2)
+ src_y2 = min(h, iy2)
+
+ patch = image_rgb[src_y1:src_y2, src_x1:src_x2]
+ if pad_left or pad_top or pad_right or pad_bottom:
+ patch = cv2.copyMakeBorder(
+ patch, pad_top, pad_bottom, pad_left, pad_right,
+ cv2.BORDER_CONSTANT, value=(0, 0, 0),
+ )
+
+ crop_224 = cv2.resize(patch, (INPUT_SIZE, INPUT_SIZE), interpolation=cv2.INTER_LINEAR)
+ crop = crop_224.astype(np.float32) / 255.0
+ crop = (crop - IMAGENET_MEAN) / IMAGENET_STD
+ crop = np.transpose(crop, (2, 0, 1)).astype(np.float32) # (3, 224, 224)
+
+ return crop, float(sq_x1), float(sq_y1), float(sq_size), cliff_cond
+
+ # ------------------------------------------------------------------
+ # Provider selection
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _default_providers(device: str) -> list[str]:
+ """Pick ORT execution providers based on the requested *device*.
+
+ TensorRT is intentionally **not** requested by default: the wheel
+ distinction between ``onnxruntime-gpu`` and the ``tensorrt`` system
+ libraries is fiddly, and asking for an EP whose runtime libraries
+ aren't installed produces a verbose error before falling back.
+ Pass ``providers=`` explicitly if you want TensorRT.
+ """
+ import onnxruntime as ort
+
+ available = set(ort.get_available_providers())
+ wanted: list[str] = []
+ if "cuda" in device.lower():
+ if "CUDAExecutionProvider" in available:
+ wanted.append("CUDAExecutionProvider")
+ wanted.append("CPUExecutionProvider")
+ return wanted
diff --git a/instanthmr/pipeline.py b/instanthmr/pipeline.py
new file mode 100644
index 0000000..8da5eb5
--- /dev/null
+++ b/instanthmr/pipeline.py
@@ -0,0 +1,126 @@
+"""End-to-end pose estimation pipeline.
+
+Combines the RF-DETR person detector with InstantHMR to produce per-person
+3D + 2D joints from a single RGB frame, while reporting **separate**
+detector and HMR timings so the demo can surface them.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+
+from .detector import RFDETRDetector
+from .inference import InstantHMR, HMRPrediction
+
+
+@dataclass
+class FrameResult:
+ """Output of one ``PosePipeline.predict`` call.
+
+ Attributes:
+ persons: Per-person ``HMRPrediction``s (possibly empty).
+ detector_ms: Wall-clock time for the detector forward pass.
+ hmr_ms: Wall-clock time for InstantHMR across all persons in the
+ frame (sum, not average).
+ total_ms: ``detector_ms + hmr_ms`` plus any pipeline overhead.
+ """
+
+ persons: list[HMRPrediction]
+ detector_ms: float
+ hmr_ms: float
+ total_ms: float
+
+
+class PosePipeline:
+ """Full RF-DETR → InstantHMR pipeline.
+
+ Args:
+ onnx_path: Path to ``instanthmr.onnx``.
+ device: ``"cuda"`` or ``"cpu"`` for InstantHMR (RF-DETR auto-selects).
+ detector_variant: ``"nano" | "small" | "medium" | "base" | "large"``.
+ det_confidence: RF-DETR confidence threshold.
+ max_persons: Maximum number of persons returned per frame.
+
+ Example
+ -------
+ pipeline = PosePipeline("models/instanthmr.onnx")
+ result = pipeline.predict(image_rgb)
+ print(f"det {result.detector_ms:.1f} ms / hmr {result.hmr_ms:.1f} ms")
+ for p in result.persons:
+ print(p.joints_3d_cam.shape) # (70, 3)
+ """
+
+ def __init__(
+ self,
+ onnx_path: str | Path,
+ device: str = "cuda",
+ detector_variant: str = "medium",
+ det_confidence: float = 0.5,
+ max_persons: int = 5,
+ ):
+ self.hmr = InstantHMR(onnx_path, device=device)
+ self.detector = RFDETRDetector(
+ variant=detector_variant,
+ confidence=det_confidence,
+ max_persons=max_persons,
+ )
+ self._last_bbox: Optional[np.ndarray] = None
+
+ def predict(self, image_rgb: np.ndarray) -> FrameResult:
+ """Detect all persons in *image_rgb* and run InstantHMR on each.
+
+ Returns a :class:`FrameResult` with per-stage timings so callers
+ can display detector vs HMR cost separately.
+ """
+ t_total_start = time.perf_counter()
+
+ # ---- Detector ----
+ t0 = time.perf_counter()
+ detections = self.detector.detect(image_rgb)
+ detector_ms = (time.perf_counter() - t0) * 1000.0
+
+ # Single-frame fallback: reuse the last bbox if the detector drops
+ # a frame. Useful for live camera mode where a transient miss is
+ # common.
+ if not detections and self._last_bbox is not None:
+ h, w = image_rgb.shape[:2]
+ lb = self._last_bbox
+ bw, bh = lb[2] - lb[0], lb[3] - lb[1]
+ expand = 0.1
+ fb = np.array([
+ max(0, lb[0] - bw * expand),
+ max(0, lb[1] - bh * expand),
+ min(w, lb[2] + bw * expand),
+ min(h, lb[3] + bh * expand),
+ ], dtype=np.float32)
+ detections = [{"bbox": fb, "confidence": 0.0}]
+
+ # ---- InstantHMR (sum across all persons) ----
+ outputs: list[HMRPrediction] = []
+ t1 = time.perf_counter()
+ for det in detections:
+ out = self.hmr.predict(
+ image_rgb,
+ bbox=det["bbox"],
+ confidence=det["confidence"],
+ )
+ outputs.append(out)
+ hmr_ms = (time.perf_counter() - t1) * 1000.0
+
+ if outputs:
+ best = max(outputs, key=lambda o: o.confidence)
+ self._last_bbox = best.bbox.copy()
+
+ total_ms = (time.perf_counter() - t_total_start) * 1000.0
+
+ return FrameResult(
+ persons=outputs,
+ detector_ms=detector_ms,
+ hmr_ms=hmr_ms,
+ total_ms=total_ms,
+ )
diff --git a/instanthmr/skeleton.py b/instanthmr/skeleton.py
new file mode 100644
index 0000000..2345253
--- /dev/null
+++ b/instanthmr/skeleton.py
@@ -0,0 +1,73 @@
+"""MHR70 joint ordering and skeleton connectivity.
+
+The student model regresses 70 keypoints in the MHR70 ordering, matching
+the SAM3D teacher output (`pred_keypoints_3d`). Body landmarks first, then
+hands, then olecranon / cubital / acromion / neck.
+"""
+
+from __future__ import annotations
+
+
+JOINT_NAMES: list[str] = [
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear", # 0-4
+ "left_shoulder", "right_shoulder", # 5-6
+ "left_elbow", "right_elbow", # 7-8
+ "left_hip", "right_hip", # 9-10
+ "left_knee", "right_knee", # 11-12
+ "left_ankle", "right_ankle", # 13-14
+ "left_big_toe_tip", "left_small_toe_tip", "left_heel", # 15-17
+ "right_big_toe_tip", "right_small_toe_tip", "right_heel", # 18-20
+ "right_thumb_tip", "right_thumb_first_joint",
+ "right_thumb_second_joint", "right_thumb_third_joint", # 21-24
+ "right_index_tip", "right_index_first_joint",
+ "right_index_second_joint", "right_index_third_joint", # 25-28
+ "right_middle_tip", "right_middle_first_joint",
+ "right_middle_second_joint", "right_middle_third_joint", # 29-32
+ "right_ring_tip", "right_ring_first_joint",
+ "right_ring_second_joint", "right_ring_third_joint", # 33-36
+ "right_pinky_tip", "right_pinky_first_joint",
+ "right_pinky_second_joint", "right_pinky_third_joint", # 37-40
+ "right_wrist", # 41
+ "left_thumb_tip", "left_thumb_first_joint",
+ "left_thumb_second_joint", "left_thumb_third_joint", # 42-45
+ "left_index_tip", "left_index_first_joint",
+ "left_index_second_joint", "left_index_third_joint", # 46-49
+ "left_middle_tip", "left_middle_first_joint",
+ "left_middle_second_joint", "left_middle_third_joint", # 50-53
+ "left_ring_tip", "left_ring_first_joint",
+ "left_ring_second_joint", "left_ring_third_joint", # 54-57
+ "left_pinky_tip", "left_pinky_first_joint",
+ "left_pinky_second_joint", "left_pinky_third_joint", # 58-61
+ "left_wrist", # 62
+ "left_olecranon", "right_olecranon", # 63-64
+ "left_cubital_fossa", "right_cubital_fossa", # 65-66
+ "left_acromion", "right_acromion", # 67-68
+ "neck", # 69
+]
+
+
+SKELETON_EDGES: list[tuple[int, int]] = [
+ # Head <-> neck
+ (0, 69),
+ # Torso
+ (69, 5), (69, 6),
+ (5, 9), (6, 10), (9, 10),
+ # Arms
+ (5, 7), (7, 62),
+ (6, 8), (8, 41),
+ # Legs
+ (9, 11), (11, 13), (13, 17),
+ (10, 12), (12, 14), (14, 20),
+ # Left hand fan
+ (62, 42), (62, 46), (62, 50), (62, 54), (62, 58),
+ # Right hand fan
+ (41, 21), (41, 25), (41, 29), (41, 33), (41, 37),
+]
+
+
+NUM_JOINTS = 70
+
+
+def edges_for(num_joints: int) -> list[tuple[int, int]]:
+ """Return the subset of edges that index into the available joints."""
+ return [(a, b) for a, b in SKELETON_EDGES if a < num_joints and b < num_joints]
diff --git a/instanthmr/visualizer.py b/instanthmr/visualizer.py
new file mode 100644
index 0000000..92f7ffb
--- /dev/null
+++ b/instanthmr/visualizer.py
@@ -0,0 +1,276 @@
+"""Rerun-based 3D visualization for the InstantHMR demo.
+
+Logs, per frame:
+ - ``camera/image`` : annotated RGB frame (2D skeleton + bbox + a
+ text panel showing detector / HMR / total ms).
+ - ``camera`` : pinhole intrinsics so the image becomes a
+ proper frustum in the 3D scene.
+ - ``world/persons/...`` : 3D joints + skeleton lines per person.
+ - ``timing/detector_ms`` : scalar plot of RF-DETR latency.
+ - ``timing/hmr_ms`` : scalar plot of InstantHMR latency.
+ - ``timing/total_ms`` : scalar plot of full pipeline latency.
+ - ``timing/fps`` : scalar plot of effective frames-per-second.
+
+``rr.init(..., spawn=True)`` is called automatically so the viewer opens
+on construction (pass ``spawn_viewer=False`` to suppress).
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Sequence
+
+import cv2
+import numpy as np
+
+from .inference import HMRPrediction
+from .skeleton import edges_for
+
+
+class RerunVisualizer:
+ def __init__(
+ self,
+ application_id: str = "instanthmr_demo",
+ spawn_viewer: bool = True,
+ save_path: str | None = None,
+ ):
+ import rerun as rr
+
+ self._rr = rr
+ rr.init(application_id, spawn=spawn_viewer)
+
+ # SAM3D / MHR convention: right-handed, Y-down camera frame.
+ rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True)
+
+ if save_path:
+ rr.save(save_path)
+
+ self._prev_num_persons = 0
+ self._blueprint_sent = False
+
+ # ------------------------------------------------------------------
+ # Per-frame logging
+ # ------------------------------------------------------------------
+
+ def log_frame(
+ self,
+ image_rgb: np.ndarray,
+ persons: Sequence[HMRPrediction],
+ frame_idx: int,
+ timestamp: float | None = None,
+ detector_ms: Optional[float] = None,
+ hmr_ms: Optional[float] = None,
+ total_ms: Optional[float] = None,
+ ) -> None:
+ """Log one frame of the pipeline to Rerun.
+
+ Args:
+ image_rgb: full-frame RGB image, ``(H, W, 3)`` uint8.
+ persons: per-person HMR predictions for this frame.
+ frame_idx: monotonic frame counter (sets the Rerun timeline).
+ timestamp: optional wall-clock timestamp (seconds).
+ detector_ms: RF-DETR latency for this frame (ms).
+ hmr_ms: InstantHMR latency for this frame, summed across
+ persons (ms).
+ total_ms: full pipeline latency for this frame (ms).
+ """
+ rr = self._rr
+ rr.set_time("frame", sequence=frame_idx)
+ if timestamp is not None:
+ rr.set_time("timestamp", duration=timestamp)
+
+ # Per-stage scalar plots
+ if detector_ms is not None:
+ rr.log("timing/detector_ms", rr.Scalars(float(detector_ms)))
+ if hmr_ms is not None:
+ rr.log("timing/hmr_ms", rr.Scalars(float(hmr_ms)))
+ if total_ms is not None:
+ rr.log("timing/total_ms", rr.Scalars(float(total_ms)))
+ if total_ms > 0:
+ rr.log("timing/fps", rr.Scalars(1000.0 / float(total_ms)))
+
+ # Clear stale person entities from previous frames
+ n = len(persons)
+ for stale in range(n, self._prev_num_persons):
+ rr.log(f"world/persons/person_{stale}", rr.Clear(recursive=True))
+ self._prev_num_persons = n
+
+ h, w = image_rgb.shape[:2]
+
+ # Always log the image — even when no person was detected — so the
+ # viewer keeps showing the camera feed and the timing HUD.
+ if not persons:
+ blank = self._draw_overlay(image_rgb, [], detector_ms, hmr_ms, total_ms)
+ rr.log("camera/image", rr.Image(blank))
+ return
+
+ for idx, person in enumerate(persons):
+ self._log_person(idx, person)
+
+ annotated = self._draw_overlay(
+ image_rgb, persons, detector_ms, hmr_ms, total_ms,
+ )
+ rr.log("camera/image", rr.Image(annotated))
+
+ # Pinhole camera in the 3D scene — full-frame focal so the image
+ # plane sits in front of the joints, not on top of them.
+ first = persons[0]
+ fx, fy = float(first.focal_length[0]), float(first.focal_length[1])
+ cx, cy = w / 2.0, h / 2.0
+ depth = float(first.joints_3d_cam[:, 2].mean())
+ plane_dist = max(depth * 0.25, 0.2)
+
+ rr.log(
+ "camera",
+ rr.Pinhole(
+ width=w,
+ height=h,
+ focal_length=[fx, fy],
+ principal_point=[cx, cy],
+ image_plane_distance=plane_dist,
+ ),
+ )
+
+ if not self._blueprint_sent:
+ self._send_blueprint(first)
+ self._blueprint_sent = True
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+
+ def _log_person(self, idx: int, person: HMRPrediction) -> None:
+ rr = self._rr
+ path = f"world/persons/person_{idx}"
+
+ joints = person.joints_3d_cam
+ rr.log(
+ f"{path}/joints",
+ rr.Points3D(positions=joints, radii=0.012, colors=[0, 230, 0]),
+ )
+
+ edges = edges_for(joints.shape[0])
+ if edges:
+ lines = [
+ [joints[i].tolist(), joints[j].tolist()]
+ for i, j in edges
+ ]
+ rr.log(
+ f"{path}/skeleton",
+ rr.LineStrips3D(lines, colors=[255, 230, 0], radii=0.004),
+ )
+
+ @staticmethod
+ def _draw_overlay(
+ image_rgb: np.ndarray,
+ persons: Sequence[HMRPrediction],
+ detector_ms: Optional[float],
+ hmr_ms: Optional[float],
+ total_ms: Optional[float],
+ ) -> np.ndarray:
+ """Draw per-person bbox + 2D skeleton + a timing HUD onto the frame."""
+ out = image_rgb.copy()
+ h, w = out.shape[:2]
+
+ for person in persons:
+ x1, y1, x2, y2 = person.bbox.astype(int)
+ cv2.rectangle(out, (x1, y1), (x2, y2), (0, 200, 255), 2)
+ label = f"{person.confidence:.2f}"
+ (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
+ cv2.rectangle(out, (x1, y1 - th - 6), (x1 + tw + 4, y1), (0, 200, 255), -1)
+ cv2.putText(
+ out, label, (x1 + 2, y1 - 4),
+ cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1,
+ )
+
+ j2d = person.joints_2d
+ valid = (
+ (j2d[:, 0] >= 0) & (j2d[:, 0] < w)
+ & (j2d[:, 1] >= 0) & (j2d[:, 1] < h)
+ )
+
+ for i, j in edges_for(j2d.shape[0]):
+ if valid[i] and valid[j]:
+ pt1 = tuple(j2d[i].astype(int))
+ pt2 = tuple(j2d[j].astype(int))
+ cv2.line(out, pt1, pt2, (255, 230, 0), 2)
+
+ for k, (x, y) in enumerate(j2d):
+ if valid[k]:
+ cv2.circle(out, (int(x), int(y)), 3, (0, 230, 0), -1)
+
+ # ---- Timing HUD (top-left, semi-transparent black panel) ----
+ lines: list[str] = []
+ if detector_ms is not None:
+ lines.append(f"RF-DETR : {detector_ms:6.1f} ms")
+ if hmr_ms is not None:
+ lines.append(f"InstantHMR: {hmr_ms:6.1f} ms")
+ if total_ms is not None:
+ fps = 1000.0 / total_ms if total_ms > 0 else 0.0
+ lines.append(f"Total : {total_ms:6.1f} ms ({fps:5.1f} fps)")
+ if lines:
+ font = cv2.FONT_HERSHEY_SIMPLEX
+ scale = max(0.5, min(w, h) / 1500.0)
+ thick = max(1, int(round(scale * 1.5)))
+ sizes = [
+ cv2.getTextSize(s, font, scale, thick)[0] for s in lines
+ ]
+ line_h = max(s[1] for s in sizes) + 8
+ box_w = max(s[0] for s in sizes) + 16
+ box_h = line_h * len(lines) + 8
+
+ overlay = out.copy()
+ cv2.rectangle(overlay, (8, 8), (8 + box_w, 8 + box_h), (0, 0, 0), -1)
+ cv2.addWeighted(overlay, 0.55, out, 0.45, 0, out)
+
+ y = 8 + line_h
+ for s in lines:
+ cv2.putText(
+ out, s, (16, y),
+ font, scale, (255, 255, 255), thick, cv2.LINE_AA,
+ )
+ y += line_h
+
+ return out
+
+ def _send_blueprint(self, first: HMRPrediction) -> None:
+ """Send a 3-panel blueprint: camera image | 3D scene | timings."""
+ import rerun.blueprint as rrb
+ rr = self._rr
+
+ joints = first.joints_3d_cam
+ center = joints.mean(axis=0)
+ extent = float(np.abs(joints - center).max())
+ cam_distance = max(extent * 4.0, 1.5)
+ # Y-down convention: place the orbit camera "in front" along -Z.
+ eye_pos = [
+ float(center[0]),
+ float(center[1]),
+ float(center[2] - cam_distance),
+ ]
+
+ blueprint = rrb.Blueprint(
+ rrb.Horizontal(
+ rrb.Vertical(
+ rrb.Spatial2DView(origin="camera/image", name="Camera"),
+ rrb.TimeSeriesView(
+ origin="timing",
+ name="Latency (ms)",
+ ),
+ row_shares=[3, 1],
+ ),
+ rrb.Spatial3DView(
+ origin="/",
+ name="3D Scene",
+ eye_controls=rrb.EyeControls3D(
+ position=eye_pos,
+ look_target=[float(center[0]), float(center[1]), float(center[2])],
+ eye_up=[0.0, -1.0, 0.0],
+ kind=rrb.Eye3DKind.Orbital,
+ tracking_entity="world/persons/person_0",
+ ),
+ background=rrb.Background(color=[25, 25, 25]),
+ ),
+ column_shares=[2, 3],
+ ),
+ )
+ rr.send_blueprint(blueprint)
diff --git a/models/README.md b/models/README.md
new file mode 100644
index 0000000..1434ee4
--- /dev/null
+++ b/models/README.md
@@ -0,0 +1,119 @@
+# InstantHMR — model weights
+
+A lightweight, ONNX-exportable distillation of
+[`facebook/sam-3d-body-dinov3`](https://huggingface.co/facebook/sam-3d-body-dinov3)
+for **3D human pose estimation and mesh recovery**: a RepViT-M1.5
+backbone + a 9-token cross-attention decoder + CLIFF camera conditioning.
+Trained to mimic the SAM3D teacher's per-person 70-keypoint outputs from
+a single 224×224 crop.
+
+Demo, training, and inference code live at the main InstantHMR
+repository.
+
+## Files
+
+| File | Size | Purpose |
+|---|---|---|
+| `instanthmr.onnx` | ~77 MB | fp16 ONNX export — what the demo and `InstantHMR(...)` wrapper load. |
+| `instanthmr.pth` | ~461 MB | PyTorch checkpoint for fine-tuning / further distillation. |
+
+Most users only need `instanthmr.onnx`. The `.pth` is included for people
+who want to continue training, swap the head, or re-export with different
+options — see [`docs/training.md`](https://github.com/mohamdev/InstantHMR/blob/main/docs/training.md)
+in the main repo.
+
+## ONNX I/O
+
+- **Inputs**
+ - `image` — `(N, 3, 224, 224)` float32, ImageNet-normalised RGB.
+ - `cliff_cond` — `(N, 3)` float32, `[cx_norm, cy_norm, b_scale]` of the
+ detector bbox in the **full** frame (`cx_norm, cy_norm ∈ [-1, 1]`,
+ `b_scale = max(bw, bh) / max(W, H)`).
+- **Outputs**
+ - `mhr_params` — `(N, 204)` MHR pose + scale parameters.
+ - `shape_params` — `(N, 45)` MHR identity blendshapes.
+ - `cam_trans` — `(N, 3)` body translation in camera frame (metres).
+ - `joints_2d` — `(N, 70, 2)` keypoints in normalised crop coords `[-1, 1]`.
+ - `joints_3d` — `(N, 70, 3)` body-centred 3D keypoints (metres, Y-down).
+
+The 70 keypoints follow the MHR70 ordering — see
+[`instanthmr/skeleton.py`](https://github.com/mohamdev/InstantHMR/blob/main/instanthmr/skeleton.py)
+for joint names and skeleton edges.
+
+## Quick start (ONNX)
+
+Minimal end-to-end inference with `onnxruntime`, no extra dependencies:
+
+```python
+import cv2
+import numpy as np
+import onnxruntime as ort
+
+IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+
+# 1. Load the ONNX session (CUDA if available, otherwise CPU).
+sess = ort.InferenceSession(
+ "instanthmr.onnx",
+ providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
+)
+
+# 2. Load an RGB image and a person bbox [x1, y1, x2, y2].
+image = cv2.cvtColor(cv2.imread("person.jpg"), cv2.COLOR_BGR2RGB)
+H, W = image.shape[:2]
+x1, y1, x2, y2 = 120, 40, 380, 600 # replace with your detector output
+
+# 3. Square 1.2x crop around the bbox, resized to 224x224.
+cx, cy = (x1 + x2) / 2, (y1 + y2) / 2
+size = max(x2 - x1, y2 - y1) * 1.2
+sx1, sy1 = int(cx - size / 2), int(cy - size / 2)
+sx2, sy2 = int(cx + size / 2), int(cy + size / 2)
+pad = [max(0, -sy1), max(0, sy2 - H), max(0, -sx1), max(0, sx2 - W)]
+patch = image[max(0, sy1):min(H, sy2), max(0, sx1):min(W, sx2)]
+patch = cv2.copyMakeBorder(patch, *pad, cv2.BORDER_CONSTANT, value=0)
+crop = cv2.resize(patch, (224, 224), interpolation=cv2.INTER_LINEAR)
+
+# 4. Normalise + CHW.
+img = (crop.astype(np.float32) / 255.0 - IMAGENET_MEAN) / IMAGENET_STD
+img = np.transpose(img, (2, 0, 1))[None] # (1, 3, 224, 224)
+
+# 5. CLIFF conditioning vector (full-frame coords).
+cliff = np.array([
+ 2.0 * cx / W - 1.0,
+ 2.0 * cy / H - 1.0,
+ max(x2 - x1, y2 - y1) / max(W, H),
+], dtype=np.float32)[None] # (1, 3)
+
+# 6. Run the model.
+mhr_params, shape_params, cam_trans, joints_2d, joints_3d = sess.run(
+ None, {"image": img, "cliff_cond": cliff}
+)
+
+print(joints_3d.shape) # (1, 70, 3) — body-centred metres, Y-down
+print(joints_2d.shape) # (1, 70, 2) — normalised crop coords [-1, 1]
+print(mhr_params.shape) # (1, 204) — feed into Meta's MHR module for a mesh
+```
+
+For a full pipeline (RF-DETR person detector + InstantHMR + Rerun
+visualiser), use the [`PosePipeline`](https://github.com/mohamdev/InstantHMR/blob/main/instanthmr/pipeline.py)
+helper from the main repository — `pip install -r requirements.txt` then
+`python demo.py --image path/to/photo.jpg`.
+
+## Body mesh
+
+InstantHMR does **not** run the MHR mesh decoder — rendering the full
+mesh is resource-heavy and the default pipeline stays lightweight. The
+raw `mhr_params` and `shape_params` are exposed so you can feed them
+into Meta's MHR TorchScript module from the original
+[SAM 3D Body release](https://github.com/facebookresearch/sam-3d-body)
+when you need vertices.
+
+## License
+
+These weights are released under the **SAM license**, since InstantHMR
+is a distillation of `facebook/sam-3d-body-dinov3`. Refer to
+<https://github.com/facebookresearch/sam-3d-body> for the full license
+text and applicable use restrictions.
+
+The accompanying source code in the InstantHMR repository is released
+separately under the Apache License 2.0.
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynb
new file mode 100644
index 0000000..2d43919
--- /dev/null
+++ b/notebooks/distill_transformer_decoder.ipynb
@@ -0,0 +1,2365 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ad5fd1be",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "import torch.optim as optim\n",
+ "import torchvision.transforms as T\n",
+ "import matplotlib.pyplot as plt\n",
+ "import timm\n",
+ "import cv2\n",
+ "from pathlib import Path\n",
+ "from tqdm.notebook import tqdm\n",
+ "import onnxruntime as ort\n",
+ "print(ort.get_available_providers())\n",
+ "\n",
+ "\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "print(f\"Using device: {device}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7e2ae437",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 1 — Configuration\n",
+ "# ============================================================\n",
+ "from dataclasses import dataclass \n",
+ "from pathlib import Path\n",
+ "\n",
+ "PROJECT_ROOT = Path(\"\") #set your path\n",
+ "\n",
+ "@dataclass\n",
+ "class DistillConfig:\n",
+ " \"\"\"All hyperparameters in one place.\"\"\"\n",
+ " # --- Paths ---\n",
+ " images_dir: str = str(PROJECT_ROOT / \"dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",
+ " annotations_dir: str = str(PROJECT_ROOT / \"dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",
+ " mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt\") #path to MHR model\n",
+ " log_dir: str = str(PROJECT_ROOT / \"runs/distill_repvit_cliff_v1\") # Bumped to v1!\n",
+ "\n",
+ " # --- Data ---\n",
+ " image_size: int = 224\n",
+ " max_images: int = 100_000 \n",
+ " val_split: float = 0.1\n",
+ " num_workers: int = 4\n",
+ " augment: bool = True\n",
+ "\n",
+ " # --- Model (Transformer + CLIFF) ---\n",
+ " backbone: str = \"repvit_m2_3\" \n",
+ " backbone_feat_dim: int = 640 # RepViT m2_3 usually outputs 640\n",
+ " d_model: int = 512 \n",
+ " n_heads: int = 8 \n",
+ " n_decoder_layers: int = 4 \n",
+ " dropout: float = 0.1\n",
+ "\n",
+ " # --- Output dimensions ---\n",
+ " pose_dim: int = 136 \n",
+ " scale_dim: int = 68 \n",
+ " shape_dim: int = 45 \n",
+ " cam_dim: int = 3 \n",
+ " cliff_dim: int = 3 \n",
+ " num_joints: int = 70 # We track 70 joints\n",
+ "\n",
+ " @property\n",
+ " def model_params_dim(self) -> int:\n",
+ " return self.pose_dim + self.scale_dim # 204\n",
+ "\n",
+ " # --- Training ---\n",
+ " batch_size: int = 64\n",
+ " lr: float = 3e-4\n",
+ " weight_decay: float = 1e-4\n",
+ " epochs: int = 300\n",
+ " warmup_epochs: int = 3\n",
+ " grad_clip: float = 5.0\n",
+ " use_amp: bool = True \n",
+ "\n",
+ " # --- Loss weights (UPDATED for HMR Balance) ---\n",
+ " w_pose: float = 1.0\n",
+ " w_scale: float = 0.1\n",
+ " w_shape: float = 1.0\n",
+ " w_cam: float = 0.1 \n",
+ " w_keypoints3d: float = 10.0 # Bumped up: 3D joints are the strongest spatial anchor\n",
+ " w_keypoints2d: float = 10.0 # Bumped up: Forces pixel-perfect alignment\n",
+ " w_3d_joints: float = 1e-3\n",
+ " w_structural: float = 0.01\n",
+ " w_reproj_3d: float = 0.01 # High weight to enforce strict scale/camera alignment\n",
+ " w_reproj_mhr: float = 0.01 # High weight to enforce strict scale/camera alignment\n",
+ "\n",
+ " # ==========================================================\n",
+ " # Feet Barycenter Anchors\n",
+ " # ==========================================================\n",
+ " w_feet = 0.01 # Soft prior weight, matching w_structural\n",
+ " \n",
+ " # Native 70-joint format indices\n",
+ " native_left_foot = [15, 16, 17]\n",
+ " native_right_foot = [18, 19, 20]\n",
+ " \n",
+ " # MHR 127-joint format indices\n",
+ " mhr_left_foot = [6, 7, 8] \n",
+ " mhr_right_foot = [22, 23, 24]\n",
+ " \n",
+ " # Data-Driven Topology Mapping (Native 70 <-> MHR 127)\n",
+ " native_mapping_ids = [ 1, 2, 5, 6, 9, 10, 11, 12, 13, 14, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62 ]\n",
+ " mhr_mapping_ids = [ 125, 123, 75, 39, 2, 18, 3, 19, 5, 21, 64, 63, 62, 61, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 41, 100, 99, 98, 97, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 77 ]\n",
+ "\n",
+ "cfg = DistillConfig()\n",
+ "print(f\"Config loaded. Backbone: {cfg.backbone}. Ready for V3 Training.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "561a65f3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ==============================================================================\n",
+ "# Cell 2 — X-Ray Dataset Visualizer (Math Verification)\n",
+ "# ==============================================================================\n",
+ "import numpy as np\n",
+ "import cv2\n",
+ "import torch\n",
+ "import matplotlib.pyplot as plt\n",
+ "from pathlib import Path\n",
+ "import random\n",
+ "\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "\n",
+ "\n",
+ "# --- 1. Load Lightweight MHR Wrapper ---\n",
+ "class MHRForwardPass:\n",
+ " def __init__(self, mhr_path, device):\n",
+ " self.device = device\n",
+ " self.mhr = torch.jit.load(mhr_path, map_location=device).eval()\n",
+ " for p in self.mhr.parameters():\n",
+ " p.requires_grad_(False)\n",
+ "\n",
+ " @torch.no_grad()\n",
+ " def get_joints(self, model_params, shape_params):\n",
+ " B = model_params.shape[0]\n",
+ " dummy_identity = torch.zeros(B, 45, device=self.device, dtype=model_params.dtype)\n",
+ " concat_params = torch.cat([model_params, dummy_identity], dim=1)\n",
+ " joint_params = self.mhr.character_torch.model_parameters_to_joint_parameters(concat_params)\n",
+ " skel_state = self.mhr.character_torch.joint_parameters_to_skeleton_state(joint_params)\n",
+ " return skel_state\n",
+ "\n",
+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",
+ "\n",
+ "# --- 2. Load a Random Sample ---\n",
+ "npz_files = list(Path(cfg.annotations_dir).glob(\"*.npz\"))\n",
+ "sample_npz = random.choice(npz_files)\n",
+ "sample_stem = sample_npz.stem\n",
+ "\n",
+ "img_path = Path(cfg.images_dir) / f\"{sample_stem}.jpg\"\n",
+ "if not img_path.exists():\n",
+ " img_path = Path(cfg.images_dir) / f\"{sample_stem}.png\"\n",
+ "\n",
+ "ann = np.load(str(sample_npz))\n",
+ "img_bgr = cv2.imread(str(img_path))\n",
+ "img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n",
+ "\n",
+ "# --- 3. Extract Ground Truths ---\n",
+ "gt_2d_full = ann[\"joints_2d\"] \n",
+ "gt_3d_raw = ann[\"joints_3d\"] \n",
+ "model_params = ann[\"mhr_model_params\"] \n",
+ "shape_params = ann[\"shape_params\"] \n",
+ "cam_trans = ann[\"cam_trans\"] \n",
+ "focal_length = ann[\"cam_focal_length\"] \n",
+ "orig_shape = ann[\"orig_shape\"] \n",
+ "bbox_square = ann[\"bbox_square\"] \n",
+ "\n",
+ "# --- 4. Forward Pass through MHR with 6D Math Injection ---\n",
+ "mp_tensor = torch.from_numpy(model_params).unsqueeze(0).to(device)\n",
+ "sp_tensor = torch.from_numpy(shape_params).unsqueeze(0).to(device)\n",
+ "raw_mhr_joints = mhr_module.get_joints(mp_tensor, sp_tensor)[0, :, :3].cpu().numpy()\n",
+ "\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# THE HOLY GRAIL MATH: Coordinate Transformations\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# A. MHR -> Vision Coordinates\n",
+ "raw_mhr_joints = raw_mhr_joints / 100.0 \n",
+ "raw_mhr_joints[:, 1] = -raw_mhr_joints[:, 1] \n",
+ "raw_mhr_joints[:, 2] = -raw_mhr_joints[:, 2] \n",
+ "\n",
+ "# B. SAM3D 70-Joints -> Camera Space\n",
+ "j3d_70_cam = gt_3d_raw + cam_trans \n",
+ "\n",
+ "# --- 5. Project 3D to Full-Frame 2D ---\n",
+ "orig_h, orig_w = orig_shape\n",
+ "principal_point = np.array([orig_w / 2.0, orig_h / 2.0])\n",
+ "\n",
+ "mhr_cam = raw_mhr_joints + cam_trans\n",
+ "z_mhr = np.maximum(mhr_cam[:, 2], 0.01)\n",
+ "u_mhr_full = (focal_length[0] * mhr_cam[:, 0] / z_mhr) + principal_point[0]\n",
+ "v_mhr_full = (focal_length[1] * mhr_cam[:, 1] / z_mhr) + principal_point[1]\n",
+ "\n",
+ "z_70 = np.maximum(j3d_70_cam[:, 2], 0.01)\n",
+ "u_70_full = (focal_length[0] * j3d_70_cam[:, 0] / z_70) + principal_point[0]\n",
+ "v_70_full = (focal_length[1] * j3d_70_cam[:, 1] / z_70) + principal_point[1]\n",
+ "\n",
+ "# --- 6. Map Full-Frame 2D to 224x224 Crop Space ---\n",
+ "x1, y1, x2, y2 = bbox_square\n",
+ "crop_w, crop_h = x2 - x1, y2 - y1\n",
+ "\n",
+ "u_mhr_crop = (u_mhr_full - x1) / max(crop_w, 1) * cfg.image_size\n",
+ "v_mhr_crop = (v_mhr_full - y1) / max(crop_h, 1) * cfg.image_size\n",
+ "\n",
+ "u_70_crop = (u_70_full - x1) / max(crop_w, 1) * cfg.image_size\n",
+ "v_70_crop = (v_70_full - y1) / max(crop_h, 1) * cfg.image_size\n",
+ "\n",
+ "gt_u_crop = (gt_2d_full[:, 0] - x1) / max(crop_w, 1) * cfg.image_size\n",
+ "gt_v_crop = (gt_2d_full[:, 1] - y1) / max(crop_h, 1) * cfg.image_size\n",
+ "\n",
+ "# --- 7. Visualization ---\n",
+ "fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n",
+ "\n",
+ "axes[0].imshow(img_rgb)\n",
+ "axes[0].scatter(gt_u_crop, gt_v_crop, s=15, c='lime', edgecolors='black', label='Native 2D')\n",
+ "axes[0].scatter(u_70_crop, v_70_crop, s=40, facecolors='none', edgecolors='cyan', linewidths=1.5, label='Projected 3D (70pts)')\n",
+ "axes[0].set_title(f\"Target 1: Native 2D vs Projected 3D Head\\n{sample_stem}\", fontsize=10)\n",
+ "axes[0].legend(loc='upper right')\n",
+ "axes[0].axis('off')\n",
+ "\n",
+ "axes[1].imshow(img_rgb)\n",
+ "axes[1].scatter(u_mhr_crop, v_mhr_crop, s=15, c='red', edgecolors='black', marker='X')\n",
+ "axes[1].set_title(f\"Target 2: Projected MHR Params\\nMath verification (6D Cycle)\", fontsize=10)\n",
+ "axes[1].axis('off')\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1d5e32ef",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 3 — Dataset and Data Augmentation\n",
+ "# ============================================================\n",
+ "import os\n",
+ "import glob\n",
+ "import random\n",
+ "import numpy as np\n",
+ "from pathlib import Path\n",
+ "from PIL import Image\n",
+ "\n",
+ "import torch\n",
+ "from torch.utils.data import Dataset, DataLoader, random_split\n",
+ "import torchvision.transforms as transforms\n",
+ "import torchvision.transforms.functional as F_t\n",
+ "from torchvision.transforms import InterpolationMode\n",
+ "\n",
+ "class RandomPixelate:\n",
+ " \"\"\"Simulates low-quality webcams or low-bandwidth video.\"\"\"\n",
+ " def __init__(self, p=0.2, min_res=32, max_res=112):\n",
+ " self.p = p\n",
+ " self.min_res = min_res \n",
+ " self.max_res = max_res\n",
+ "\n",
+ " def __call__(self, img):\n",
+ " if random.random() < self.p:\n",
+ " orig_w, orig_h = img.size\n",
+ " degraded_h = random.randint(self.min_res, self.max_res)\n",
+ " degraded_w = int(orig_w * (degraded_h / orig_h))\n",
+ " # Downsample then upsample to create pixelation effect\n",
+ " img = F_t.resize(img, [degraded_h, degraded_w], interpolation=InterpolationMode.NEAREST)\n",
+ " img = F_t.resize(img, [orig_h, orig_w], interpolation=InterpolationMode.NEAREST)\n",
+ " return img\n",
+ " \n",
+ "\n",
+ "class SAM3DStudentDataset(Dataset):\n",
+ " def __init__(self, images_dir: str, annotations_dir: str,\n",
+ " image_size: int = 224, max_images: int | None = None,\n",
+ " augment: bool = True):\n",
+ " super().__init__()\n",
+ " self.image_size = image_size\n",
+ " self.images_dir = Path(images_dir)\n",
+ " self.annotations_dir = Path(annotations_dir)\n",
+ "\n",
+ " # --- 1. Build Transform Pipeline ---\n",
+ " # The images on disk are already 224x224, but Resize guarantees safety.\n",
+ " tfm_list = [transforms.Resize((image_size, image_size))]\n",
+ " \n",
+ " if augment:\n",
+ " tfm_list.extend([\n",
+ " transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n",
+ " RandomPixelate(p=0.2, min_res=48, max_res=112), \n",
+ " transforms.RandomApply([transforms.GaussianBlur(kernel_size=5)], p=0.2), \n",
+ " ])\n",
+ " \n",
+ " tfm_list.extend([\n",
+ " transforms.ToTensor(),\n",
+ " # Standard ImageNet Normalization\n",
+ " transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
+ " std=[0.229, 0.224, 0.225]),\n",
+ " ])\n",
+ " \n",
+ " if augment:\n",
+ " tfm_list.append(transforms.RandomErasing(p=0.2, scale=(0.02, 0.15)))\n",
+ " \n",
+ " self.transform = transforms.Compose(tfm_list)\n",
+ "\n",
+ " # --- 2. Pair Discovery ---\n",
+ " # Fast indexing of available NPZ files and corresponding images\n",
+ " ann_files = list(self.annotations_dir.glob(\"*.npz\"))\n",
+ " \n",
+ " self.pairs = []\n",
+ " for npz_path in sorted(ann_files):\n",
+ " stem = npz_path.stem\n",
+ " # Check for jpg or png\n",
+ " img_path = self.images_dir / f\"{stem}.jpg\"\n",
+ " if not img_path.exists():\n",
+ " img_path = self.images_dir / f\"{stem}.png\"\n",
+ " \n",
+ " if img_path.exists():\n",
+ " self.pairs.append((img_path, npz_path))\n",
+ "\n",
+ " if max_images is not None:\n",
+ " self.pairs = self.pairs[:max_images]\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.pairs)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " img_path, npz_path = self.pairs[idx]\n",
+ "\n",
+ " # --- 3. Load and Augment Image ---\n",
+ " image = Image.open(img_path).convert(\"RGB\")\n",
+ " image = self.transform(image) # Returns float tensor [3, 224, 224]\n",
+ "\n",
+ " # --- 4. Load Annotations ---\n",
+ " ann = np.load(npz_path)\n",
+ " \n",
+ " orig_h, orig_w = ann[\"orig_shape\"]\n",
+ " \n",
+ " # --- 5. Correct Mathematical Rescaling for 2D Joints ---\n",
+ " sq_bbox = ann[\"bbox_square\"] \n",
+ " sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]\n",
+ " orig_crop_size = sq_bbox[2] - sq_bbox[0]\n",
+ " \n",
+ " # Scale factor from the original crop square to the 224x224 tensor\n",
+ " true_scale = self.image_size / max(orig_crop_size, 1.0)\n",
+ "\n",
+ " # Shift full-frame 2D joints relative to crop top-left, then scale\n",
+ " joints_2d = ann[\"joints_2d\"].copy()\n",
+ " joints_2d[:, 0] = (joints_2d[:, 0] - sq_x1) * true_scale\n",
+ " joints_2d[:, 1] = (joints_2d[:, 1] - sq_y1) * true_scale\n",
+ "\n",
+ " # NEW: Normalize to [-1, 1] for neural network stability!\n",
+ " joints_2d[:, 0] = (joints_2d[:, 0] / self.image_size) * 2.0 - 1.0\n",
+ " joints_2d[:, 1] = (joints_2d[:, 1] / self.image_size) * 2.0 - 1.0\n",
+ "\n",
+ " joints_2d = torch.from_numpy(joints_2d).float()\n",
+ "\n",
+ " # --- 6. CLIFF Conditioning Vectors ---\n",
+ " # Using the tighter YOLO bbox to tell the network where the person actually is\n",
+ " tight_bbox = ann[\"bbox\"] \n",
+ " cx = (tight_bbox[0] + tight_bbox[2]) / 2.0\n",
+ " cy = (tight_bbox[1] + tight_bbox[3]) / 2.0\n",
+ " \n",
+ " # Normalize to [-1, 1] space for better network stability\n",
+ " cx_norm = 2.0 * (cx / orig_w) - 1.0\n",
+ " cy_norm = 2.0 * (cy / orig_h) - 1.0\n",
+ " \n",
+ " # Scale relative to the longest edge of the original image\n",
+ " b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])\n",
+ " b_scale = b_size / max(orig_w, orig_h)\n",
+ " cliff_cond = torch.tensor([cx_norm, cy_norm, b_scale], dtype=torch.float32)\n",
+ "\n",
+ " return {\n",
+ " \"image\": image, # [3, 224, 224]\n",
+ " \"cliff_cond\": cliff_cond, # [3]\n",
+ " \"mhr_model_params\": torch.from_numpy(ann[\"mhr_model_params\"]).float(), # [204]\n",
+ " \"shape_params\": torch.from_numpy(ann[\"shape_params\"]).float(), # [45]\n",
+ " \"cam_trans\": torch.from_numpy(ann[\"cam_trans\"]).float(), # [3]\n",
+ " \"cam_focal\": torch.from_numpy(ann[\"cam_focal_length\"]).float(), # [2]\n",
+ " \"joints_2d\": joints_2d, # [70, 2] Crop Space\n",
+ " \"joints_3d\": torch.from_numpy(ann[\"joints_3d\"]).float(), # [70, 3] Root-centered\n",
+ " \"orig_shape\": torch.tensor([orig_h, orig_w], dtype=torch.float32), # [2]\n",
+ " \"bbox_square\": sq_bbox\n",
+ " }\n",
+ "\n",
+ "# --- Initialization & Dataloaders ---\n",
+ "full_dataset = SAM3DStudentDataset(\n",
+ " cfg.images_dir, \n",
+ " cfg.annotations_dir, \n",
+ " augment=cfg.augment, \n",
+ " max_images=cfg.max_images\n",
+ ")\n",
+ "\n",
+ "val_dataset_clean = SAM3DStudentDataset(\n",
+ " cfg.images_dir, \n",
+ " cfg.annotations_dir, \n",
+ " augment=False, \n",
+ " max_images=cfg.max_images\n",
+ ")\n",
+ "\n",
+ "n_val = int(len(full_dataset) * cfg.val_split)\n",
+ "n_train = len(full_dataset) - n_val\n",
+ "\n",
+ "# Use a generator for reproducible splits\n",
+ "generator = torch.Generator().manual_seed(42)\n",
+ "train_dataset, _ = random_split(full_dataset, [n_train, n_val], generator=generator)\n",
+ "\n",
+ "generator = torch.Generator().manual_seed(42)\n",
+ "_, val_dataset = random_split(val_dataset_clean, [n_train, n_val], generator=generator)\n",
+ "\n",
+ "train_loader = DataLoader(\n",
+ " train_dataset, \n",
+ " batch_size=cfg.batch_size, \n",
+ " shuffle=True, \n",
+ " num_workers=cfg.num_workers, \n",
+ " pin_memory=True, \n",
+ " drop_last=True\n",
+ ")\n",
+ "\n",
+ "val_loader = DataLoader(\n",
+ " val_dataset, \n",
+ " batch_size=cfg.batch_size, \n",
+ " shuffle=False, \n",
+ " num_workers=cfg.num_workers, \n",
+ " pin_memory=True\n",
+ ")\n",
+ "\n",
+ "\n",
+ "print(f\"Dataset Loaded! Train batches: {len(train_loader)} | Val batches: {len(val_loader)}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0f003098",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ==============================================================================\n",
+ "# Cell 4 — CLIFF & Augmentation Sanity Check\n",
+ "# ==============================================================================\n",
+ "import matplotlib.pyplot as plt\n",
+ "import cv2\n",
+ "\n",
+ "# 1. Grab a single batch from the DataLoader\n",
+ "sample_batch = next(iter(train_loader))\n",
+ "\n",
+ "# Extract the first item in the batch\n",
+ "idx = 0\n",
+ "img_tensor = sample_batch[\"image\"][idx]\n",
+ "# Un-normalize the 2D joints for visualization ---\n",
+ "joints_2d_norm = sample_batch[\"joints_2d\"][idx].numpy() # Currently in [-1, 1] space\n",
+ "joints_2d_crop = ((joints_2d_norm + 1.0) / 2.0) * cfg.image_size # Map back to [0, 224] pixel space\n",
+ "\n",
+ "# Get MHR parameters and Camera data\n",
+ "mhr_params = sample_batch[\"mhr_model_params\"][idx:idx+1].to(device)\n",
+ "shape_params = sample_batch[\"shape_params\"][idx:idx+1].to(device)\n",
+ "cam_trans = sample_batch[\"cam_trans\"][idx].numpy()\n",
+ "cam_focal = sample_batch[\"cam_focal\"][idx].numpy() # Original UNTOUCHED focal length\n",
+ "\n",
+ "# Get CLIFF Conditioning & Geometry data\n",
+ "cliff_cond = sample_batch[\"cliff_cond\"][idx].numpy()\n",
+ "orig_shape = sample_batch[\"orig_shape\"][idx].numpy() # [H, W]\n",
+ "bbox_square = sample_batch[\"bbox_square\"][idx].numpy() # [x1, y1, x2, y2]\n",
+ "\n",
+ "orig_h, orig_w = orig_shape[0], orig_shape[1]\n",
+ "\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# SANITY CHECK 1: Reverse-Engineer CLIFF Conditioning\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# cliff_cond = [cx_norm, cy_norm, scale]\n",
+ "cx_full = ((cliff_cond[0] + 1.0) / 2.0) * orig_w\n",
+ "cy_full = ((cliff_cond[1] + 1.0) / 2.0) * orig_h\n",
+ "box_size_full = cliff_cond[2] * max(orig_h, orig_w)\n",
+ "\n",
+ "print(\"--- CLIFF Conditioning Verification ---\")\n",
+ "print(f\"Original Image Size: {orig_w}x{orig_h}\")\n",
+ "print(f\"BBox Center in Full Image: X={cx_full:.1f}, Y={cy_full:.1f}\")\n",
+ "print(f\"BBox Max Dimension: {box_size_full:.1f} pixels\")\n",
+ "print(f\"Does this look like a human bounding box? (Yes, if center is within image bounds)\")\n",
+ "print(\"---------------------------------------\")\n",
+ "\n",
+ "# 2. Un-normalize the image tensor for viewing\n",
+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",
+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",
+ "img_unnorm = img_tensor * std + mean\n",
+ "img_np = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",
+ "\n",
+ "# 3. Generate 3D Joints using MHR\n",
+ "raw_3d_joints = mhr_module.get_joints(mhr_params, shape_params)[0, :, :3].cpu().numpy()\n",
+ "raw_3d_joints = raw_3d_joints / 100.0 # cm to meters\n",
+ "raw_3d_joints[:, 1] = -raw_3d_joints[:, 1] # Y-down convention\n",
+ "raw_3d_joints[:, 2] = -raw_3d_joints[:, 2] # Z-forward convention\n",
+ "\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# SANITY CHECK 2: The CLIFF Projection Math\n",
+ "# ------------------------------------------------------------------------------\n",
+ "# A. Project natively into the FULL FRAME space\n",
+ "j3d_cam = raw_3d_joints + cam_trans\n",
+ "z = np.maximum(j3d_cam[:, 2], 0.01)\n",
+ "\n",
+ "principal_point_x = orig_w / 2.0\n",
+ "principal_point_y = orig_h / 2.0\n",
+ "\n",
+ "u_full = (cam_focal[0] * j3d_cam[:, 0] / z) + principal_point_x\n",
+ "v_full = (cam_focal[1] * j3d_cam[:, 1] / z) + principal_point_y\n",
+ "\n",
+ "# B. Map the full-frame 2D points into the 224x224 crop space\n",
+ "sq_x1, sq_y1, sq_x2, sq_y2 = bbox_square\n",
+ "crop_w = sq_x2 - sq_x1\n",
+ "crop_h = sq_y2 - sq_y1\n",
+ "true_scale = 224.0 / max(crop_w, 1.0)\n",
+ "\n",
+ "u_crop_proj = (u_full - sq_x1) * true_scale\n",
+ "v_crop_proj = (v_full - sq_y1) * true_scale\n",
+ "\n",
+ "# 4. Visualize\n",
+ "fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n",
+ "\n",
+ "# Plot 1: The Native 2D labels from the dataset (Checks augmentation/mapping)\n",
+ "axes[0].imshow(img_np)\n",
+ "axes[0].scatter(joints_2d_crop[:, 0], joints_2d_crop[:, 1], s=15, c='lime', edgecolors='black', label='Native 2D')\n",
+ "axes[0].set_title(f\"Target 1: Native 2D Joints\\n(Verifies Augmentations & Crop mapping)\", fontsize=10)\n",
+ "axes[0].legend(loc='upper right')\n",
+ "axes[0].axis('off')\n",
+ "\n",
+ "# Plot 2: The 3D -> 2D CLIFF Projection (Checks focal length & cam_trans)\n",
+ "axes[1].imshow(img_np)\n",
+ "axes[1].scatter(u_crop_proj, v_crop_proj, s=15, c='red', edgecolors='black', marker='X')\n",
+ "axes[1].set_title(f\"Target 2: CLIFF Projected 3D MHR\\n(Verifies Full-Frame Camera Math)\", fontsize=10)\n",
+ "axes[1].axis('off')\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "47dea531",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ==============================================================================\n",
+ "# Cell 5 — Data-Driven Topology Discovery (Pixel-Space Edition - FIXED)\n",
+ "# ==============================================================================\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib.lines as mlines\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "print(\"--- Starting 2D Pixel-Space Topology Discovery ---\")\n",
+ "\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",
+ "\n",
+ "# --- Configuration ---\n",
+ "NUM_BATCHES_TO_SCAN = 100 \n",
+ "PIXEL_THRESHOLD = 1.0 # 15 pixels is a great sweet spot for 224x224 anatomical matches\n",
+ "\n",
+ "cumulative_dist_matrix = torch.zeros((cfg.num_joints, 127), device=device)\n",
+ "total_samples = 0\n",
+ "\n",
+ "print(f\"Scanning {NUM_BATCHES_TO_SCAN} batches to compute average 2D pixel distances...\")\n",
+ "train_iter = iter(train_loader)\n",
+ "\n",
+ "for _ in tqdm(range(NUM_BATCHES_TO_SCAN)):\n",
+ " batch = next(train_iter)\n",
+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " B = batch[\"mhr_model_params\"].shape[0]\n",
+ " \n",
+ " # 1. Un-normalize Native 2D to Crop Space [0, 224]\n",
+ " native_2d_norm = batch[\"joints_2d\"][..., :2] \n",
+ " native_2d_crop = ((native_2d_norm + 1.0) / 2.0) * cfg.image_size \n",
+ " \n",
+ " # 2. Extract 3D MHR Joints\n",
+ " with torch.no_grad():\n",
+ " mhr_3d_raw = mhr_module.get_joints(batch[\"mhr_model_params\"], batch[\"shape_params\"])[..., :3]\n",
+ " \n",
+ " mhr_3d = mhr_3d_raw / 100.0 \n",
+ " mhr_3d[..., 1] = -mhr_3d[..., 1] \n",
+ " mhr_3d[..., 2] = -mhr_3d[..., 2] \n",
+ " \n",
+ " # 3. Apply CLIFF Projection Math (Batched properly!)\n",
+ " cam_trans = batch[\"cam_trans\"].unsqueeze(1) # [B, 1, 3]\n",
+ " j3d_cam = mhr_3d + cam_trans # [B, 127, 3]\n",
+ " z = torch.clamp(j3d_cam[..., 2], min=0.01) # [B, 127]\n",
+ " \n",
+ " # FIXED: Removed the buggy .unsqueeze(1) so shapes match [B, 1] * [B, 127]\n",
+ " orig_shape = batch[\"orig_shape\"]\n",
+ " orig_w, orig_h = orig_shape[:, 1:2], orig_shape[:, 0:1] # [B, 1]\n",
+ " cx, cy = orig_w / 2.0, orig_h / 2.0 # [B, 1]\n",
+ " \n",
+ " focal_key = \"cam_focal\" if \"cam_focal\" in batch else \"cam_focal_length\"\n",
+ " cam_focal = batch[focal_key]\n",
+ " fx, fy = cam_focal[:, 0:1], cam_focal[:, 1:2] # [B, 1]\n",
+ " \n",
+ " u_full = (fx * j3d_cam[..., 0] / z) + cx # [B, 127]\n",
+ " v_full = (fy * j3d_cam[..., 1] / z) + cy # [B, 127]\n",
+ " \n",
+ " bbox = batch[\"bbox_square\"]\n",
+ " sq_x1, sq_y1, sq_x2 = bbox[:, 0:1], bbox[:, 1:2], bbox[:, 2:3] # [B, 1]\n",
+ " crop_w = torch.clamp(sq_x2 - sq_x1, min=1.0) # [B, 1]\n",
+ " true_scale = cfg.image_size / crop_w # [B, 1]\n",
+ " \n",
+ " u_crop = (u_full - sq_x1) * true_scale # [B, 127]\n",
+ " v_crop = (v_full - sq_y1) * true_scale # [B, 127]\n",
+ " mhr_2d_crop = torch.stack([u_crop, v_crop], dim=-1) # [B, 127, 2]\n",
+ " \n",
+ " # 4. Compute Pairwise Distance in Pixel Space!\n",
+ " dist_matrix = torch.cdist(native_2d_crop, mhr_2d_crop) # [B, 70, 127]\n",
+ " cumulative_dist_matrix += dist_matrix.sum(dim=0)\n",
+ " total_samples += B\n",
+ "\n",
+ "# Average the distance matrix\n",
+ "avg_dist_matrix = cumulative_dist_matrix / total_samples\n",
+ "\n",
+ "# --- Extract the Mapping ---\n",
+ "min_dists, closest_mhr_indices = avg_dist_matrix.min(dim=1)\n",
+ "\n",
+ "valid_mask = min_dists < PIXEL_THRESHOLD\n",
+ "native_mapping_ids = torch.arange(70, device=device)[valid_mask].cpu().numpy()\n",
+ "mhr_mapping_ids = closest_mhr_indices[valid_mask].cpu().numpy()\n",
+ "matched_distances = min_dists[valid_mask].cpu().numpy()\n",
+ "\n",
+ "native_ids_2d, mhr_ids_2d = native_mapping_ids, mhr_mapping_ids \n",
+ "print(\"\\n--- Discovery Complete ---\")\n",
+ "print(f\"Found {len(native_mapping_ids)} valid anchor pairs with average distance < {PIXEL_THRESHOLD} pixels.\")\n",
+ "\n",
+ "# ==============================================================================\n",
+ "# VISUALIZATION: Plotting the Anchors on a Sample\n",
+ "# ==============================================================================\n",
+ "sample_idx = 0\n",
+ "nat_2d_viz = native_2d_crop[sample_idx].cpu().numpy()\n",
+ "mhr_2d_viz = mhr_2d_crop[sample_idx].cpu().numpy()\n",
+ "\n",
+ "# Un-normalize image\n",
+ "img_tensor = batch[\"image\"][sample_idx].cpu()\n",
+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",
+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",
+ "img_unnorm = img_tensor * std + mean\n",
+ "img_rgb = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(8, 8))\n",
+ "ax.imshow(img_rgb)\n",
+ "ax.set_title(f\"Data-Driven Anchor Mapping (Thresh: {PIXEL_THRESHOLD} px)\", fontsize=14)\n",
+ "\n",
+ "# Plot all joints faintly\n",
+ "ax.scatter(nat_2d_viz[:, 0], nat_2d_viz[:, 1], s=20, c='cyan', alpha=0.3, label='All Native')\n",
+ "ax.scatter(mhr_2d_viz[:, 0], mhr_2d_viz[:, 1], s=20, c='red', alpha=0.3, label='All MHR')\n",
+ "\n",
+ "# Draw connections for the valid mapped anchors\n",
+ "for n_idx, m_idx in zip(native_mapping_ids, mhr_mapping_ids):\n",
+ " ax.scatter(nat_2d_viz[n_idx, 0], nat_2d_viz[n_idx, 1], s=50, c='lime', edgecolors='black', zorder=5)\n",
+ " ax.scatter(mhr_2d_viz[m_idx, 0], mhr_2d_viz[m_idx, 1], s=50, c='yellow', marker='X', edgecolors='black', zorder=5)\n",
+ " \n",
+ " line = mlines.Line2D([nat_2d_viz[n_idx, 0], mhr_2d_viz[m_idx, 0]], \n",
+ " [nat_2d_viz[n_idx, 1], mhr_2d_viz[m_idx, 1]], \n",
+ " color='magenta', linewidth=1.5, zorder=4)\n",
+ " ax.add_line(line)\n",
+ "\n",
+ "# Clean up legend\n",
+ "handles, labels = ax.get_legend_handles_labels()\n",
+ "match_marker = mlines.Line2D([], [], color='magenta', marker='o', markersize=8, markerfacecolor='lime', label='Discovered Anchors')\n",
+ "ax.legend(handles=[handles[0], handles[1], match_marker], loc='upper right')\n",
+ "ax.axis('off')\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "af8c0277",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ==============================================================================\n",
+ "# Cell 6 — Data-Driven Topology Discovery (3D Meter Space - FIXED ALIGNMENT)\n",
+ "# ==============================================================================\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from mpl_toolkits.mplot3d import Axes3D\n",
+ "import matplotlib.lines as mlines\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "print(\"--- Starting 3D Topology Discovery ---\")\n",
+ "\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",
+ "\n",
+ "# --- Configuration ---\n",
+ "NUM_BATCHES_TO_SCAN = 100 \n",
+ "# 4 cm is a rigorous but realistic threshold for overlapping anatomical joints\n",
+ "DISTANCE_THRESHOLD_METERS = 0.01 \n",
+ "\n",
+ "cumulative_dist_matrix = torch.zeros((cfg.num_joints, 127), device=device)\n",
+ "total_samples = 0\n",
+ "\n",
+ "print(f\"Scanning {NUM_BATCHES_TO_SCAN} batches to compute true 3D distances...\")\n",
+ "train_iter = iter(train_loader)\n",
+ "\n",
+ "for _ in tqdm(range(NUM_BATCHES_TO_SCAN)):\n",
+ " batch = next(train_iter)\n",
+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " B = batch[\"mhr_model_params\"].shape[0]\n",
+ " \n",
+ " # 1. Extract 3D Native Joints (Body-Centered, Meters, Y-Down, Z-Forward)\n",
+ " native_3d = batch[\"joints_3d\"][..., :3] \n",
+ " \n",
+ " # 2. Extract 3D MHR Joints (Internal coords, cm)\n",
+ " with torch.no_grad():\n",
+ " mhr_3d_raw = mhr_module.get_joints(batch[\"mhr_model_params\"], batch[\"shape_params\"])[..., :3]\n",
+ " \n",
+ " # Convert MHR to the exact same frame as Native 3D!\n",
+ " mhr_3d = mhr_3d_raw / 100.0 \n",
+ " mhr_3d[..., 1] = -mhr_3d[..., 1] \n",
+ " mhr_3d[..., 2] = -mhr_3d[..., 2] \n",
+ " \n",
+ " # NO MEAN CENTERING! They are already in the same coordinate space!\n",
+ " \n",
+ " # 3. Compute Pairwise Distance in 3D METER Space [B, 70, 127]\n",
+ " dist_matrix = torch.cdist(native_3d, mhr_3d) \n",
+ " cumulative_dist_matrix += dist_matrix.sum(dim=0)\n",
+ " total_samples += B\n",
+ "\n",
+ "# Average the distance matrix\n",
+ "avg_dist_matrix = cumulative_dist_matrix / total_samples\n",
+ "\n",
+ "# --- Extract the Mapping ---\n",
+ "min_dists, closest_mhr_indices = avg_dist_matrix.min(dim=1)\n",
+ "\n",
+ "valid_mask = min_dists < DISTANCE_THRESHOLD_METERS\n",
+ "native_mapping_ids = torch.arange(70, device=device)[valid_mask].cpu().numpy()\n",
+ "mhr_mapping_ids = closest_mhr_indices[valid_mask].cpu().numpy()\n",
+ "matched_distances = min_dists[valid_mask].cpu().numpy()\n",
+ "\n",
+ "native_ids_3d, mhr_ids_3d = native_mapping_ids, mhr_mapping_ids \n",
+ "print(\"\\n--- Discovery Complete ---\")\n",
+ "print(f\"Found {len(native_mapping_ids)} valid anchor pairs with average distance < {DISTANCE_THRESHOLD_METERS} meters.\")\n",
+ "\n",
+ "# ==============================================================================\n",
+ "# VISUALIZATION: Interactive 3D Plot with Plotly\n",
+ "# ==============================================================================\n",
+ "import plotly.graph_objects as go\n",
+ "import numpy as np\n",
+ "\n",
+ "sample_idx = 0\n",
+ "# Extract raw coordinates\n",
+ "nat_3d_viz = native_3d[sample_idx].cpu().numpy()\n",
+ "mhr_3d_viz = mhr_3d[sample_idx].cpu().numpy()\n",
+ "\n",
+ "# To make the person stand upright in Plotly (where Z is usually up):\n",
+ "# Data X -> Plotly X\n",
+ "# Data Z (Forward) -> Plotly Y (Depth)\n",
+ "# Data -Y (Down) -> Plotly Z (Height)\n",
+ "def map_coords(pts):\n",
+ " return pts[:, 0], pts[:, 2], -pts[:, 1]\n",
+ "\n",
+ "nx, ny, nz = map_coords(nat_3d_viz)\n",
+ "mx, my, mz = map_coords(mhr_3d_viz)\n",
+ "\n",
+ "fig = go.Figure()\n",
+ "\n",
+ "# 1. Plot Unmatched Native Joints (Cyan)\n",
+ "fig.add_trace(go.Scatter3d(\n",
+ " x=nx, y=ny, z=nz,\n",
+ " mode='markers',\n",
+ " marker=dict(size=4, color='cyan', opacity=0.4),\n",
+ " name='All Native (70)',\n",
+ " text=[f\"Native Joint: {i}\" for i in range(70)],\n",
+ " hoverinfo='text'\n",
+ "))\n",
+ "\n",
+ "# 2. Plot Unmatched MHR Joints (Red)\n",
+ "fig.add_trace(go.Scatter3d(\n",
+ " x=mx, y=my, z=mz,\n",
+ " mode='markers',\n",
+ " marker=dict(size=4, color='red', opacity=0.4),\n",
+ " name='All MHR (127)',\n",
+ " text=[f\"MHR Joint: {i}\" for i in range(127)],\n",
+ " hoverinfo='text'\n",
+ "))\n",
+ "\n",
+ "# 3. Plot Matched Anchors\n",
+ "# Extract coordinates for the matched pairs\n",
+ "nat_anchors_x, nat_anchors_y, nat_anchors_z = nx[native_mapping_ids], ny[native_mapping_ids], nz[native_mapping_ids]\n",
+ "mhr_anchors_x, mhr_anchors_y, mhr_anchors_z = mx[mhr_mapping_ids], my[mhr_mapping_ids], mz[mhr_mapping_ids]\n",
+ "\n",
+ "# Native Anchors (Green)\n",
+ "fig.add_trace(go.Scatter3d(\n",
+ " x=nat_anchors_x, y=nat_anchors_y, z=nat_anchors_z,\n",
+ " mode='markers',\n",
+ " marker=dict(size=7, color='lime', line=dict(color='black', width=1)),\n",
+ " name='Native Anchors',\n",
+ " text=[f\"Matched Native: {i}\" for i in native_mapping_ids],\n",
+ " hoverinfo='text'\n",
+ "))\n",
+ "\n",
+ "# MHR Anchors (Yellow)\n",
+ "fig.add_trace(go.Scatter3d(\n",
+ " x=mhr_anchors_x, y=mhr_anchors_y, z=mhr_anchors_z,\n",
+ " mode='markers',\n",
+ " marker=dict(size=7, color='yellow', symbol='cross', line=dict(color='black', width=1)),\n",
+ " name='MHR Anchors',\n",
+ " text=[f\"Matched MHR: {i}\" for i in mhr_mapping_ids],\n",
+ " hoverinfo='text'\n",
+ "))\n",
+ "\n",
+ "# 4. Draw Connection Lines (Magenta)\n",
+ "# To draw multiple disconnected lines efficiently in Plotly, we interleave with None\n",
+ "line_x, line_y, line_z = [], [], []\n",
+ "for i in range(len(native_mapping_ids)):\n",
+ " line_x.extend([nat_anchors_x[i], mhr_anchors_x[i], None])\n",
+ " line_y.extend([nat_anchors_y[i], mhr_anchors_y[i], None])\n",
+ " line_z.extend([nat_anchors_z[i], mhr_anchors_z[i], None])\n",
+ "\n",
+ "fig.add_trace(go.Scatter3d(\n",
+ " x=line_x, y=line_y, z=line_z,\n",
+ " mode='lines',\n",
+ " line=dict(color='magenta', width=4),\n",
+ " name='Physical Distance',\n",
+ " hoverinfo='none'\n",
+ "))\n",
+ "\n",
+ "# 5. Layout and Camera Setup\n",
+ "fig.update_layout(\n",
+ " title=f\"Interactive 3D Anchor Mapping (Thresh: {DISTANCE_THRESHOLD_METERS*100:.1f} cm)\",\n",
+ " scene=dict(\n",
+ " xaxis_title='X (Meters)',\n",
+ " yaxis_title='Z Depth (Meters)',\n",
+ " zaxis_title='Y Height (Meters)',\n",
+ " aspectmode='data' # THIS IS CRITICAL: Keeps the human proportions physically accurate!\n",
+ " ),\n",
+ " margin=dict(l=0, r=0, b=0, t=40),\n",
+ " legend=dict(x=0.8, y=0.9)\n",
+ ")\n",
+ "\n",
+ "fig.show()\n",
+ "\n",
+ "print(\"--- Anchor Verification: 2D vs 3D ---\")\n",
+ "\n",
+ "# 1. Create sets of tuples for easy comparison\n",
+ "mapping_2d = set(zip(native_ids_2d, mhr_ids_2d))\n",
+ "mapping_3d = set(zip(native_ids_3d, mhr_ids_3d))\n",
+ "\n",
+ "# 2. Check for perfect agreement\n",
+ "if mapping_2d == mapping_3d:\n",
+ " print(\"✅ PERFECT MATCH! Both 2D and 3D discovery found the exact same 52 anchor pairs.\")\n",
+ "else:\n",
+ " print(\"⚠️ MISMATCH DETECTED. The lists differ slightly.\")\n",
+ " \n",
+ " # Find what's unique to 2D\n",
+ " only_in_2d = mapping_2d - mapping_3d\n",
+ " if only_in_2d:\n",
+ " print(f\"\\nPairs found ONLY in 2D ({len(only_in_2d)}):\")\n",
+ " for nat, mhr in sorted(only_in_2d):\n",
+ " print(f\" Native [{nat:02d}] <--> MHR [{mhr:03d}]\")\n",
+ " \n",
+ " # Find what's unique to 3D\n",
+ " only_in_3d = mapping_3d - mapping_2d\n",
+ " if only_in_3d:\n",
+ " print(f\"\\nPairs found ONLY in 3D ({len(only_in_3d)}):\")\n",
+ " for nat, mhr in sorted(only_in_3d):\n",
+ " print(f\" Native [{nat:02d}] <--> MHR [{mhr:03d}]\")\n",
+ "\n",
+ "# 3. Print the final agreed-upon list to copy into your config!\n",
+ "agreed_mapping = sorted(list(mapping_2d.intersection(mapping_3d)))\n",
+ "\n",
+ "print(\"\\n--- Final Verified Anchor Dictionary ---\")\n",
+ "print(\"Copy this into your DistillConfig:\")\n",
+ "print(\"native_mapping_ids = [\", \", \".join(str(pair[0]) for pair in agreed_mapping), \"]\")\n",
+ "print(\"mhr_mapping_ids = [\", \", \".join(str(pair[1]) for pair in agreed_mapping), \"]\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10f2586d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 7 — Student Architecture (Mini-SAM3D with PE)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import timm\n",
+ "\n",
+ "def get_2d_sincos_pos_embed(embed_dim, grid_size):\n",
+ " \"\"\"Generate 2D sine-cosine positional embedding for preserving spatial topology.\"\"\"\n",
+ " grid_h, grid_w = grid_size, grid_size\n",
+ " grid_y, grid_x = torch.meshgrid(torch.arange(grid_h), torch.arange(grid_w), indexing='ij')\n",
+ " \n",
+ " omega = torch.arange(embed_dim // 4).float() / (embed_dim // 4)\n",
+ " omega = 1.0 / (10000 ** omega)\n",
+ " \n",
+ " out_y = grid_y.flatten().unsqueeze(1) * omega.unsqueeze(0)\n",
+ " out_x = grid_x.flatten().unsqueeze(1) * omega.unsqueeze(0)\n",
+ " \n",
+ " pe_y = torch.cat([torch.sin(out_y), torch.cos(out_y)], dim=1)\n",
+ " pe_x = torch.cat([torch.sin(out_x), torch.cos(out_x)], dim=1)\n",
+ " \n",
+ " pos_embed = torch.cat([pe_y, pe_x], dim=1) # (H*W, embed_dim)\n",
+ " return pos_embed.unsqueeze(0) # (1, H*W, embed_dim)\n",
+ "\n",
+ "\n",
+ "class InstantHMRStudent(nn.Module):\n",
+ " def __init__(self, cfg, pretrained=True):\n",
+ " super().__init__()\n",
+ " self.cfg = cfg\n",
+ " \n",
+ " # 1. The Vision Backbone (RepViT)\n",
+ " self.backbone = timm.create_model(cfg.backbone, pretrained=pretrained, num_classes=0)\n",
+ " embed_dim = self.backbone.num_features \n",
+ " self.feat_proj = nn.Linear(embed_dim, cfg.d_model)\n",
+ " \n",
+ " # --- NEW: Positional Embedding for Transformer Memory ---\n",
+ " # RepViT naturally downsamples by 32x. For a 224x224 image, the grid is 7x7.\n",
+ " self.grid_size = cfg.image_size // 32\n",
+ " pos_embed = get_2d_sincos_pos_embed(cfg.d_model, self.grid_size)\n",
+ " # register_buffer ensures it moves to GPU automatically but isn't updated by the optimizer\n",
+ " self.register_buffer(\"mem_pos_embed\", pos_embed) \n",
+ " \n",
+ " # 2. Learnable Query Tokens (1 Global + 70 2D + 70 3D = 141 Queries)\n",
+ " self.num_global = 1\n",
+ " self.num_2d = cfg.num_joints\n",
+ " self.num_3d = cfg.num_joints\n",
+ " self.total_queries = self.num_global + self.num_2d + self.num_3d\n",
+ " \n",
+ " # self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model))\n",
+ " self.query_embed = nn.Parameter(torch.randn(1, self.total_queries, cfg.d_model) * 0.02)\n",
+ " \n",
+ " # CLIFF Conditioning Injector\n",
+ " self.cond_proj = nn.Sequential(\n",
+ " nn.Linear(cfg.cliff_dim, 128),\n",
+ " nn.GELU(),\n",
+ " nn.Linear(128, cfg.d_model)\n",
+ " )\n",
+ " \n",
+ " # 3. Transformer Cross-Attention Decoder\n",
+ " decoder_layer = nn.TransformerDecoderLayer(\n",
+ " d_model=cfg.d_model,\n",
+ " nhead=cfg.n_heads,\n",
+ " dim_feedforward=cfg.d_model * 4,\n",
+ " dropout=cfg.dropout,\n",
+ " activation=\"gelu\",\n",
+ " batch_first=True,\n",
+ " norm_first=True\n",
+ " )\n",
+ " self.transformer = nn.TransformerDecoder(decoder_layer, num_layers=cfg.n_decoder_layers)\n",
+ " \n",
+ " # 4. The Output Heads\n",
+ " \n",
+ " # Head A: Global Head (Token 0 -> MHR Params + Shape Params + Camera)\n",
+ " self.global_out_dim = cfg.model_params_dim + cfg.shape_dim + cfg.cam_dim\n",
+ " self.head_global = nn.Linear(cfg.d_model, self.global_out_dim)\n",
+ " \n",
+ " # --- Safely Split 2D Head ---\n",
+ " self.head_2d_feat = nn.Sequential(\n",
+ " nn.Linear(cfg.d_model, 256),\n",
+ " nn.GELU()\n",
+ " )\n",
+ " self.head_2d_xy = nn.Sequential(\n",
+ " nn.Linear(256, 2)\n",
+ " # nn.Tanh() # Bounds predictions to [-1, 1]!\n",
+ " )\n",
+ " if getattr(cfg, 'learn_uncertainty', False):\n",
+ " # Uncertainty must remain unbounded!\n",
+ " self.head_2d_unc = nn.Linear(256, 1) \n",
+ " \n",
+ " # Head C: 3D Keypoint Head (No Tanh here, 3D meters can exceed [-1, 1])\n",
+ " self.dim_3d = 4 if getattr(cfg, 'learn_uncertainty', False) else 3\n",
+ " self.head_3d = nn.Sequential(\n",
+ " nn.Linear(cfg.d_model, 256),\n",
+ " nn.GELU(),\n",
+ " nn.Linear(256, self.dim_3d)\n",
+ " )\n",
+ " \n",
+ " # Initialize camera Z translation\n",
+ " nn.init.constant_(self.head_global.bias[-1], 2.0)\n",
+ " nn.init.constant_(self.head_global.bias[-2], 0.0)\n",
+ " nn.init.constant_(self.head_global.bias[-3], 0.0)\n",
+ "\n",
+ " # ==========================================================\n",
+ " # HMR-Standard Zero-Initialization for Stability\n",
+ " # ==========================================================\n",
+ " # 1. Global Head (MHR Params) - Must start near neutral pose!\n",
+ " nn.init.normal_(self.head_global.weight, mean=0.0, std=1e-4)\n",
+ " # Zero out the biases (except the last 3 camera params we just set)\n",
+ " nn.init.constant_(self.head_global.bias[:-3], 0.0)\n",
+ " \n",
+ " # 2. 2D Spatial Head - Start with tiny weights\n",
+ " nn.init.normal_(self.head_2d_xy[0].weight, mean=0.0, std=1e-4)\n",
+ " nn.init.constant_(self.head_2d_xy[0].bias, 0.0)\n",
+ " \n",
+ " # 3. 3D Spatial Head - Start with tiny weights\n",
+ " nn.init.normal_(self.head_3d[-1].weight, mean=0.0, std=1e-4)\n",
+ " nn.init.constant_(self.head_3d[-1].bias, 0.0)\n",
+ "\n",
+ " def forward(self, images, cliff_cond):\n",
+ " B = images.shape[0]\n",
+ " \n",
+ " # 1. Extract spatial image features\n",
+ " img_feats = self.backbone.forward_features(images)\n",
+ " \n",
+ " # Defensive flattening \n",
+ " if img_feats.dim() == 4:\n",
+ " img_feats = img_feats.flatten(2).transpose(1, 2) # [B, H*W, C]\n",
+ " \n",
+ " memory = self.feat_proj(img_feats) \n",
+ " \n",
+ " # --- NEW: Inject Spatial Topology into Memory ---\n",
+ " # Broadcasting [1, H*W, d_model] across Batch size\n",
+ " memory = memory + self.mem_pos_embed \n",
+ " \n",
+ " # 2. Prepare Query Tokens [B, 141, d_model]\n",
+ " queries = self.query_embed.expand(B, -1, -1)\n",
+ " \n",
+ " cond = self.cond_proj(cliff_cond).unsqueeze(1)\n",
+ " queries = queries + cond\n",
+ " \n",
+ " # 3. Cross Attention\n",
+ " shared_feats = self.transformer(tgt=queries, memory=memory)\n",
+ " \n",
+ " # 4. Route tokens to their dedicated heads\n",
+ " feat_global = shared_feats[:, 0, :] \n",
+ " feat_2d = shared_feats[:, 1 : 1 + self.num_2d, :] \n",
+ " feat_3d = shared_feats[:, 1 + self.num_2d :, :] \n",
+ " \n",
+ " # --- Decode Global Token ---\n",
+ " global_preds = self.head_global(feat_global) \n",
+ " \n",
+ " idx_mhr = self.cfg.model_params_dim\n",
+ " idx_shape = idx_mhr + self.cfg.shape_dim\n",
+ " \n",
+ " pred_mhr_params = global_preds[:, :idx_mhr]\n",
+ " pred_shape_params = global_preds[:, idx_mhr : idx_shape]\n",
+ " pred_cam_trans = global_preds[:, idx_shape :]\n",
+ " \n",
+ " # --- Decode 2D Spatial Tokens (Safely!) ---\n",
+ " feat_2d_processed = self.head_2d_feat(feat_2d)\n",
+ " pred_xy = self.head_2d_xy(feat_2d_processed) # [B, 70, 2] bounded by Tanh\n",
+ " \n",
+ " if getattr(self.cfg, 'learn_uncertainty', False):\n",
+ " pred_unc = self.head_2d_unc(feat_2d_processed) # [B, 70, 1] unbounded\n",
+ " pred_joints_2d = torch.cat([pred_xy, pred_unc], dim=-1) # [B, 70, 3]\n",
+ " else:\n",
+ " pred_joints_2d = pred_xy\n",
+ " \n",
+ " # --- Decode 3D Spatial Tokens ---\n",
+ " pred_joints_3d = self.head_3d(feat_3d) \n",
+ " \n",
+ " return {\n",
+ " \"mhr_params\": pred_mhr_params,\n",
+ " \"shape_params\": pred_shape_params,\n",
+ " \"cam_trans\": pred_cam_trans,\n",
+ " \"joints_2d\": pred_joints_2d,\n",
+ " \"joints_3d\": pred_joints_3d\n",
+ " }\n",
+ "\n",
+ "# --- Quick Architecture Sanity Check ---\n",
+ "if __name__ == \"__main__\":\n",
+ " test_model = InstantHMRStudent(cfg, pretrained=False)\n",
+ " dummy_img = torch.randn(2, 3, 224, 224)\n",
+ " dummy_cliff = torch.randn(2, 3)\n",
+ " \n",
+ " out = test_model(dummy_img, dummy_cliff)\n",
+ " print(\"--- Architecture Output Shapes ---\")\n",
+ " for k, v in out.items():\n",
+ " print(f\"{k}: {v.shape}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9239175a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 8 — The Distillation Loss Module (Foolproof Edition)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "\n",
+ "# --- Modified Forward Pass (Differentiable) ---\n",
+ "class MHRForwardPass:\n",
+ " def __init__(self, mhr_path, device):\n",
+ " self.device = device\n",
+ " self.mhr = torch.jit.load(mhr_path, map_location=device).eval()\n",
+ " for p in self.mhr.parameters():\n",
+ " p.requires_grad_(False)\n",
+ "\n",
+ " def get_joints(self, model_params, shape_params):\n",
+ " B = model_params.shape[0]\n",
+ " dummy_identity = torch.zeros(B, 45, device=model_params.device, dtype=model_params.dtype)\n",
+ " concat_params = torch.cat([model_params, dummy_identity], dim=1)\n",
+ " \n",
+ " joint_params = self.mhr.character_torch.model_parameters_to_joint_parameters(concat_params.to(self.device))\n",
+ " skel_state = self.mhr.character_torch.joint_parameters_to_skeleton_state(joint_params)\n",
+ " \n",
+ " return skel_state.to(model_params.device)\n",
+ "\n",
+ "class DistillationLoss(nn.Module):\n",
+ " def __init__(self, cfg, mhr_module):\n",
+ " super().__init__()\n",
+ " self.cfg = cfg\n",
+ " self.mhr_module = mhr_module \n",
+ " self.mse = nn.MSELoss()\n",
+ " self.l1 = nn.SmoothL1Loss()\n",
+ " self.strict_l1 = nn.L1Loss()\n",
+ "\n",
+ " def _project_3d_to_norm_2d(self, j3d, pred_cam_trans, targets):\n",
+ " \"\"\"Helper to project 3D points (in meters) to normalized [-1, 1] 2D crop space.\"\"\"\n",
+ " # A. Move to Camera Space\n",
+ " j3d_cam = j3d + pred_cam_trans.unsqueeze(1)\n",
+ " \n",
+ " # Prevent 1/Z^2 Gradient Explosions\n",
+ " # 0.01 allows a gradient multiplier of 10,000. \n",
+ " # 0.4 meters (40cm) limits the multiplier to a safe limit, protecting the kinematic tree.\n",
+ " z_cam = torch.clamp(j3d_cam[..., 2], min=0.4)\n",
+ " \n",
+ " # B. Extract Intrinsics\n",
+ " fx = targets[\"cam_focal\"][:, 0:1] # Shape: [B, 1]\n",
+ " fy = targets[\"cam_focal\"][:, 1:2] # Shape: [B, 1]\n",
+ " cx = targets[\"orig_shape\"][:, 1:2] / 2.0 # Shape: [B, 1]\n",
+ " cy = targets[\"orig_shape\"][:, 0:1] / 2.0 # Shape: [B, 1]\n",
+ " \n",
+ " # C. Full-Frame Projection\n",
+ " u_full = (fx * j3d_cam[..., 0] / z_cam) + cx\n",
+ " v_full = (fy * j3d_cam[..., 1] / z_cam) + cy\n",
+ " \n",
+ " # D. Map to Crop Space\n",
+ " bbox = targets[\"bbox_square\"]\n",
+ " sq_x1 = bbox[:, 0:1] # [B, 1]\n",
+ " sq_y1 = bbox[:, 1:2] # [B, 1]\n",
+ " crop_w = torch.clamp(bbox[:, 2:3] - bbox[:, 0:1], min=1.0) # [B, 1]\n",
+ " \n",
+ " true_scale = self.cfg.image_size / crop_w\n",
+ " u_crop = (u_full - sq_x1) * true_scale\n",
+ " v_crop = (v_full - sq_y1) * true_scale\n",
+ " \n",
+ " # E. Normalize to [-1, 1]\n",
+ " u_norm = (u_crop / self.cfg.image_size) * 2.0 - 1.0\n",
+ " v_norm = (v_crop / self.cfg.image_size) * 2.0 - 1.0\n",
+ " \n",
+ " return torch.stack([u_norm, v_norm], dim=-1) # Shape: [B, J, 2]\n",
+ "\n",
+ " def forward(self, preds, targets):\n",
+ " losses = {}\n",
+ " idx_pose = self.cfg.pose_dim\n",
+ " \n",
+ " pred_pose = preds[\"mhr_params\"][:, :idx_pose]\n",
+ " pred_scale = preds[\"mhr_params\"][:, idx_pose:]\n",
+ " tgt_pose = targets[\"mhr_model_params\"][:, :idx_pose]\n",
+ " tgt_scale = targets[\"mhr_model_params\"][:, idx_pose:]\n",
+ " \n",
+ " # --- 1. Base Parameters ---\n",
+ " losses['loss_pose'] = self.l1(pred_pose, tgt_pose) * self.cfg.w_pose\n",
+ " losses['loss_scale'] = self.mse(pred_scale, tgt_scale) * self.cfg.w_scale\n",
+ " losses['loss_shape'] = self.mse(preds[\"shape_params\"], targets[\"shape_params\"]) * self.cfg.w_shape\n",
+ " losses['loss_cam'] = self.mse(preds[\"cam_trans\"], targets[\"cam_trans\"]) * self.cfg.w_cam\n",
+ " \n",
+ " # --- 2. Differentiable 3D Joint Alignment (Centimeter space corrected!) ---\n",
+ " pred_mhr_joints = self.mhr_module.get_joints(preds[\"mhr_params\"], preds[\"shape_params\"])[..., :3]\n",
+ " with torch.no_grad():\n",
+ " tgt_mhr_joints = self.mhr_module.get_joints(targets[\"mhr_model_params\"], targets[\"shape_params\"])[..., :3]\n",
+ " \n",
+ " losses['loss_mhr_joints'] = self.l1(pred_mhr_joints, tgt_mhr_joints.detach()) * self.cfg.w_3d_joints\n",
+ "\n",
+ " # --- Base MHR Vision Coordinates (Needed for Structural & Reprojection) ---\n",
+ " pred_mhr_vision = pred_mhr_joints.clone() / 100.0\n",
+ " pred_mhr_vision[..., 1] = -pred_mhr_vision[..., 1]\n",
+ " pred_mhr_vision[..., 2] = -pred_mhr_vision[..., 2]\n",
+ "\n",
+ " # ==============================================================================\n",
+ " # 2.5 Structural Consistency Loss (MHR mesh vs Native Keypoints)\n",
+ " # ==============================================================================\n",
+ " if hasattr(self.cfg, 'native_mapping_ids') and hasattr(self.cfg, 'mhr_mapping_ids'):\n",
+ " pred_native_anchors = preds[\"joints_3d\"][:, self.cfg.native_mapping_ids, :3]\n",
+ " pred_mhr_anchors = pred_mhr_vision[:, self.cfg.mhr_mapping_ids, :3]\n",
+ " losses['loss_structural'] = self.l1(pred_native_anchors, pred_mhr_anchors) * self.cfg.w_structural\n",
+ "\n",
+ " # ==============================================================================\n",
+ " # Feet Barycenter Alignment Loss\n",
+ " # ==============================================================================\n",
+ " if hasattr(self.cfg, 'native_left_foot') and hasattr(self.cfg, 'w_feet'):\n",
+ " nat_lf_pts = preds[\"joints_3d\"][:, self.cfg.native_left_foot, :3]\n",
+ " nat_rf_pts = preds[\"joints_3d\"][:, self.cfg.native_right_foot, :3]\n",
+ " \n",
+ " mhr_lf_pts = pred_mhr_vision[:, self.cfg.mhr_left_foot, :3]\n",
+ " mhr_rf_pts = pred_mhr_vision[:, self.cfg.mhr_right_foot, :3]\n",
+ " \n",
+ " nat_lf_bary = nat_lf_pts.mean(dim=1)\n",
+ " nat_rf_bary = nat_rf_pts.mean(dim=1)\n",
+ " mhr_lf_bary = mhr_lf_pts.mean(dim=1)\n",
+ " mhr_rf_bary = mhr_rf_pts.mean(dim=1)\n",
+ " \n",
+ " losses['loss_feet'] = (self.strict_l1(nat_lf_bary, mhr_lf_bary) + self.strict_l1(nat_rf_bary, mhr_rf_bary)) * self.cfg.w_feet\n",
+ "\n",
+ " # --- 3. DYNAMIC Native Keypoints ---\n",
+ " pred_2d = preds[\"joints_2d\"]\n",
+ " tgt_2d = targets[\"joints_2d\"]\n",
+ " losses['loss_2d_native'] = self.l1(pred_2d[..., :2], tgt_2d) * self.cfg.w_keypoints2d\n",
+ " \n",
+ " pred_3d = preds[\"joints_3d\"]\n",
+ " tgt_3d = targets[\"joints_3d\"]\n",
+ " losses['loss_3d_native'] = self.l1(pred_3d[..., :3], tgt_3d) * self.cfg.w_keypoints3d\n",
+ "\n",
+ " # ==============================================================================\n",
+ " # 4. 2D Reprojection Loss (Camera-Pose Consistency)\n",
+ " # ==============================================================================\n",
+ " # A. Native 3D Head Reprojection (All 70 Joints)\n",
+ " pred_reproj_native = self._project_3d_to_norm_2d(preds[\"joints_3d\"][..., :3], preds[\"cam_trans\"], targets)\n",
+ " losses['loss_reproj_native'] = self.strict_l1(pred_reproj_native, tgt_2d) * self.cfg.w_reproj_3d\n",
+ "\n",
+ " # B. MHR Mesh Anchors Reprojection (52 Anchors)\n",
+ " if hasattr(self.cfg, 'native_mapping_ids') and hasattr(self.cfg, 'mhr_mapping_ids'):\n",
+ " # Only project the 52 matched anchors from the MHR mesh\n",
+ " pred_mhr_anchors_vision = pred_mhr_vision[:, self.cfg.mhr_mapping_ids, :3]\n",
+ " pred_reproj_mhr = self._project_3d_to_norm_2d(pred_mhr_anchors_vision, preds[\"cam_trans\"], targets) #detach the MHR anchors to prevent gradients from flowing back through the projection during this loss calculation\n",
+ " # pred_reproj_mhr = self._project_3d_to_norm_2d(pred_mhr_anchors_vision, preds[\"cam_trans\"], targets)\n",
+ " \n",
+ " # Compare against the corresponding 52 ground truth 2D targets\n",
+ " tgt_2d_anchors = tgt_2d[:, self.cfg.native_mapping_ids, :]\n",
+ " losses['loss_reproj_mhr'] = self.strict_l1(pred_reproj_mhr, tgt_2d_anchors) * self.cfg.w_reproj_mhr\n",
+ " \n",
+ " losses['total_loss'] = sum(losses.values())\n",
+ " return losses\n",
+ "\n",
+ "# ==============================================================================\n",
+ "# SANITY CHECK: The Perfect Student\n",
+ "# ==============================================================================\n",
+ "if __name__ == \"__main__\":\n",
+ " print(\"--- Running Loss Function Sanity Check ---\")\n",
+ " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ " mhr_module = MHRForwardPass(cfg.mhr_model_path, device)\n",
+ " criterion = DistillationLoss(cfg, mhr_module)\n",
+ " \n",
+ " sample_batch_cpu = next(iter(train_loader))\n",
+ " sample_batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in sample_batch_cpu.items()}\n",
+ " \n",
+ " mock_preds = {\n",
+ " \"mhr_params\": sample_batch[\"mhr_model_params\"].clone(),\n",
+ " \"shape_params\": sample_batch[\"shape_params\"].clone(),\n",
+ " \"cam_trans\": sample_batch[\"cam_trans\"].clone(),\n",
+ " \"joints_2d\": sample_batch[\"joints_2d\"].clone(),\n",
+ " \"joints_3d\": sample_batch[\"joints_3d\"].clone()\n",
+ " }\n",
+ " \n",
+ " losses = criterion(mock_preds, sample_batch)\n",
+ " \n",
+ " print(\"Losses for the Perfect Student:\")\n",
+ " for k, v in losses.items():\n",
+ " print(f\" {k:<20}: {v.item():.6f}\")\n",
+ " \n",
+ " if abs(losses['total_loss'].item()) < 1e-4:\n",
+ " print(\"\\n✅ SUCCESS: The Perfect Student achieved a loss of 0! Math is fully verified.\")\n",
+ " else:\n",
+ " print(\"\\n❌ ERROR: Loss is not zero. Check the math.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ba2129c1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 9 — The 1-Batch Overfit Test (VRAM Optimized)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import torch.optim as optim\n",
+ "import matplotlib.pyplot as plt\n",
+ "import gc\n",
+ "\n",
+ "print(\"--- Starting 1-Batch Overfit Test ---\")\n",
+ "\n",
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+ "\n",
+ "# 1. Flush Notebook Ghost Memory\n",
+ "gc.collect()\n",
+ "torch.cuda.empty_cache()\n",
+ "\n",
+ "# 2. Instantiate Model, Loss, and Optimizer\n",
+ "overfit_model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",
+ "criterion = DistillationLoss(cfg, mhr_module)\n",
+ "\n",
+ "optimizer = optim.AdamW(overfit_model.parameters(), lr=1e-3, weight_decay=0.0)\n",
+ "\n",
+ "# Setup Automatic Mixed Precision (AMP) Scaler to save VRAM\n",
+ "scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",
+ "\n",
+ "# 3. Grab exactly ONE batch and SLICE it to fit in VRAM\n",
+ "overfit_batch_cpu = next(iter(train_loader))\n",
+ "\n",
+ "# We only need 8 images to prove the network can learn!\n",
+ "def slice_and_move(batch_dict, dev, subset_size=8):\n",
+ " return {k: v[:subset_size].to(dev) if isinstance(v, torch.Tensor) else v for k, v in batch_dict.items()}\n",
+ "\n",
+ "overfit_batch = slice_and_move(overfit_batch_cpu, device, subset_size=8)\n",
+ "\n",
+ "# 4. Training Loop (150 epochs on the exact same 8 images)\n",
+ "epochs = 1000\n",
+ "loss_history = []\n",
+ "\n",
+ "overfit_model.train()\n",
+ "for epoch in range(epochs):\n",
+ " optimizer.zero_grad(set_to_none=True) # slightly more memory efficient than zero_grad()\n",
+ " \n",
+ " # Forward pass with Mixed Precision\n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " images = overfit_batch[\"image\"]\n",
+ " cliff_cond = overfit_batch[\"cliff_cond\"]\n",
+ " \n",
+ " preds = overfit_model(images, cliff_cond)\n",
+ " \n",
+ " # Compute loss\n",
+ " losses = criterion(preds, overfit_batch)\n",
+ " total_loss = losses['total_loss']\n",
+ " \n",
+ " # Backward pass with scaled gradients\n",
+ " scaler.scale(total_loss).backward()\n",
+ " \n",
+ " # Unscale before gradient clipping\n",
+ " scaler.unscale_(optimizer)\n",
+ " torch.nn.utils.clip_grad_norm_(overfit_model.parameters(), cfg.grad_clip)\n",
+ " \n",
+ " # Optimizer step\n",
+ " scaler.step(optimizer)\n",
+ " scaler.update()\n",
+ " \n",
+ " loss_history.append(total_loss.item())\n",
+ " \n",
+ " # Print progress every 10 epochs\n",
+ " if epoch % 10 == 0 or epoch == epochs - 1:\n",
+ " print(f\"Epoch {epoch:03d} | Total Loss: {total_loss.item():.4f} | \"\n",
+ " f\"2D Loss: {losses['loss_2d_native'].item():.4f} | \"\n",
+ " f\"3D Loss: {losses['loss_3d_native'].item():.4f} | \"\n",
+ " f\"Loss pose: {losses['loss_pose'].item():.4f} | \"\n",
+ " f\"Scale Loss: {losses['loss_scale'].item():.4f} | \"\n",
+ " f\"Shape Loss: {losses['loss_shape'].item():.4f} | \"\n",
+ " f\"Cam Loss: {losses['loss_cam'].item():.4f}\"\n",
+ " f\"Mhr joints Loss: {losses['loss_mhr_joints'].item():.4f}\",\n",
+ " f\"Structural Loss: {losses['loss_structural'].item():.4f}\")\n",
+ "\n",
+ "# 5. Plot the results\n",
+ "plt.figure(figsize=(10, 5))\n",
+ "plt.plot(loss_history, label='Total Loss', color='blue', linewidth=2)\n",
+ "plt.title('1-Batch Overfit Curve (8 Images)', fontsize=14)\n",
+ "plt.xlabel('Epoch', fontsize=12)\n",
+ "plt.ylabel('Total Loss', fontsize=12)\n",
+ "plt.yscale('log') # Log scale helps visualize the exponential decay\n",
+ "plt.grid(True, which=\"both\", ls=\"--\", alpha=0.5)\n",
+ "plt.legend()\n",
+ "plt.tight_layout()\n",
+ "plt.show()\n",
+ "\n",
+ "if loss_history[-1] < loss_history[0] * 0.05:\n",
+ " print(\"\\n✅ SUCCESS: The model successfully overfit the batch! Gradients are flowing.\")\n",
+ "else:\n",
+ " print(\"\\n❌ WARNING: The loss did not decrease significantly. Check for detached tensors or learning rate issues.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "27616862",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 10 — HMR Evaluation Metrics (MPJPE & PA-MPJPE)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "\n",
+ "def batched_procrustes_alignment(pred_pts, gt_pts):\n",
+ " \"\"\"\n",
+ " Computes the optimal Similarity Transform (Scale, Rotation, Translation) \n",
+ " to align pred_pts to gt_pts using Batched SVD.\n",
+ " \"\"\"\n",
+ " B, N, D = pred_pts.shape\n",
+ " \n",
+ " # 1. Mean center the point clouds\n",
+ " mu_pred = pred_pts.mean(dim=1, keepdim=True)\n",
+ " mu_gt = gt_pts.mean(dim=1, keepdim=True)\n",
+ " \n",
+ " pred_centered = pred_pts - mu_pred\n",
+ " gt_centered = gt_pts - mu_gt\n",
+ " \n",
+ " # 2. Calculate scale (Frobenius norm)\n",
+ " norm_pred = torch.linalg.norm(pred_centered, dim=(1,2), keepdim=True)\n",
+ " norm_gt = torch.linalg.norm(gt_centered, dim=(1,2), keepdim=True)\n",
+ " \n",
+ " pred_normalized = pred_centered / torch.clamp(norm_pred, min=1e-8)\n",
+ " gt_normalized = gt_centered / torch.clamp(norm_gt, min=1e-8)\n",
+ " \n",
+ " # 3. Compute covariance matrix H (X^T Y)\n",
+ " H = torch.bmm(pred_normalized.transpose(1, 2), gt_normalized)\n",
+ " \n",
+ " # 4. Singular Value Decomposition (SVD)\n",
+ " # PyTorch returns U, S, and Vh (where Vh is V^T)\n",
+ " U, S, Vh = torch.linalg.svd(H)\n",
+ " \n",
+ " # 5. Compute Rotation matrix R (U * V^T) - FIXED!\n",
+ " R = torch.bmm(U, Vh)\n",
+ " \n",
+ " # 6. Fix reflection (Strict SO(3) rotation)\n",
+ " det = torch.linalg.det(R)\n",
+ " \n",
+ " # If det < 0, we must flip the sign of the 3rd column of U\n",
+ " # Shape of det_sign is [B, 1]\n",
+ " det_sign = torch.where(det < 0, torch.tensor(-1.0, device=pred_pts.device), torch.tensor(1.0, device=pred_pts.device)).unsqueeze(-1)\n",
+ " \n",
+ " U_fixed = U.clone()\n",
+ " U_fixed[:, :, 2] *= det_sign\n",
+ " \n",
+ " # Recompute R with the reflection fix\n",
+ " R_fixed = torch.bmm(U_fixed, Vh)\n",
+ " \n",
+ " # 7. Optimal Scale\n",
+ " # We must apply the same sign fix to the 3rd singular value!\n",
+ " S_fixed = S.clone()\n",
+ " S_fixed[:, 2] *= det_sign.squeeze(-1)\n",
+ " \n",
+ " scale = S_fixed.sum(dim=-1, keepdim=True).unsqueeze(-1) * (norm_gt / torch.clamp(norm_pred, min=1e-8))\n",
+ " \n",
+ " # 8. Apply the transformation\n",
+ " pred_aligned = scale * torch.bmm(pred_centered, R_fixed) + mu_gt\n",
+ " \n",
+ " return pred_aligned\n",
+ "\n",
+ "@torch.no_grad()\n",
+ "def evaluate_hmr_batch(preds, targets, cfg, mhr_module):\n",
+ " \"\"\"\n",
+ " Calculates MPJPE and PA-MPJPE for both the 3D Keypoints Head and the MHR Mesh.\n",
+ " Returns errors in MILLIMETERS (mm).\n",
+ " \"\"\"\n",
+ " metrics = {}\n",
+ " \n",
+ " # ---------------------------------------------------------\n",
+ " # 1. Evaluate the Native 3D Spatial Head (70 Joints)\n",
+ " # ---------------------------------------------------------\n",
+ " pred_3d_native = preds[\"joints_3d\"][..., :3]\n",
+ " gt_3d_native = targets[\"joints_3d\"][..., :3]\n",
+ " \n",
+ " # MPJPE requires root-relative alignment (usually joint 0 is the root/pelvis)\n",
+ " pred_3d_root_rel = pred_3d_native - pred_3d_native[:, 0:1, :]\n",
+ " gt_3d_root_rel = gt_3d_native - gt_3d_native[:, 0:1, :]\n",
+ " \n",
+ " # MPJPE in mm (* 1000)\n",
+ " err_native = torch.linalg.norm(pred_3d_root_rel - gt_3d_root_rel, dim=-1)\n",
+ " metrics['Head_3D_MPJPE'] = err_native.mean().item() * 1000.0\n",
+ " \n",
+ " # PA-MPJPE in mm\n",
+ " pred_3d_aligned = batched_procrustes_alignment(pred_3d_native, gt_3d_native)\n",
+ " err_native_pa = torch.linalg.norm(pred_3d_aligned - gt_3d_native, dim=-1)\n",
+ " metrics['Head_3D_PA_MPJPE'] = err_native_pa.mean().item() * 1000.0\n",
+ "\n",
+ " # ---------------------------------------------------------\n",
+ " # 2. Evaluate the MHR Mesh Head (Using the 52 Anchors)\n",
+ " # ---------------------------------------------------------\n",
+ " if hasattr(cfg, 'native_mapping_ids') and hasattr(cfg, 'mhr_mapping_ids'):\n",
+ " # Generate 3D mesh joints in cm\n",
+ " pred_mhr_raw = mhr_module.get_joints(preds[\"mhr_params\"], preds[\"shape_params\"])[..., :3]\n",
+ " \n",
+ " # Convert to Meters and align axes\n",
+ " pred_mhr_meters = pred_mhr_raw / 100.0\n",
+ " pred_mhr_meters[..., 1] = -pred_mhr_meters[..., 1]\n",
+ " pred_mhr_meters[..., 2] = -pred_mhr_meters[..., 2]\n",
+ " \n",
+ " # Extract the 52 matching anchors\n",
+ " pred_mesh_anchors = pred_mhr_meters[:, cfg.mhr_mapping_ids, :]\n",
+ " gt_mesh_anchors = gt_3d_native[:, cfg.native_mapping_ids, :]\n",
+ " \n",
+ " # Mean-Center for MPJPE (To bypass differing origin definitions between rigs)\n",
+ " pred_mesh_centered = pred_mesh_anchors - pred_mesh_anchors.mean(dim=1, keepdim=True)\n",
+ " gt_mesh_centered = gt_mesh_anchors - gt_mesh_anchors.mean(dim=1, keepdim=True)\n",
+ " \n",
+ " # Mesh MPJPE in mm\n",
+ " err_mesh = torch.linalg.norm(pred_mesh_centered - gt_mesh_centered, dim=-1)\n",
+ " metrics['Mesh_MPJPE'] = err_mesh.mean().item() * 1000.0\n",
+ " \n",
+ " # Mesh PA-MPJPE in mm (Procrustes naturally handles translation offsets!)\n",
+ " pred_mesh_aligned = batched_procrustes_alignment(pred_mesh_anchors, gt_mesh_anchors)\n",
+ " err_mesh_pa = torch.linalg.norm(pred_mesh_aligned - gt_mesh_anchors, dim=-1)\n",
+ " metrics['Mesh_PA_MPJPE'] = err_mesh_pa.mean().item() * 1000.0\n",
+ " \n",
+ " return metrics\n",
+ "\n",
+ "# # ==============================================================================\n",
+ "# # SANITY CHECK: Evaluating Your Overfit Model\n",
+ "# # ==============================================================================\n",
+ "# if __name__ == \"__main__\":\n",
+ "# print(\"--- Running Evaluation on Overfit Model ---\")\n",
+ "# overfit_model.eval()\n",
+ " \n",
+ "# with torch.no_grad():\n",
+ "# with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ "# eval_preds = overfit_model(overfit_batch[\"image\"], overfit_batch[\"cliff_cond\"])\n",
+ " \n",
+ "# # Upcast to float32 for rigorous geometric math\n",
+ "# eval_preds_fp32 = {k: v.float() for k, v in eval_preds.items()}\n",
+ " \n",
+ "# metrics = evaluate_hmr_batch(eval_preds_fp32, overfit_batch, cfg, mhr_module)\n",
+ " \n",
+ "# print(\"\\n--- Final Metrics on 1-Batch Overfit ---\")\n",
+ "# print(f\"3D Keypoint Head MPJPE: {metrics['Head_3D_MPJPE']:.2f} mm\")\n",
+ "# print(f\"3D Keypoint Head PA-MPJPE: {metrics['Head_3D_PA_MPJPE']:.2f} mm\")\n",
+ "# print(\"-\" * 40)\n",
+ "# print(f\"MHR Mesh Anchors MPJPE: {metrics['Mesh_MPJPE']:.2f} mm\")\n",
+ "# print(f\"MHR Mesh Anchors PA-MPJPE: {metrics['Mesh_PA_MPJPE']:.2f} mm\")\n",
+ " \n",
+ "# if metrics['Mesh_PA_MPJPE'] < 10.0:\n",
+ "# print(\"\\n✅ OUTSTANDING: Mesh PA-MPJPE is under 10mm! Your model has achieved sub-centimeter physical accuracy.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2ede5144",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 11 — The Full Distillation Training Loop (Phase 2/Fresh)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import torch.optim as optim\n",
+ "import torch.optim.lr_scheduler as lr_scheduler\n",
+ "import os\n",
+ "import math\n",
+ "from tqdm.notebook import tqdm\n",
+ "\n",
+ "def train_instant_hmr():\n",
+ " print(f\"--- Starting Full Training Pipeline: {cfg.backbone} ---\")\n",
+ " os.makedirs(cfg.log_dir, exist_ok=True)\n",
+ " \n",
+ " # --- CONFIGURATION ---\n",
+ " RESUME_FROM_CHECKPOINT = False # Set to False to train from scratch\n",
+ " \n",
+ " # 1. Initialize Model & Loss\n",
+ " model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",
+ " criterion = DistillationLoss(cfg, mhr_module)\n",
+ " \n",
+ " # 2. Optimizer & AMP Scaler\n",
+ " optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n",
+ " scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",
+ " \n",
+ " # --- RESUME / FRESH START LOGIC ---\n",
+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model.pth\")\n",
+ " start_epoch = 0 \n",
+ " best_val_metric = float('inf') \n",
+ " \n",
+ " if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n",
+ " print(f\"🔄 Resuming from checkpoint: {best_ckpt_path}\")\n",
+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",
+ " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False) \n",
+ " else:\n",
+ " print(\"🚀 Starting training from scratch with fresh weights.\")\n",
+ "\n",
+ " # Disable torch.compile() to prevent the LLVM pthread_join crash \n",
+ " # and to stop BatchNorm running stats from corrupting during validation.\n",
+ " # print(\"⏳ Compiling model with torch.compile()...\")\n",
+ " # model = torch.compile(model, mode=\"default\") \n",
+ "\n",
+ " # 3. Learning Rate Scheduler\n",
+ " steps_per_epoch = len(train_loader)\n",
+ " scheduler = lr_scheduler.OneCycleLR(\n",
+ " optimizer, \n",
+ " max_lr=cfg.lr, \n",
+ " epochs=cfg.epochs, \n",
+ " steps_per_epoch=steps_per_epoch,\n",
+ " pct_start=0.1\n",
+ " )\n",
+ " \n",
+ " # --- MAIN EPOCH LOOP ---\n",
+ " for epoch in range(start_epoch, cfg.epochs):\n",
+ " model.train()\n",
+ " \n",
+ " train_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
+ " valid_train_batches = 0\n",
+ " \n",
+ " pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Train]\")\n",
+ " for batch in pbar:\n",
+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " \n",
+ " optimizer.zero_grad(set_to_none=True)\n",
+ " \n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",
+ " \n",
+ " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",
+ " losses = criterion(preds_fp32, batch)\n",
+ " loss = losses['total_loss']\n",
+ " \n",
+ " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",
+ " print(\"⚠️ WARNING: NaN/Inf loss detected! Skipping step.\")\n",
+ " continue\n",
+ " \n",
+ " scaler.scale(loss).backward()\n",
+ " scaler.unscale_(optimizer)\n",
+ " torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)\n",
+ " \n",
+ " # 🛑 Only step the scheduler if the optimizer actually stepped\n",
+ " scale_before = scaler.get_scale()\n",
+ " scaler.step(optimizer)\n",
+ " scaler.update()\n",
+ " scale_after = scaler.get_scale()\n",
+ " \n",
+ " # If the scale didn't drop (meaning no NaNs were found in gradients), step the LR scheduler\n",
+ " if scale_before <= scale_after:\n",
+ " scheduler.step()\n",
+ " \n",
+ " # Accumulate metrics\n",
+ " train_metrics['total'] += loss.item()\n",
+ " train_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
+ " train_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
+ " train_metrics['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",
+ " train_metrics['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",
+ " train_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
+ " train_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
+ " valid_train_batches += 1\n",
+ " \n",
+ " pbar.set_postfix({\n",
+ " 'Tot': f\"{loss.item():.3f}\", \n",
+ " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\",\n",
+ " 'Str': f\"{losses.get('loss_structural', torch.tensor(0)).item():.3f}\"\n",
+ " })\n",
+ " \n",
+ " pbar.close() \n",
+ " avg_train = {k: v / max(valid_train_batches, 1) for k, v in train_metrics.items()}\n",
+ " \n",
+ " # --- VALIDATION LOOP ---\n",
+ " model.eval()\n",
+ " val_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
+ " \n",
+ " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",
+ " valid_val_batches = 0\n",
+ " \n",
+ " with torch.no_grad():\n",
+ " for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val]\", leave=False):\n",
+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " \n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",
+ " \n",
+ " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",
+ " losses = criterion(preds_fp32, batch)\n",
+ " loss = losses['total_loss']\n",
+ " \n",
+ " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",
+ " continue\n",
+ " \n",
+ " val_metrics['total'] += loss.item()\n",
+ " val_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
+ " val_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
+ " val_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
+ " val_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
+ " \n",
+ " batch_hmr_metrics = evaluate_hmr_batch(preds_fp32, batch, cfg, mhr_module)\n",
+ " for k in hmr_trackers.keys():\n",
+ " hmr_trackers[k] += batch_hmr_metrics.get(k, 0.0)\n",
+ " \n",
+ " valid_val_batches += 1\n",
+ " \n",
+ " avg_val = {k: v / max(valid_val_batches, 1) for k, v in val_metrics.items()}\n",
+ " avg_hmr = {k: v / max(valid_val_batches, 1) for k, v in hmr_trackers.items()}\n",
+ " \n",
+ " # --- DETAILED EPOCH SUMMARY ---\n",
+ " print(f\"\\n📈 Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}\")\n",
+ " print(f\" [Train Loss] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | Struct: {avg_train['struct']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",
+ " print(f\" [Val Loss] Tot: {avg_val['total']:.4f} | MHR: {avg_val['mhr']:.4f} | Struct: {avg_val['struct']:.4f} | 2D: {avg_val['2d']:.4f} | 3D: {avg_val['3d']:.4f}\")\n",
+ " print(f\" 📏 [Val Metrics] Mesh PA-MPJPE: {avg_hmr['Mesh_PA_MPJPE']:.1f} mm | Mesh MPJPE: {avg_hmr['Mesh_MPJPE']:.1f} mm\")\n",
+ " print(f\" 📏 [Val Metrics] Head PA-MPJPE: {avg_hmr['Head_3D_PA_MPJPE']:.1f} mm | Head MPJPE: {avg_hmr['Head_3D_MPJPE']:.1f} mm\")\n",
+ " \n",
+ " # --- CHECKPOINTING (Now based on Mesh PA-MPJPE!) ---\n",
+ " current_metric = avg_hmr['Mesh_PA_MPJPE']\n",
+ " if current_metric < best_val_metric and valid_val_batches > 0:\n",
+ " best_val_metric = current_metric\n",
+ " save_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",
+ " torch.save({\n",
+ " 'epoch': epoch,\n",
+ " 'model_state_dict': model.state_dict(),\n",
+ " 'optimizer_state_dict': optimizer.state_dict(),\n",
+ " 'val_pa_mpjpe': best_val_metric,\n",
+ " }, save_path)\n",
+ " print(f\"💾 New Best Model saved! Mesh PA-MPJPE dropped to {best_val_metric:.1f} mm.\")\n",
+ "\n",
+ " print(\"\\n🎉 Training Complete!\")\n",
+ "\n",
+ "# Kick off the training!\n",
+ "if __name__ == \"__main__\":\n",
+ " import gc\n",
+ " gc.collect()\n",
+ " torch.cuda.empty_cache()\n",
+ " \n",
+ " train_instant_hmr()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e12c985b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 13 — Validation Visualization (The Reveal!)\n",
+ "# ============================================================\n",
+ "import torch\n",
+ "import matplotlib.pyplot as plt\n",
+ "import os\n",
+ "\n",
+ "print(\"--- Running Inference Visualization ---\")\n",
+ "\n",
+ "# 1. Load the Best Saved Model\n",
+ "best_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",
+ "checkpoint_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",
+ "\n",
+ "if os.path.exists(checkpoint_path):\n",
+ " checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=True)\n",
+ " \n",
+ " # --- THE FIX: Clean the torch.compile() prefix ---\n",
+ " clean_state_dict = {}\n",
+ " for k, v in checkpoint['model_state_dict'].items():\n",
+ " # Remove '_orig_mod.' prefix if it exists\n",
+ " new_key = k.replace('_orig_mod.', '')\n",
+ " clean_state_dict[new_key] = v\n",
+ " \n",
+ " # Load the cleaned dictionary\n",
+ " best_model.load_state_dict(clean_state_dict, strict=False)\n",
+ " print(f\"✅ Loaded Best Model from Epoch {checkpoint.get('epoch', 0)+1} with PA-MPJPE: {checkpoint.get('val_pa_mpjpe', 0):.1f} mm\")\n",
+ "else:\n",
+ " print(\"⚠️ No checkpoint found! Using current model in memory.\")\n",
+ " best_model = model\n",
+ "\n",
+ "best_model.eval()\n",
+ "\n",
+ "# 2. Grab a Validation Batch\n",
+ "val_batch = next(iter(val_loader))\n",
+ "images = val_batch[\"image\"].to(device)\n",
+ "cliff_cond = val_batch[\"cliff_cond\"].to(device)\n",
+ "gt_2d = val_batch[\"joints_2d\"].cpu()\n",
+ "\n",
+ "# 3. Run Inference\n",
+ "with torch.no_grad():\n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " preds = best_model(images, cliff_cond)\n",
+ "\n",
+ "# Extract only the X,Y coordinates\n",
+ "pred_2d = preds[\"joints_2d\"][..., :2].cpu()\n",
+ "\n",
+ "# 4. Math Helper: Un-normalize coordinates from [-1, 1] back to [0, 224] pixels\n",
+ "def unnorm_coords(coords, size=224.0):\n",
+ " return ((coords + 1.0) / 2.0) * size\n",
+ "\n",
+ "gt_2d_px = unnorm_coords(gt_2d[..., :2], cfg.image_size)\n",
+ "pred_2d_px = unnorm_coords(pred_2d, cfg.image_size)\n",
+ "\n",
+ "# 5. Image Helper: Un-normalize ImageNet pixels for matplotlib\n",
+ "mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",
+ "std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",
+ "\n",
+ "# 6. Plot a 2x2 Grid of Predictions\n",
+ "fig, axes = plt.subplots(2, 2, figsize=(10, 10))\n",
+ "axes = axes.flatten()\n",
+ "\n",
+ "for i in range(4): # Plot the first 4 images in the batch\n",
+ " # Prepare image\n",
+ " img_unnorm = images[i].cpu() * std + mean\n",
+ " img_np = img_unnorm.permute(1, 2, 0).numpy().clip(0, 1)\n",
+ " \n",
+ " # Plot Image\n",
+ " axes[i].imshow(img_np)\n",
+ " \n",
+ " # Plot Ground Truth (Teacher) - Lime Green Dots\n",
+ " axes[i].scatter(gt_2d_px[i, :, 0], gt_2d_px[i, :, 1], \n",
+ " c='lime', s=25, edgecolors='black', label='Teacher (GT)' if i==0 else \"\")\n",
+ " \n",
+ " # Plot Predictions (Student) - Red Crosses\n",
+ " axes[i].scatter(pred_2d_px[i, :, 0], pred_2d_px[i, :, 1], \n",
+ " c='red', s=35, marker='X', edgecolors='black', linewidths=0.5, label='Student' if i==0 else \"\")\n",
+ " \n",
+ " axes[i].axis('off')\n",
+ " axes[i].set_title(f\"Validation Sample {i+1}\", fontsize=12)\n",
+ "\n",
+ "fig.legend(loc='upper center', ncol=2, fontsize=14, bbox_to_anchor=(0.5, 1.05))\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2cd9f6d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 14 — Inference Speed Benchmark (Fixed Loader)\n",
+ "# ============================================================\n",
+ "import time\n",
+ "import os\n",
+ "import torch\n",
+ "\n",
+ "print(\"--- Running Inference Speed Benchmark ---\")\n",
+ "\n",
+ "@torch.no_grad()\n",
+ "def benchmark_fps(model, image_size=224, batch_size=1, n_warmup=50, n_iters=200):\n",
+ " \"\"\"Benchmark model throughput on GPU.\"\"\"\n",
+ " model.eval()\n",
+ " # Dummy inputs matching our architecture\n",
+ " dummy_img = torch.randn(batch_size, 3, image_size, image_size, device=device)\n",
+ " dummy_cliff = torch.randn(batch_size, 3, device=device) # [cx_norm, cy_norm, b_scale]\n",
+ "\n",
+ " # Warmup (Wraps the forward pass in AMP for real-world float16 speed)\n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " for _ in range(n_warmup):\n",
+ " _ = model(dummy_img, dummy_cliff)\n",
+ " \n",
+ " if torch.cuda.is_available():\n",
+ " torch.cuda.synchronize()\n",
+ "\n",
+ " # Timed run\n",
+ " t0 = time.perf_counter()\n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " for _ in range(n_iters):\n",
+ " _ = model(dummy_img, dummy_cliff)\n",
+ " \n",
+ " if torch.cuda.is_available():\n",
+ " torch.cuda.synchronize()\n",
+ " \n",
+ " elapsed = time.perf_counter() - t0\n",
+ "\n",
+ " fps = n_iters * batch_size / elapsed\n",
+ " ms_per_image = elapsed / (n_iters * batch_size) * 1000\n",
+ " return fps, ms_per_image\n",
+ "\n",
+ "# 1. Instantiate the Model\n",
+ "benchmark_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",
+ "\n",
+ "# 2. Load best checkpoint with the compile fix!\n",
+ "best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",
+ "if os.path.exists(best_ckpt_path):\n",
+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",
+ " \n",
+ " # --- Clean the torch.compile() prefix ---\n",
+ " clean_state_dict = {}\n",
+ " for k, v in ckpt['model_state_dict'].items():\n",
+ " new_key = k.replace('_orig_mod.', '')\n",
+ " clean_state_dict[new_key] = v\n",
+ " \n",
+ " benchmark_model.load_state_dict(clean_state_dict, strict=False)\n",
+ " print(f\"✅ Loaded best model from epoch {ckpt.get('epoch', 0)+1} (PA-MPJPE={ckpt.get('val_pa_mpjpe', 0):.1f} mm)\")\n",
+ "else:\n",
+ " print(\"⚠️ No checkpoint found! Benchmarking with random weights.\")\n",
+ "\n",
+ "# 3. Benchmark at different batch sizes\n",
+ "fps_b1, ms_b1 = benchmark_fps(benchmark_model, batch_size=1)\n",
+ "fps_b16, ms_b16 = benchmark_fps(benchmark_model, batch_size=16)\n",
+ "fps_b64, ms_b64 = benchmark_fps(benchmark_model, batch_size=64)\n",
+ "\n",
+ "# 4. Compute total parameters\n",
+ "total_params = sum(p.numel() for p in benchmark_model.parameters())\n",
+ "\n",
+ "print(f\"\\n🚀 Inference Speed (RTX {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}):\")\n",
+ "print(f\" Batch=1: {fps_b1:.0f} FPS ({ms_b1:.1f} ms/image)\")\n",
+ "print(f\" Batch=16: {fps_b16:.0f} FPS ({ms_b16:.1f} ms/image)\")\n",
+ "print(f\" Batch=64: {fps_b64:.0f} FPS ({ms_b64:.1f} ms/image)\")\n",
+ "print(f\"\\n📊 Comparison:\")\n",
+ "print(f\" SAM3D Teacher: ~3 FPS (670M params)\")\n",
+ "print(f\" Student: {fps_b1:.0f} FPS ({total_params/1e6:.1f}M params) → {fps_b1/3:.0f}x faster\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "334859a9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 15 — Export & Static Quantization (FP32, FP16, INT8)\n",
+ "# ============================================================\n",
+ "import os\n",
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import numpy as np\n",
+ "from pathlib import Path\n",
+ "\n",
+ "print(\"--- Exporting and Quantizing Full Model for Deployment ---\")\n",
+ "\n",
+ "# --- 1. Deploy Wrapper (full outputs) ---\n",
+ "class HMRDeployWrapper(nn.Module):\n",
+ " def __init__(self, model):\n",
+ " super().__init__()\n",
+ " self.model = model\n",
+ " def forward(self, image, cliff_cond):\n",
+ " out = self.model(image, cliff_cond)\n",
+ " return (out[\"mhr_params\"], out[\"shape_params\"], out[\"cam_trans\"], \n",
+ " out[\"joints_2d\"], out[\"joints_3d\"])\n",
+ "\n",
+ "# --- 2. INT8 Calibration Data Reader ---\n",
+ "try:\n",
+ " from onnxruntime.quantization import (\n",
+ " CalibrationDataReader, quantize_static,\n",
+ " QuantType, QuantFormat, CalibrationMethod,\n",
+ " )\n",
+ " \n",
+ " class HMRCalibrationDataReader(CalibrationDataReader):\n",
+ " def __init__(self, dataloader, max_samples=100):\n",
+ " self.data_list = []\n",
+ " \n",
+ " print(f\"Extracting {max_samples} individual images for INT8 calibration...\")\n",
+ " for batch in dataloader:\n",
+ " images = batch[\"image\"].numpy().astype(np.float32)\n",
+ " cliffs = batch[\"cliff_cond\"].numpy().astype(np.float32)\n",
+ " \n",
+ " for b_idx in range(images.shape[0]):\n",
+ " if len(self.data_list) >= max_samples:\n",
+ " break\n",
+ " self.data_list.append({\n",
+ " \"image\": images[b_idx:b_idx+1],\n",
+ " \"cliff_cond\": cliffs[b_idx:b_idx+1]\n",
+ " })\n",
+ " if len(self.data_list) >= max_samples:\n",
+ " break\n",
+ " \n",
+ " self.enum_data = iter(self.data_list)\n",
+ "\n",
+ " def get_next(self):\n",
+ " return next(self.enum_data, None)\n",
+ "\n",
+ "except ImportError:\n",
+ " print(\"⚠️ 'onnxruntime' is not installed. INT8 quantization will fail.\")\n",
+ "\n",
+ "\n",
+ "def topological_sort_onnx(model):\n",
+ " \"\"\"Sort ONNX graph nodes in topological order.\n",
+ " \n",
+ " The ORT float16 converter with keep_io_types=True inserts Cast nodes at graph\n",
+ " inputs but doesn't re-sort the graph, breaking topological order. This fixes it.\n",
+ " \"\"\"\n",
+ " available = set()\n",
+ " for inp in model.graph.input:\n",
+ " available.add(inp.name)\n",
+ " for init in model.graph.initializer:\n",
+ " available.add(init.name)\n",
+ " \n",
+ " remaining = list(model.graph.node)\n",
+ " sorted_nodes = []\n",
+ " \n",
+ " while remaining:\n",
+ " progress = False\n",
+ " for node in remaining:\n",
+ " if all(inp in available or inp == '' for inp in node.input):\n",
+ " sorted_nodes.append(node)\n",
+ " remaining.remove(node)\n",
+ " for out in node.output:\n",
+ " available.add(out)\n",
+ " progress = True\n",
+ " break\n",
+ " if not progress:\n",
+ " sorted_nodes.extend(remaining)\n",
+ " break\n",
+ " \n",
+ " del model.graph.node[:]\n",
+ " model.graph.node.extend(sorted_nodes)\n",
+ " return model\n",
+ "\n",
+ "\n",
+ "# --- 3. Export & Quantize ---\n",
+ "export_dir = Path(cfg.log_dir) / \"export\"\n",
+ "export_dir.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "onnx_fp32_path = export_dir / \"sam3d_full_fp32.onnx\"\n",
+ "onnx_fp16_path = export_dir / \"sam3d_full_fp16.onnx\"\n",
+ "onnx_int8_path = export_dir / \"sam3d_full_int8.onnx\"\n",
+ "\n",
+ "output_keys = [\"mhr_params\", \"shape_params\", \"cam_trans\", \"joints_2d\", \"joints_3d\"]\n",
+ "\n",
+ "export_target_model = InstantHMRStudent(cfg, pretrained=False).to(device)\n",
+ "\n",
+ "best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",
+ "\n",
+ "if os.path.exists(best_ckpt_path):\n",
+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",
+ " clean_state_dict = {k.replace('_orig_mod.', ''): v for k, v in ckpt['model_state_dict'].items()}\n",
+ " export_target_model.load_state_dict(clean_state_dict, strict=False)\n",
+ " print(f\"✅ Loaded best model from epoch {ckpt.get('epoch', 0)+1}.\\n\")\n",
+ " \n",
+ " deploy_model = HMRDeployWrapper(export_target_model)\n",
+ " deploy_model.eval()\n",
+ " \n",
+ " dummy_img = torch.randn(1, 3, cfg.image_size, cfg.image_size, device=device)\n",
+ " dummy_cliff = torch.randn(1, 3, device=device)\n",
+ " \n",
+ " dynamic_axes = {\"image\": {0: \"batch\"}, \"cliff_cond\": {0: \"batch\"}}\n",
+ " for k in output_keys:\n",
+ " dynamic_axes[k] = {0: \"batch\"}\n",
+ "\n",
+ " # A. Export FP32\n",
+ " try:\n",
+ " import onnx\n",
+ " torch.onnx.export(\n",
+ " deploy_model, (dummy_img, dummy_cliff), str(onnx_fp32_path),\n",
+ " input_names=[\"image\", \"cliff_cond\"], output_names=output_keys,\n",
+ " dynamic_axes=dynamic_axes, opset_version=17\n",
+ " )\n",
+ " print(f\"✅ FP32 saved: {onnx_fp32_path.name} | Size: {onnx_fp32_path.stat().st_size / 1e6:.1f} MB\")\n",
+ " except Exception as e:\n",
+ " print(f\"❌ FP32 export failed: {e}\")\n",
+ "\n",
+ " # B. Export FP16 — uses ORT's converter which correctly updates Cast node `to` attributes\n",
+ " # + topological sort to fix node ordering after keep_io_types Cast insertion\n",
+ " try:\n",
+ " from onnxruntime.transformers.float16 import convert_float_to_float16\n",
+ " import warnings\n",
+ " with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " model_fp32 = onnx.load(str(onnx_fp32_path))\n",
+ " model_fp16 = convert_float_to_float16(model_fp32, keep_io_types=True)\n",
+ " model_fp16 = topological_sort_onnx(model_fp16)\n",
+ " onnx.save(model_fp16, str(onnx_fp16_path))\n",
+ " onnx.checker.check_model(onnx.load(str(onnx_fp16_path)), full_check=True)\n",
+ " print(f\"✅ FP16 saved: {onnx_fp16_path.name} | Size: {onnx_fp16_path.stat().st_size / 1e6:.1f} MB\")\n",
+ " except ImportError:\n",
+ " print(\"⚠️ FP16 skipped (onnxruntime.transformers not available)\")\n",
+ " except Exception as e:\n",
+ " print(f\"❌ FP16 conversion failed: {e}\")\n",
+ "\n",
+ " # C. Export INT8 — selective: only quantize Conv ops (backbone), skip transformer/heads\n",
+ " # Naive quantization of all ops destroys accuracy (Softmax, LayerNorm, attention MatMuls\n",
+ " # need FP32 precision) and is slower due to unfused QDQ overhead blocking NCHWc optimization.\n",
+ " try:\n",
+ " calib_reader = HMRCalibrationDataReader(train_loader, max_samples=100)\n",
+ " calib_reader.enum_data = iter(calib_reader.data_list)\n",
+ " \n",
+ " # Collect transformer + head + projection node names to exclude from quantization\n",
+ " # (defense-in-depth: op_types_to_quantize=['Conv'] already excludes them since they\n",
+ " # have zero Conv ops, but explicit exclusion prevents issues if the model changes)\n",
+ " fp32_model = onnx.load(str(onnx_fp32_path))\n",
+ " nodes_to_exclude = []\n",
+ " for node in fp32_model.graph.node:\n",
+ " name = node.name or ''\n",
+ " if any(kw in name for kw in ['/model/transformer/', '/model/head_',\n",
+ " '/model/feat_proj', '/model/cond_proj']):\n",
+ " nodes_to_exclude.append(name)\n",
+ " del fp32_model\n",
+ " print(f\" Excluding {len(nodes_to_exclude)} transformer/head nodes from quantization\")\n",
+ " \n",
+ " quantize_static(\n",
+ " model_input=str(onnx_fp32_path),\n",
+ " model_output=str(onnx_int8_path),\n",
+ " calibration_data_reader=calib_reader,\n",
+ " quant_format=QuantFormat.QDQ,\n",
+ " op_types_to_quantize=['Conv'], # Only quantize Conv (backbone)\n",
+ " per_channel=True, # Better accuracy for Conv weights\n",
+ " reduce_range=False, # Not needed with VNNI\n",
+ " activation_type=QuantType.QInt8,\n",
+ " weight_type=QuantType.QInt8,\n",
+ " nodes_to_exclude=nodes_to_exclude,\n",
+ " calibrate_method=CalibrationMethod.Percentile,\n",
+ " extra_options={\n",
+ " 'CalibTensorRangeSymmetric': True,\n",
+ " },\n",
+ " )\n",
+ " print(f\"✅ INT8 saved: {onnx_int8_path.name} | Size: {onnx_int8_path.stat().st_size / 1e6:.1f} MB\")\n",
+ " except NameError:\n",
+ " print(\"⚠️ INT8 skipped (train_loader not available)\")\n",
+ " except Exception as e:\n",
+ " import traceback\n",
+ " print(f\"❌ INT8 conversion failed: {e}\")\n",
+ " traceback.print_exc()\n",
+ "\n",
+ " # D. Save PyTorch state dict\n",
+ " sd_path = export_dir / \"sam3d_student_state_dict.pth\"\n",
+ " torch.save(export_target_model.state_dict(), sd_path)\n",
+ " print(f\"\\n✅ PyTorch state dict saved: {sd_path.name}\")\n",
+ " \n",
+ "else:\n",
+ " print(\"⚠️ No checkpoint found at:\", best_ckpt_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "11d916bf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 17 — ONNX Profiler (FP32, FP16, INT8) x (CPU, GPU)\n",
+ "# ============================================================\n",
+ "import time\n",
+ "import numpy as np\n",
+ "from pathlib import Path\n",
+ "\n",
+ "print(\"--- ONNX Hardware Profiler ---\")\n",
+ "\n",
+ "try:\n",
+ " import onnxruntime as ort\n",
+ " ONNX_AVAILABLE = True\n",
+ "except ImportError:\n",
+ " ONNX_AVAILABLE = False\n",
+ " print(\"⚠️ Missing: pip install onnxruntime-gpu\")\n",
+ "\n",
+ "if ONNX_AVAILABLE:\n",
+ " def profile_onnx_model(onnx_path, provider, batch_size=1, warmup_iters=20, benchmark_iters=100):\n",
+ " \"\"\"Profile latency for a specific ONNX model and execution provider.\"\"\"\n",
+ " \n",
+ " sess_options = ort.SessionOptions()\n",
+ " sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\n",
+ " sess_options.log_severity_level = 3\n",
+ " \n",
+ " try:\n",
+ " session = ort.InferenceSession(str(onnx_path), sess_options=sess_options, providers=[provider])\n",
+ " except Exception as e:\n",
+ " return None, None, f\"Error: {e}\"\n",
+ " \n",
+ " active_provider = session.get_providers()[0]\n",
+ " if active_provider != provider:\n",
+ " return None, None, f\"Fallback! Requested {provider} but got {active_provider}\"\n",
+ "\n",
+ " ort_inputs = {}\n",
+ " dtype_map = {'tensor(float)': np.float32, 'tensor(float16)': np.float16}\n",
+ " \n",
+ " for inp in session.get_inputs():\n",
+ " shape = [batch_size if isinstance(dim, str) else dim for dim in inp.shape]\n",
+ " np_dtype = dtype_map.get(inp.type, np.float32)\n",
+ " ort_inputs[inp.name] = np.random.randn(*shape).astype(np_dtype)\n",
+ "\n",
+ " for _ in range(warmup_iters):\n",
+ " session.run(None, ort_inputs)\n",
+ "\n",
+ " t0 = time.perf_counter()\n",
+ " for _ in range(benchmark_iters):\n",
+ " session.run(None, ort_inputs)\n",
+ " t1 = time.perf_counter()\n",
+ "\n",
+ " total_time = t1 - t0\n",
+ " avg_ms = (total_time / benchmark_iters) * 1000.0\n",
+ " fps = (benchmark_iters * batch_size) / total_time\n",
+ " return avg_ms, fps, \"OK\"\n",
+ "\n",
+ " # --- Setup ---\n",
+ " export_dir = Path(cfg.log_dir) / \"export\"\n",
+ " precisions = [\"fp32\", \"fp16\", \"int8\"]\n",
+ "\n",
+ " available_providers = ort.get_available_providers()\n",
+ " providers_to_test = ['CPUExecutionProvider']\n",
+ " if 'CUDAExecutionProvider' in available_providers:\n",
+ " providers_to_test.append('CUDAExecutionProvider')\n",
+ " else:\n",
+ " print(\"⚠️ CUDAExecutionProvider not found. Profiling CPU only.\")\n",
+ " print(\" Fix: pip uninstall onnxruntime onnxruntime-gpu -y && pip install onnxruntime-gpu\")\n",
+ "\n",
+ " print(f\"\\n{'Precision':<6} | {'Size':<7} | {'HW':<4} | {'Latency':<12} | {'FPS':<8} | Notes\")\n",
+ " print(\"-\" * 75)\n",
+ "\n",
+ " for prec in precisions:\n",
+ " model_path = export_dir / f\"sam3d_full_{prec}.onnx\"\n",
+ " \n",
+ " if not model_path.exists():\n",
+ " print(f\"{prec:<6} | --- | --- | {'Not Found':<12} | ---\")\n",
+ " continue\n",
+ " \n",
+ " size_mb = f\"{model_path.stat().st_size / 1e6:.1f}MB\"\n",
+ " \n",
+ " for provider in providers_to_test:\n",
+ " hw_tag = \"GPU\" if \"CUDA\" in provider else \"CPU\"\n",
+ " \n",
+ " # FP16 on CPU: not supported (no native x86 FP16 compute)\n",
+ " if prec == \"fp16\" and provider == \"CPUExecutionProvider\":\n",
+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {'N/A':<12} | --- | x86 has no FP16 ALU\")\n",
+ " continue\n",
+ " \n",
+ " note = \"\"\n",
+ " if prec == \"fp16\" and \"CUDA\" in provider:\n",
+ " note = \"Best for GPU deploy\"\n",
+ " elif prec == \"int8\" and \"CPU\" in provider:\n",
+ " note = \"Conv-only quant, for edge/NPU\"\n",
+ " elif prec == \"int8\" and \"CUDA\" in provider:\n",
+ " note = \"Needs TensorRT EP for gains\"\n",
+ " \n",
+ " ms, fps, status = profile_onnx_model(model_path, provider)\n",
+ " \n",
+ " if ms is not None:\n",
+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {ms:>7.2f} ms | {fps:>6.1f} | {note}\")\n",
+ " else:\n",
+ " print(f\"{prec:<6} | {size_mb:<7} | {hw_tag:<4} | {'Failed':<12} | --- | {status}\")\n",
+ "\n",
+ " print(\"-\" * 75)\n",
+ " print(\"\\nDeployment guide:\")\n",
+ " print(\" GPU (CUDA EP) → FP16 (best latency + half memory)\")\n",
+ " print(\" GPU (TensorRT EP) → INT8 (if TRT available, even faster)\")\n",
+ " print(\" CPU (x86 VNNI) → FP32 or INT8 (Conv-only quant, test both)\")\n",
+ " print(\" Mobile NPU → INT8 with NNAPI/QNN EP (native INT8 HW)\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eikwijqvez7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ============================================================\n",
+ "# Cell 18 — ONNX Accuracy Comparison (FP32 vs FP16 vs INT8)\n",
+ "# ============================================================\n",
+ "import numpy as np\n",
+ "import torch\n",
+ "from pathlib import Path\n",
+ "from tqdm.notebook import tqdm\n",
+ "\n",
+ "print(\"--- ONNX Quantization Accuracy Comparison ---\")\n",
+ "\n",
+ "try:\n",
+ " import onnxruntime as ort\n",
+ " ONNX_AVAILABLE = True\n",
+ "except ImportError:\n",
+ " ONNX_AVAILABLE = False\n",
+ " print(\"⚠️ Missing: pip install onnxruntime-gpu\")\n",
+ "\n",
+ "if ONNX_AVAILABLE:\n",
+ " export_dir = Path(cfg.log_dir) / \"export\"\n",
+ " output_names = [\"mhr_params\", \"shape_params\", \"cam_trans\", \"joints_2d\", \"joints_3d\"]\n",
+ " \n",
+ " models_to_eval = {\n",
+ " \"FP32\": export_dir / \"sam3d_full_fp32.onnx\",\n",
+ " \"FP16\": export_dir / \"sam3d_full_fp16.onnx\",\n",
+ " \"INT8\": export_dir / \"sam3d_full_int8.onnx\",\n",
+ " }\n",
+ " \n",
+ " # Filter to existing models only\n",
+ " models_to_eval = {k: v for k, v in models_to_eval.items() if v.exists()}\n",
+ " print(f\"Found models: {', '.join(models_to_eval.keys())}\")\n",
+ " \n",
+ " # Use GPU if available, otherwise CPU (FP16 needs GPU)\n",
+ " available_providers = ort.get_available_providers()\n",
+ " providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if 'CUDAExecutionProvider' in available_providers else ['CPUExecutionProvider']\n",
+ " \n",
+ " results = {}\n",
+ " \n",
+ " for model_name, model_path in models_to_eval.items():\n",
+ " print(f\"\\nEvaluating {model_name} ({model_path.stat().st_size / 1e6:.1f} MB)...\")\n",
+ " \n",
+ " sess_options = ort.SessionOptions()\n",
+ " sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL\n",
+ " sess_options.log_severity_level = 3\n",
+ " \n",
+ " try:\n",
+ " session = ort.InferenceSession(str(model_path), sess_options=sess_options, providers=providers)\n",
+ " except Exception as e:\n",
+ " print(f\" ❌ Failed to load: {e}\")\n",
+ " continue\n",
+ " \n",
+ " active_provider = session.get_providers()[0]\n",
+ " print(f\" Running on: {active_provider}\")\n",
+ " \n",
+ " # Accumulate metrics across all val batches\n",
+ " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",
+ " valid_batches = 0\n",
+ " \n",
+ " for batch in tqdm(val_loader, desc=f\" {model_name}\", leave=False):\n",
+ " images_np = batch[\"image\"].numpy().astype(np.float32)\n",
+ " cliff_np = batch[\"cliff_cond\"].numpy().astype(np.float32)\n",
+ " \n",
+ " ort_inputs = {\"image\": images_np, \"cliff_cond\": cliff_np}\n",
+ " ort_outputs = session.run(None, ort_inputs)\n",
+ " \n",
+ " # Convert ONNX outputs to PyTorch tensor dict\n",
+ " preds = {}\n",
+ " for name, arr in zip(output_names, ort_outputs):\n",
+ " preds[name] = torch.from_numpy(arr.astype(np.float32)).to(device)\n",
+ " \n",
+ " # Move GT targets to device\n",
+ " targets = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " \n",
+ " batch_metrics = evaluate_hmr_batch(preds, targets, cfg, mhr_module)\n",
+ " for k in hmr_trackers:\n",
+ " hmr_trackers[k] += batch_metrics.get(k, 0.0)\n",
+ " valid_batches += 1\n",
+ " \n",
+ " avg_metrics = {k: v / max(valid_batches, 1) for k, v in hmr_trackers.items()}\n",
+ " results[model_name] = avg_metrics\n",
+ " print(f\" ✅ {valid_batches} batches evaluated\")\n",
+ " \n",
+ " # --- Print Comparison Table ---\n",
+ " print(f\"\\n{'='*72}\")\n",
+ " print(f\"{'Metric':<22} |\", end=\"\")\n",
+ " for name in results:\n",
+ " print(f\" {name:>10}\", end=\" |\")\n",
+ " \n",
+ " # Show degradation vs FP32\n",
+ " if \"FP32\" in results and len(results) > 1:\n",
+ " print(f\" {'vs FP32':>10}\", end=\"\")\n",
+ " print()\n",
+ " print(\"-\" * 72)\n",
+ " \n",
+ " metric_labels = {\n",
+ " 'Head_3D_MPJPE': 'Head 3D MPJPE (mm)',\n",
+ " 'Head_3D_PA_MPJPE': 'Head 3D PA-MPJPE (mm)',\n",
+ " 'Mesh_MPJPE': 'Mesh MPJPE (mm)',\n",
+ " 'Mesh_PA_MPJPE': 'Mesh PA-MPJPE (mm)',\n",
+ " }\n",
+ " \n",
+ " for key, label in metric_labels.items():\n",
+ " print(f\"{label:<22} |\", end=\"\")\n",
+ " for name in results:\n",
+ " print(f\" {results[name][key]:>10.2f}\", end=\" |\")\n",
+ " \n",
+ " # Show delta vs FP32 for non-FP32 models\n",
+ " if \"FP32\" in results and len(results) > 1:\n",
+ " deltas = []\n",
+ " for name in results:\n",
+ " if name != \"FP32\":\n",
+ " delta = results[name][key] - results[\"FP32\"][key]\n",
+ " deltas.append(f\"{delta:+.2f}\")\n",
+ " print(f\" {', '.join(deltas):>10}\", end=\"\")\n",
+ " print()\n",
+ " \n",
+ " print(f\"{'='*72}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "sam3d_video_312",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..5bb1471
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,24 @@
+# ----- inference + viz (required by demo.py) -----
+numpy>=1.24
+opencv-python>=4.8
+pillow>=10
+onnxruntime-gpu>=1.19 ; platform_system != "Darwin"
+onnxruntime>=1.19 ; platform_system == "Darwin"
+rfdetr>=1.0
+rerun-sdk>=0.21
+
+# ----- CUDA runtime libs for onnxruntime-gpu -----
+# onnxruntime-gpu wheels are built against CUDA 12 + cuDNN 9 but do NOT
+# bundle them. These pip packages provide the .so files so you don't need
+# a system CUDA install. Instaharm calls onnxruntime.preload_dlls() at
+# session creation, which finds them automatically (ORT >= 1.19).
+nvidia-cudnn-cu12>=9.0,<10 ; platform_system != "Darwin"
+nvidia-cuda-runtime-cu12 ; platform_system != "Darwin"
+nvidia-cublas-cu12 ; platform_system != "Darwin"
+
+# ----- training / annotation (optional, for reproduction) -----
+# torch>=2.2
+# timm>=1.0
+# tqdm
+# loguru
+# (For annotation: install Meta's sam-3d-body separately — see docs/annotation.md)
diff --git a/tools/annotate_dataset.py b/tools/annotate_dataset.py
new file mode 100755
index 0000000..7298948
--- /dev/null
+++ b/tools/annotate_dataset.py
@@ -0,0 +1,876 @@
+#!/usr/bin/env python3
+"""
+Cloud-grade annotation pipeline for CLIFF-style student distillation.
+
+Scans a root directory tree for images, runs YOLO person detection + SAM3D
+teacher inference, and saves tight 224×224 crops alongside full-frame-aware
+annotations (.npz) ready for CLIFF-style training.
+
+Output layout:
+ <output_root>/
+ images/ <datasetName>_<imageStem>_p<personID>.jpg (224×224 crop)
+ annotations/ <datasetName>_<imageStem>_p<personID>.npz
+
+NPZ payload (per crop):
+ orig_shape (2,) [H, W] of the full uncropped image
+ bbox (4,) [x1, y1, x2, y2] raw YOLO detection in original image coords
+ (used by CLIFF student for bbox_center / bbox_scale conditioning)
+ bbox_square (4,) [x1, y1, x2, y2] expanded square crop (1.2×) used for 224×224 resize
+ cam_focal_length (2,) [fx, fy] where f = sqrt(H² + W²)
+ cam_trans (3,) [tx, ty, tz] from SAM3D (computed with raw bbox, full-frame perspective)
+ mhr_model_params (204,) pose + scale params
+ shape_params (45,) identity blendshapes
+ joints_3d (70, 3) regressed 3D joints (Y-down camera convention, meters)
+ joints_2d (70, 2) projected into ORIGINAL full-frame pixel space
+
+Resume-safe: existing annotations are skipped automatically.
+
+Usage:
+ python tools/annotate_dataset_sam3d_cloud.py \\
+ --root_dir /data/training_images \\
+ --output_dir /data/cliff_annotations \\
+ --confidence 0.7
+
+ # Quick diverse subset (50k crops sampled across all datasets):
+ python tools/annotate_dataset_sam3d_cloud.py \\
+ --root_dir /data/training_images \\
+ --output_dir /data/cliff_annotations \\
+ --shuffle --max_samples 50000
+
+ # Structure of --root_dir:
+ # /data/training_images/
+ # Harmony4D/ *.jpg *.png ...
+ # COCO/ *.jpg ...
+ # Synthetic/ *.png ...
+"""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import math
+import os
+import random
+import signal
+import sys
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+# Ensure project root is importable when running from repo root.
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(_PROJECT_ROOT))
+
+import cv2
+import numpy as np
+import torch
+from loguru import logger
+from tqdm import tqdm
+
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
+CROP_SIZE = 224
+
+
+# ---------------------------------------------------------------------------
+# Rerun helpers
+# ---------------------------------------------------------------------------
+def setup_rerun_blueprint():
+ """Send a 3-panel Rerun blueprint: raw image | overlay | 3D mesh."""
+ import rerun as rr
+ import rerun.blueprint as rrb
+
+ blueprint = rrb.Blueprint(
+ rrb.Horizontal(
+ rrb.Spatial2DView(
+ origin="raw_image",
+ name="Raw Image",
+ ),
+ rrb.Spatial2DView(
+ origin="overlay/image",
+ name="Overlay",
+ ),
+ rrb.Spatial3DView(
+ origin="scene_3d",
+ name="3D Mesh",
+ background=rrb.Background(color=[25, 25, 25]),
+ ),
+ ),
+ )
+ rr.send_blueprint(blueprint)
+
+
+_MAX_RERUN_PERSONS = 10 # upper bound for clearing stale entities
+
+
+def log_to_rerun(
+ image_rgb: np.ndarray,
+ detections: list[dict],
+ sam3d_outputs: list[dict | None],
+ faces: np.ndarray | None,
+) -> None:
+ """Log one image with all its person detections to the Rerun viewer.
+
+ All entities are logged as static so Rerun only ever holds one frame in
+ memory — previous data is overwritten, not accumulated.
+
+ Args:
+ image_rgb: Full uncropped image (H, W, 3), uint8 RGB.
+ detections: YOLO detection dicts with ``"bbox"`` keys.
+ sam3d_outputs: Parallel list of raw SAM3D output dicts (may contain None).
+ faces: Mesh triangle indices from the SAM3D teacher.
+ """
+ import rerun as rr
+
+ h, w = image_rgb.shape[:2]
+
+ # Clear all person entities from the previous frame to avoid stale meshes
+ for i in range(_MAX_RERUN_PERSONS):
+ rr.log(f"scene_3d/person_{i}", rr.Clear(recursive=True), static=True)
+ rr.log("scene_3d/camera", rr.Clear(recursive=True), static=True)
+
+ # Panel 1: raw image
+ rr.log("raw_image", rr.Image(image_rgb), static=True)
+
+ overlay = image_rgb.copy()
+ first_cam_t = None
+ first_fx, first_fy = None, None
+ first_joints_cam = None
+
+ for person_id, (det, sam3d_out) in enumerate(zip(detections, sam3d_outputs)):
+ if sam3d_out is None:
+ continue
+
+ # Extract outputs
+ cam_t = sam3d_out.get("pred_cam_t")
+ if cam_t is not None and hasattr(cam_t, "cpu"):
+ cam_t = cam_t.cpu().numpy()
+ cam_t = np.asarray(cam_t, dtype=np.float32).ravel()[:3] if cam_t is not None else np.zeros(3, dtype=np.float32)
+
+ focal = sam3d_out.get("focal_length")
+ if focal is not None and hasattr(focal, "cpu"):
+ focal = focal.cpu().numpy()
+ if focal is not None:
+ focal = np.asarray(focal, dtype=np.float32).ravel()
+ fx = float(focal[0])
+ fy = float(focal[-1]) # handles both (1,) and (2,)
+ else:
+ f = math.sqrt(h * h + w * w)
+ fx, fy = f, f
+ cx, cy = w / 2.0, h / 2.0
+
+ if first_cam_t is None:
+ first_cam_t = cam_t
+ first_fx, first_fy = fx, fy
+
+ # Vertices
+ verts = sam3d_out.get("pred_vertices", sam3d_out.get("vertices", sam3d_out.get("verts")))
+ if verts is not None:
+ if hasattr(verts, "cpu"):
+ verts = verts.cpu().numpy()
+ verts = np.asarray(verts, dtype=np.float32)
+ if verts.ndim == 3 and verts.shape[0] == 1:
+ verts = verts[0]
+
+ verts_cam = verts + cam_t
+
+ # Panel 2: project mesh onto overlay
+ z = verts_cam[:, 2]
+ valid = z > 0.01
+ u = (fx * verts_cam[valid, 0] / z[valid] + cx).astype(int)
+ v = (fy * verts_cam[valid, 1] / z[valid] + cy).astype(int)
+ mask = (u >= 0) & (u < w) & (v >= 0) & (v < h)
+ for px, py in zip(u[mask], v[mask]):
+ cv2.circle(overlay, (px, py), 1, (0, 255, 128), -1)
+
+ # Panel 3: 3D mesh
+ person_path = f"scene_3d/person_{person_id}"
+ if faces is not None:
+ rr.log(
+ f"{person_path}/mesh",
+ rr.Mesh3D(
+ vertex_positions=verts_cam,
+ triangle_indices=faces,
+ albedo_factor=[180, 200, 220, 200],
+ ),
+ static=True,
+ )
+ else:
+ rr.log(
+ f"{person_path}/mesh",
+ rr.Points3D(positions=verts_cam, radii=0.003, colors=[180, 200, 220]),
+ static=True,
+ )
+
+ # 3D joints
+ joints_3d = sam3d_out.get("pred_keypoints_3d")
+ if joints_3d is not None:
+ if hasattr(joints_3d, "cpu"):
+ joints_3d = joints_3d.cpu().numpy()
+ joints_3d = np.asarray(joints_3d, dtype=np.float32)
+ if joints_3d.ndim == 3 and joints_3d.shape[0] == 1:
+ joints_3d = joints_3d[0]
+ joints_cam = joints_3d + cam_t
+ if first_joints_cam is None:
+ first_joints_cam = joints_cam
+ rr.log(
+ f"scene_3d/person_{person_id}/joints",
+ rr.Points3D(positions=joints_cam, radii=0.008, colors=[0, 255, 0]),
+ static=True,
+ )
+
+ rr.log("overlay/image", rr.Image(overlay), static=True)
+
+ # Log Pinhole camera in the 3D scene so the image appears as a properly
+ # sized frustum. Resolution must match the full frame (not the 224 crop)
+ # because cam_focal_length is computed from sqrt(H² + W²).
+ if first_cam_t is not None:
+ mesh_depth = float(first_joints_cam[:, 2].mean()) if first_joints_cam is not None else None
+ image_plane_dist = max(mesh_depth * 0.25, 0.2) if mesh_depth else 0.5
+
+ rr.log(
+ "scene_3d/camera",
+ rr.Pinhole(
+ resolution=[w, h],
+ focal_length=[first_fx, first_fy],
+ principal_point=[w / 2.0, h / 2.0],
+ image_plane_distance=image_plane_dist,
+ ),
+ static=True,
+ )
+ rr.log("scene_3d/camera/image", rr.Image(image_rgb), static=True)
+
+
+# ---------------------------------------------------------------------------
+# Timeout helper (POSIX — uses SIGALRM)
+# ---------------------------------------------------------------------------
+class InferenceTimeout(Exception):
+ pass
+
+
+@contextmanager
+def time_limit(seconds: float):
+ """Raise InferenceTimeout if the block takes longer than *seconds*."""
+ def _handler(signum, frame):
+ raise InferenceTimeout(f"Inference exceeded {seconds}s timeout")
+
+ old = signal.signal(signal.SIGALRM, _handler)
+ signal.setitimer(signal.ITIMER_REAL, seconds)
+ try:
+ yield
+ finally:
+ signal.setitimer(signal.ITIMER_REAL, 0)
+ signal.signal(signal.SIGALRM, old)
+
+
+# ---------------------------------------------------------------------------
+# Image discovery
+# ---------------------------------------------------------------------------
+def discover_images(root_dir: Path) -> list[tuple[str, Path]]:
+ """Walk *root_dir* and return ``(dataset_name, image_path)`` pairs.
+
+ ``dataset_name`` is the name of the first-level subdirectory under
+ *root_dir*. Images directly in *root_dir* get dataset_name ``"root"``.
+ """
+ entries: list[tuple[str, Path]] = []
+ root_dir = root_dir.resolve()
+
+ for path in sorted(root_dir.rglob("*")):
+ if not path.is_file():
+ continue
+ if path.suffix.lower() not in IMAGE_EXTENSIONS:
+ continue
+
+ # Derive dataset name from the first child directory
+ try:
+ relative = path.relative_to(root_dir)
+ parts = relative.parts
+ dataset_name = parts[0] if len(parts) > 1 else "root"
+ except ValueError:
+ dataset_name = "root"
+
+ entries.append((dataset_name, path))
+
+ return entries
+
+
+# ---------------------------------------------------------------------------
+# YOLO detector (standalone, no pipeline dependency)
+# ---------------------------------------------------------------------------
+class YOLOPersonDetector:
+ """Thin wrapper around ultralytics YOLO for person detection."""
+
+ def __init__(
+ self,
+ model_name: str = "yolo11n.pt",
+ device: str = "cuda:0",
+ confidence: float = 0.7,
+ ):
+ from ultralytics import YOLO
+
+ self._model = YOLO(model_name)
+ self._device = device
+ self._confidence = confidence
+
+ def detect(self, image_rgb: np.ndarray) -> list[dict]:
+ """Return list of ``{"bbox": [x1,y1,x2,y2], "confidence": float}``."""
+ results = self._model(
+ image_rgb,
+ device=self._device,
+ classes=[0], # COCO person
+ conf=self._confidence,
+ verbose=False,
+ )
+ detections = []
+ for r in results:
+ if r.boxes is None:
+ continue
+ for box, conf in zip(
+ r.boxes.xyxy.cpu().numpy(),
+ r.boxes.conf.cpu().numpy(),
+ ):
+ detections.append({
+ "bbox": box.astype(np.float32),
+ "confidence": float(conf),
+ })
+ detections.sort(key=lambda d: d["confidence"], reverse=True)
+ return detections
+
+
+# ---------------------------------------------------------------------------
+# SAM3D teacher (uses the estimator directly for max control)
+# ---------------------------------------------------------------------------
+class SAM3DTeacher:
+ """Loads the SAM3D teacher and exposes a simple per-crop inference API.
+
+ We call the ``SAM3DBodyEstimator`` directly (not via the pipeline wrapper)
+ so that we have access to the raw output dict with ``pred_cam_t``,
+ ``focal_length``, ``mhr_model_params``, etc.
+ """
+
+ def __init__(
+ self,
+ checkpoint_path: str,
+ mhr_path: str,
+ hf_repo_id: str = "facebook/sam-3d-body-dinov3",
+ device: str = "cuda:0",
+ ):
+ self.device = device
+
+ # Add sam-3d-body to path
+ for candidate in [
+ Path.cwd() / "sam-3d-body",
+ Path.cwd().parent / "sam-3d-body",
+ Path.home() / "sam-3d-body",
+ ]:
+ if candidate.exists() and (candidate / "sam_3d_body").exists():
+ if str(candidate) not in sys.path:
+ sys.path.insert(0, str(candidate))
+ break
+
+ from sam_3d_body import SAM3DBodyEstimator
+
+ ckpt = Path(checkpoint_path)
+ if ckpt.exists():
+ from sam_3d_body import load_sam_3d_body
+ logger.info(f"Loading SAM3D from local checkpoint: {ckpt}")
+ model, cfg = load_sam_3d_body(
+ checkpoint_path=str(ckpt),
+ mhr_path=mhr_path,
+ )
+ else:
+ from sam_3d_body import load_sam_3d_body_hf
+ logger.info(f"Loading SAM3D from HuggingFace: {hf_repo_id}")
+ model, cfg = load_sam_3d_body_hf(hf_repo_id=hf_repo_id)
+
+ model = model.to(device).eval()
+ self._estimator = SAM3DBodyEstimator(model, cfg)
+ self._faces = self._estimator.faces
+
+ logger.info(f"SAM3D teacher ready on {device}")
+
+ @property
+ def faces(self) -> np.ndarray:
+ return self._faces
+
+ @torch.inference_mode()
+ def predict_person(
+ self,
+ full_image_rgb: np.ndarray,
+ bbox: np.ndarray,
+ ) -> dict | None:
+ """Run SAM3D on one person in *full_image_rgb*.
+
+ We pass the **full image** with the bbox in full-frame coordinates.
+ SAM3D's CLIFF camera head then uses the bbox position, bbox scale,
+ and full-image dimensions to compute ``pred_cam_t`` and
+ ``focal_length`` natively in full-frame perspective space — no
+ manual coordinate conversion needed.
+
+ Args:
+ full_image_rgb: The original uncropped image (H, W, 3), uint8 RGB.
+ bbox: ``[x1, y1, x2, y2]`` in full-image pixel coords.
+
+ Returns:
+ Raw SAM3D output dict for the single person, or ``None`` on
+ failure.
+ """
+ bbox_arr = np.array(bbox, dtype=np.float32).reshape(1, 4)
+
+ output_list = self._estimator.process_one_image(
+ full_image_rgb,
+ bboxes=bbox_arr,
+ inference_type="body",
+ )
+
+ if not output_list:
+ return None
+
+ return output_list[0]
+
+
+# ---------------------------------------------------------------------------
+# Crop & annotation writer
+# ---------------------------------------------------------------------------
+def make_annotation_name(
+ dataset_name: str,
+ image_stem: str,
+ person_id: int,
+) -> str:
+ """Deterministic filename: ``<dataset>_<stem>_p<id>``."""
+ # Sanitise: replace path separators / spaces that may leak from subfolders
+ safe_stem = image_stem.replace(os.sep, "_").replace(" ", "_")
+ return f"{dataset_name}_{safe_stem}_p{person_id}"
+
+
+def get_square_crop_padded(image: np.ndarray, bbox: np.ndarray, expand: float = 1.2) -> tuple[np.ndarray, np.ndarray]:
+ """
+ Extracts a perfectly square crop from the image, padding with black pixels
+ if the crop goes outside the image boundaries.
+ """
+ h, w = image.shape[:2]
+ x1, y1, x2, y2 = bbox.astype(float)
+
+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
+ size = max(x2 - x1, y2 - y1) * expand
+ half = size / 2.0
+
+ # The perfect square coordinates (may fall outside the image)
+ sq_x1, sq_y1 = int(cx - half), int(cy - half)
+ sq_x2, sq_y2 = int(cx + half), int(cy + half)
+
+ # Calculate padding needed if the square goes out of bounds
+ pad_top = max(0, -sq_y1)
+ pad_bottom = max(0, sq_y2 - h)
+ pad_left = max(0, -sq_x1)
+ pad_right = max(0, sq_x2 - w)
+
+ # Extract the valid portion of the image
+ valid_x1, valid_y1 = max(0, sq_x1), max(0, sq_y1)
+ valid_x2, valid_y2 = min(w, sq_x2), min(h, sq_y2)
+ crop_valid = image[valid_y1:valid_y2, valid_x1:valid_x2]
+
+ # Pad the valid crop to make it perfectly square
+ crop_square = cv2.copyMakeBorder(
+ crop_valid, pad_top, pad_bottom, pad_left, pad_right,
+ cv2.BORDER_CONSTANT, value=(0, 0, 0)
+ )
+
+ sq_bbox = np.array([sq_x1, sq_y1, sq_x2, sq_y2], dtype=np.float32)
+ return crop_square, sq_bbox
+
+
+def save_crop_and_annotation(
+ full_image_rgb: np.ndarray,
+ bbox: np.ndarray,
+ sam3d_output: dict,
+ name: str,
+ images_dir: Path,
+ annotations_dir: Path,
+) -> None:
+ """Resize the person crop to 224x224, save as JPEG, and write the .npz."""
+ h, w = full_image_rgb.shape[:2]
+
+ # Extract perfect square crop (padded if out of bounds)
+ crop_square, sq_bbox = get_square_crop_padded(full_image_rgb, bbox, expand=1.2)
+
+ # Resize to 224x224 and save as lossless PNG (RGB -> BGR for cv2)
+ crop_224 = cv2.resize(crop_square, (CROP_SIZE, CROP_SIZE), interpolation=cv2.INTER_LINEAR)
+ crop_bgr = cv2.cvtColor(crop_224, cv2.COLOR_RGB2BGR)
+ cv2.imwrite(str(images_dir / f"{name}.png"), crop_bgr)
+
+ # ... (The rest of Claude's function below this line stays exactly the same!) ...
+ f = math.sqrt(h * h + w * w)
+
+ # Extract SAM3D outputs
+ cam_trans = sam3d_output.get("pred_cam_t")
+ if cam_trans is not None and hasattr(cam_trans, "numpy"):
+ cam_trans = cam_trans.numpy()
+ cam_trans = np.asarray(cam_trans, dtype=np.float32).ravel()[:3]
+
+ focal = sam3d_output.get("focal_length")
+ if focal is not None and hasattr(focal, "numpy"):
+ focal = focal.numpy()
+ if focal is not None:
+ focal = np.asarray(focal, dtype=np.float32).ravel()
+ if focal.size == 1:
+ focal = np.array([focal[0], focal[0]], dtype=np.float32)
+ else:
+ focal = np.array([f, f], dtype=np.float32)
+
+ joints_3d = _to_np(sam3d_output.get("pred_keypoints_3d")) # (70, 3)
+ joints_2d = _to_np(sam3d_output.get("pred_keypoints_2d")) # (70, 2)
+ mhr_model_params = _to_np(sam3d_output.get("mhr_model_params")) # (204,)
+ shape_params = _to_np(sam3d_output.get("shape_params")) # (45,)
+
+ # Build the annotation payload
+ data: dict[str, np.ndarray] = {
+ "orig_shape": np.array([h, w], dtype=np.int32),
+ "bbox": bbox.astype(np.float32), # raw YOLO detection — CLIFF student needs this
+ "bbox_square": sq_bbox, # expanded square crop used for the 224×224 resize
+ "cam_focal_length": focal.astype(np.float32),
+ "cam_trans": cam_trans,
+ }
+ if mhr_model_params is not None:
+ data["mhr_model_params"] = mhr_model_params.astype(np.float32)
+ if shape_params is not None:
+ data["shape_params"] = shape_params.astype(np.float32)
+ if joints_3d is not None:
+ data["joints_3d"] = joints_3d.astype(np.float32)
+ if joints_2d is not None:
+ data["joints_2d"] = joints_2d.astype(np.float32)
+
+ np.savez(annotations_dir / f"{name}.npz", **data)
+
+
+def _to_np(x) -> np.ndarray | None:
+ """Convert torch.Tensor / np.ndarray to a squeezed numpy array."""
+ if x is None:
+ return None
+ if hasattr(x, "cpu"):
+ x = x.cpu().numpy()
+ x = np.asarray(x, dtype=np.float32)
+ # Squeeze leading batch dim if present: (1, ...) → (...)
+ if x.ndim >= 2 and x.shape[0] == 1:
+ x = x[0]
+ return x
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+def parse_args() -> argparse.Namespace:
+ p = argparse.ArgumentParser(
+ description="Cloud-grade CLIFF annotation pipeline (YOLO + SAM3D)",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=__doc__,
+ )
+
+ # I/O
+ p.add_argument(
+ "--root_dir", type=str, required=True,
+ help="Root directory containing dataset subdirectories of images.",
+ )
+ p.add_argument(
+ "--output_dir", type=str, required=True,
+ help="Output directory (images/ and annotations/ created inside).",
+ )
+
+ # Detection
+ p.add_argument(
+ "--yolo_model", type=str, default="yolo11n.pt",
+ help="YOLO model name or path (default: yolo11n.pt).",
+ )
+ p.add_argument(
+ "--confidence", type=float, default=0.7,
+ help="YOLO person detection confidence threshold (default: 0.7).",
+ )
+ p.add_argument(
+ "--max_persons", type=int, default=10,
+ help="Max persons to annotate per image (default: 10).",
+ )
+
+ # SAM3D teacher
+ p.add_argument(
+ "--checkpoint_path", type=str,
+ default="checkpoints/sam-3d-body-dinov3/model.ckpt",
+ )
+ p.add_argument(
+ "--mhr_path", type=str,
+ default="checkpoints/sam-3d-body-dinov3/assets/mhr_model.pt",
+ )
+ p.add_argument(
+ "--hf_repo", type=str, default="facebook/sam-3d-body-dinov3",
+ )
+
+ # Performance
+ p.add_argument(
+ "--timeout", type=float, default=5.0,
+ help="Per-image timeout in seconds (default: 5.0).",
+ )
+ p.add_argument(
+ "--gc_interval", type=int, default=200,
+ help="Force GC + CUDA cache clear every N images (default: 200).",
+ )
+ p.add_argument(
+ "--max_samples", type=int, default=None,
+ help="Stop after producing N annotation crops (not N images).",
+ )
+
+ # Misc
+ p.add_argument("--device", type=str, default="cuda:0")
+ p.add_argument("--debug", action="store_true")
+ p.add_argument(
+ "--shuffle", action="store_true",
+ help="Shuffle image order for diverse sampling across datasets.",
+ )
+ p.add_argument(
+ "--use_rerun", action="store_true",
+ help="Open Rerun viewer with 3 panels: raw image, overlay, 3D mesh.",
+ )
+
+ return p.parse_args()
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+def main() -> None:
+ args = parse_args()
+
+ # Logging
+ logger.remove()
+ fmt = (
+ "<green>{time:HH:mm:ss}</green> | <level>{level:<8}</level> | "
+ "<cyan>{function}</cyan> - <level>{message}</level>"
+ )
+ logger.add(sys.stderr, level="DEBUG" if args.debug else "INFO", format=fmt)
+
+ root_dir = Path(args.root_dir)
+ output_dir = Path(args.output_dir)
+ images_dir = output_dir / "images"
+ annotations_dir = output_dir / "annotations"
+ images_dir.mkdir(parents=True, exist_ok=True)
+ annotations_dir.mkdir(parents=True, exist_ok=True)
+
+ # ------------------------------------------------------------------
+ # 1. Discover images
+ # ------------------------------------------------------------------
+ logger.info(f"Scanning {root_dir} for images ...")
+ all_images = discover_images(root_dir)
+ if not all_images:
+ logger.error(f"No images found under {root_dir}")
+ return
+
+ if args.shuffle:
+ random.shuffle(all_images)
+ logger.info("Image order shuffled for diverse cross-dataset sampling")
+
+ logger.info(f"Found {len(all_images)} images across {len(set(d for d, _ in all_images))} dataset(s)")
+ if args.max_samples:
+ logger.info(f"Will stop after {args.max_samples} annotation crops")
+
+ # ------------------------------------------------------------------
+ # 2. Build resume set — collect existing annotation stems
+ # ------------------------------------------------------------------
+ existing_stems: set[str] = set()
+ for p in annotations_dir.glob("*.npz"):
+ existing_stems.add(p.stem)
+
+ if existing_stems:
+ logger.info(f"Resume: {len(existing_stems)} annotations already on disk")
+
+ # Count existing annotations toward the budget so resumed runs stop correctly
+ total_crops = len(existing_stems)
+
+ # ------------------------------------------------------------------
+ # 3. Load models
+ # ------------------------------------------------------------------
+ logger.info("Loading YOLO detector ...")
+ detector = YOLOPersonDetector(
+ model_name=args.yolo_model,
+ device=args.device,
+ confidence=args.confidence,
+ )
+
+ logger.info("Loading SAM3D teacher ...")
+ teacher = SAM3DTeacher(
+ checkpoint_path=args.checkpoint_path,
+ mhr_path=args.mhr_path,
+ hf_repo_id=args.hf_repo,
+ device=args.device,
+ )
+
+ # ------------------------------------------------------------------
+ # 3b. Rerun setup
+ # ------------------------------------------------------------------
+ use_rerun = args.use_rerun
+ if use_rerun:
+ try:
+ import rerun as rr
+ rr.init("sam3d_cloud_annotation", spawn=True)
+ rr.disable_timeline("frame")
+ rr.log("scene_3d", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True)
+ setup_rerun_blueprint()
+ logger.info("Rerun viewer launched (3-panel layout, no history)")
+ except ImportError:
+ logger.warning("rerun-sdk not installed, disabling --use_rerun")
+ use_rerun = False
+
+ # ------------------------------------------------------------------
+ # 4. Processing loop
+ # ------------------------------------------------------------------
+ stats = {"processed": 0, "skipped": 0, "failed": 0, "timed_out": 0, "crops": 0}
+ t0 = time.perf_counter()
+
+ pbar = tqdm(all_images, desc="Annotating", unit="img", dynamic_ncols=True)
+ for dataset_name, img_path in pbar:
+ # Use relative path from root so subfolder structure prevents collisions
+ # e.g. COCO/train/000001.jpg → "train_000001" (dataset_name = "COCO")
+ rel_path = img_path.relative_to(root_dir)
+ image_stem = str(rel_path.with_suffix("")).replace(os.sep, "_").replace(" ", "_")
+ # Strip the dataset prefix since make_annotation_name re-adds it
+ if image_stem.startswith(dataset_name + "_"):
+ image_stem = image_stem[len(dataset_name) + 1:]
+
+ # Quick check: can we skip the entire image?
+ # We check for person 0; if it exists the image was processed before.
+ first_name = make_annotation_name(dataset_name, image_stem, 0)
+ if first_name in existing_stems:
+ stats["skipped"] += 1
+ continue
+
+ # Load image
+ img_bgr = cv2.imread(str(img_path))
+ if img_bgr is None:
+ logger.warning(f"Cannot read {img_path}")
+ stats["failed"] += 1
+ continue
+ img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
+ H, W = img_rgb.shape[:2]
+
+ # Detect persons
+ try:
+ with time_limit(args.timeout):
+ detections = detector.detect(img_rgb)
+ except InferenceTimeout:
+ logger.warning(f"YOLO timeout on {img_path.name}")
+ stats["timed_out"] += 1
+ continue
+ except Exception as e:
+ logger.warning(f"YOLO failed on {img_path.name}: {e}")
+ stats["failed"] += 1
+ continue
+
+ if not detections:
+ stats["processed"] += 1
+ continue
+
+ detections = detections[: args.max_persons]
+
+ # Process each detected person — pass the FULL image so SAM3D's
+ # CLIFF camera head uses the correct bbox_center, bbox_size, and
+ # img_size to produce cam_trans in full-frame perspective space.
+ rerun_sam3d_outputs: list[dict | None] = []
+ for person_id, det in enumerate(detections):
+ name = make_annotation_name(dataset_name, image_stem, person_id)
+
+ # Skip if this specific crop already exists
+ if name in existing_stems:
+ rerun_sam3d_outputs.append(None)
+ continue
+
+ bbox = det["bbox"]
+
+ try:
+ with time_limit(args.timeout):
+ sam3d_out = teacher.predict_person(img_rgb, bbox)
+ except InferenceTimeout:
+ logger.warning(f"SAM3D timeout on {name}")
+ stats["timed_out"] += 1
+ rerun_sam3d_outputs.append(None)
+ continue
+ except Exception as e:
+ logger.warning(f"SAM3D failed on {name}: {e}")
+ torch.cuda.empty_cache()
+ if args.debug:
+ raise
+ stats["failed"] += 1
+ rerun_sam3d_outputs.append(None)
+ continue
+
+ if sam3d_out is None:
+ stats["failed"] += 1
+ rerun_sam3d_outputs.append(None)
+ continue
+
+ # Validate essential outputs
+ if sam3d_out.get("mhr_model_params") is None:
+ logger.warning(f"No mhr_model_params for {name} (keys: {list(sam3d_out.keys())})")
+ stats["failed"] += 1
+ rerun_sam3d_outputs.append(None)
+ continue
+
+ try:
+ save_crop_and_annotation(
+ img_rgb, bbox, sam3d_out, name,
+ images_dir, annotations_dir,
+ )
+ stats["crops"] += 1
+ total_crops += 1
+ except Exception as e:
+ logger.warning(f"Failed to save {name}: {e}")
+ if args.debug:
+ raise
+ stats["failed"] += 1
+ rerun_sam3d_outputs.append(None)
+ continue
+
+ rerun_sam3d_outputs.append(sam3d_out)
+
+ # Rerun visualization (after all persons for this image)
+ if use_rerun and any(o is not None for o in rerun_sam3d_outputs):
+ log_to_rerun(img_rgb, detections, rerun_sam3d_outputs, teacher.faces)
+
+ stats["processed"] += 1
+
+ # Stop once we have enough crops
+ if args.max_samples and total_crops >= args.max_samples:
+ logger.info(f"Reached {total_crops} total crops (--max_samples {args.max_samples}), stopping")
+ break
+
+ # Periodic cleanup
+ if stats["processed"] % args.gc_interval == 0:
+ torch.cuda.empty_cache()
+ gc.collect()
+
+ # Update progress bar
+ pbar.set_postfix(
+ crops=stats["crops"],
+ fail=stats["failed"],
+ skip=stats["skipped"],
+ )
+
+ elapsed = time.perf_counter() - t0
+ hours = elapsed / 3600
+
+ logger.info("=" * 60)
+ logger.info("ANNOTATION COMPLETE")
+ logger.info("=" * 60)
+ logger.info(f" Images scanned: {len(all_images)}")
+ logger.info(f" Images processed: {stats['processed']}")
+ logger.info(f" Images skipped: {stats['skipped']} (already annotated)")
+ logger.info(f" Crops saved: {stats['crops']} (total on disk: {total_crops})")
+ logger.info(f" Failures: {stats['failed']}")
+ logger.info(f" Timeouts: {stats['timed_out']}")
+ logger.info(f" Wall time: {hours:.2f}h ({elapsed / max(stats['processed'], 1):.2f}s/img)")
+ logger.info(f" Output: {output_dir}")
+ logger.info("=" * 60)
+
+
+if __name__ == "__main__":
+ main()