Python
feat: demo.py --show-reference overlays the saved SAM 3D Body teacher, packed as MHR parameters and decoded by the student's own rig
mohamdevd86702f13aa4
diff --git a/.gitignore b/.gitignoreindex a40b5c6..f305feb 100644--- a/.gitignore+++ b/.gitignore@@ -50,4 +50,9 @@ runs/# Notebook checkpoints.ipynb_checkpoints/competitors/-*.png\ No newline at end of file+*.png++# Saved teacher runs for `demo.py --show-reference` (renders + packed .npz).+# Local and regenerable: tools/pack_reference.py rebuilds the .npz from a+# video_to_pose_pipeline --save_meshes run. 247 MB here.+reference/diff --git a/demo.py b/demo.pyindex c9f3a8b..047891a 100755--- a/demo.py+++ b/demo.py@@ -37,6 +37,10 @@ Usage# Video filepython demo.py --video path/to/clip.mp4+ # Video file, with the saved SAM 3D Body teacher overlaid for comparison+ # (bare flag reads reference/<video stem>.npz)+ python demo.py --video vid1.mp4 --mhr-model --show-reference+# Live camera (index 0)python demo.py --camera 0"""@@ -54,6 +58,7 @@ import numpy as npfrom instanthmr import PosePipeline, onnx_output_namesfrom instanthmr.mhr_renderer import DEFAULT_MHR_SCRIPT+from instanthmr.reference import ReferenceTrack, resolve_referencefrom instanthmr.smoothing import TemporalSmootherfrom instanthmr.visualizer import RerunVisualizer@@ -185,6 +190,18 @@ def parse_args() -> argparse.Namespace:"LOD (18 439 verts) and ignores this."),)+ p.add_argument(+ "--show-reference", type=str, nargs="?", default=None, const="",+ metavar="NPZ",+ help=(+ "Overlay a saved SAM 3D Body (teacher) prediction for this clip, "+ "so you can compare it against the student in the same 3D scene. "+ "Bare flag = reference/<video stem>.npz. The reference stores MHR "+ "parameters and is decoded by the same rig as the student, so any "+ "difference you see is the model, not the renderer. Build one with "+ "tools/pack_reference.py — see instanthmr/reference.py."+ ),+ )p.add_argument("--mesh-alpha", type=float, default=0.35, metavar="A",help=(@@ -324,10 +341,30 @@ def build_all(args: argparse.Namespace):save_path=args.save_rrd,mhr_renderer=mhr,mesh_alpha=args.mesh_alpha,+ reference=build_reference(args),)return pipeline, viz, mhr+def build_reference(args: argparse.Namespace):+ """Load the saved teacher track for ``--show-reference``, or ``None``."""+ path = resolve_reference(args.show_reference, getattr(args, "video", None))+ if path is None:+ return None+ if not path.exists():+ sys.exit(+ f"[error] reference not found: {path}\n"+ " Build it by running the teacher once, then packing it:\n"+ " MOMENTUM_ENABLED=0 python main.py --video_path <clip> \\\n"+ " --output_path reference/<stem> --save_meshes\n"+ " python tools/pack_reference.py reference/<stem>/meshes "+ "reference/<stem>.npz"+ )+ track = ReferenceTrack(path)+ print(f"reference: {path} — {len(track)} frames, {track.num_persons} persons")+ return track++def _print_timings(prefix: str,detector_ms: float,diff --git a/instanthmr/reference.py b/instanthmr/reference.pynew file mode 100644index 0000000..1401797--- /dev/null+++ b/instanthmr/reference.py@@ -0,0 +1,104 @@+"""Saved SAM 3D Body predictions, for side-by-side comparison in the demo.++The teacher's own output on a clip is the most useful thing to hold a student+against visually: it shares the rig, so the two meshes are directly comparable+rather than needing a conversion.++A reference file is produced by running the teacher once and packing the result:++ MOMENTUM_ENABLED=0 python main.py --video_path vid1.mp4 \\+ --output_path reference/vid1 --save_meshes # in video_to_pose_pipeline+ python tools/pack_reference.py reference/vid1/meshes reference/vid1.npz++It stores MHR **parameters**, not vertices, and ``demo.py`` decodes them with+the same rig it uses for the student. That is what makes the comparison honest:+any difference you see in the viewer is the model, never the renderer.++``MOMENTUM_ENABLED=0`` matters -- it makes the teacher's MHR head load the+TorchScript rig instead of ``mhr.mhr.MHR``, whose pymomentum extension needs+``torch>=2.8`` and segfaults (not raises) against an older torch.+"""++from __future__ import annotations++from dataclasses import dataclass+from pathlib import Path++import numpy as np+++@dataclass(frozen=True)+class ReferencePerson:+ """One person in one reference frame, in the demo's own conventions."""++ mhr_params: np.ndarray # (204,) float32+ shape_params: np.ndarray # (45,) float32+ cam_trans: np.ndarray # (3,) float32, rig-local -> camera space+ joints_3d_cam: np.ndarray # (70, 3) float32, camera space+ joints_2d: np.ndarray # (70, 2) float32, pixels+ bbox: np.ndarray # (4,) float32, xyxy+++class ReferenceTrack:+ """Random access by frame index into a packed reference run.++ Frames where the teacher detected nobody were never written, so a lookup+ for them returns an empty list rather than raising.+ """++ def __init__(self, path: str | Path):+ self.path = Path(path)+ z = np.load(self.path)+ self.image_shape = z["image_shape"]+ counts = z["counts"].astype(int)+ starts = np.concatenate([[0], np.cumsum(counts)[:-1]])+ # frame index -> (offset, count) into the flat per-person arrays+ self._span = {int(f): (int(s), int(c))+ for f, s, c in zip(z["frame_idx"], starts, counts)}+ self._model = z["model_params"]+ self._shape = z["shape_params"]+ self._trans = z["cam_trans"]+ self._j3d = z["joints_3d"]+ self._j2d = z["joints_2d"]+ self._bbox = z["bbox"]+ self.num_frames = len(counts)+ self.num_persons = int(counts.sum())++ def __len__(self) -> int:+ return self.num_frames++ def persons_at(self, frame_idx: int) -> list[ReferencePerson]:+ span = self._span.get(int(frame_idx))+ if span is None:+ return []+ start, count = span+ out = []+ for i in range(start, start + count):+ # joints_3d as saved by the teacher are rig-local, the same frame+ # its vertices are in; the demo works in camera space.+ out.append(ReferencePerson(+ mhr_params=self._model[i],+ shape_params=self._shape[i],+ cam_trans=self._trans[i],+ joints_3d_cam=self._j3d[i] + self._trans[i],+ joints_2d=self._j2d[i],+ bbox=self._bbox[i],+ ))+ return out+++def resolve_reference(arg: str | None, video: str | Path | None) -> Path | None:+ """Turn ``--show-reference`` into a path.++ A bare flag means "the reference for this clip": ``reference/<stem>.npz``+ beside the repo root, named after the input video.+ """+ if arg is None:+ return None+ if arg:+ return Path(arg)+ if video is None:+ raise SystemExit(+ "[error] --show-reference with no path needs --video to name the "+ "reference file (reference/<video stem>.npz)")+ return Path("reference") / f"{Path(video).stem}.npz"diff --git a/instanthmr/visualizer.py b/instanthmr/visualizer.pyindex bbf9f3d..3a37db7 100644--- a/instanthmr/visualizer.py+++ b/instanthmr/visualizer.py@@ -42,6 +42,13 @@ MESH_COLOR = (200, 160, 130)#: body, high enough that the silhouette is still legible.MESH_ALPHA = 0.35+#: Reference (teacher) overlay colours — a cool palette against the student's+#: warm skin tone and green/yellow skeleton, so the two read apart at a glance+#: even where the bodies overlap.+REF_MESH_COLOR = (110, 170, 255)+REF_JOINT_COLOR = (0, 200, 255)+REF_SKELETON_COLOR = (255, 110, 255)+class RerunVisualizer:"""Rerun logger for the demo.@@ -56,6 +63,11 @@ class RerunVisualizer:mesh_alpha: body-mesh opacity in ``[0, 1]``. Applied through Rerun's``albedo_factor`` rather than per-vertex alpha, so one scalarcontrols the whole surface.+ reference: optional :class:`instanthmr.reference.ReferenceTrack` whose+ saved teacher prediction is logged alongside the student's under+ ``world/reference/``, in the cool palette above. It is decoded by+ the SAME ``mhr_renderer``, so a visible difference is the model and+ not the renderer."""def __init__(@@ -65,6 +77,7 @@ class RerunVisualizer:save_path: str | None = None,mhr_renderer: "MHRRenderer | None" = None,mesh_alpha: float = MESH_ALPHA,+ reference=None,):import rerun as rr@@ -74,6 +87,12 @@ class RerunVisualizer:*MESH_COLOR,int(round(255 * min(max(mesh_alpha, 0.0), 1.0))),)+ self._reference = reference+ self._ref_albedo = (+ *REF_MESH_COLOR,+ int(round(255 * min(max(mesh_alpha, 0.0), 1.0))),+ )+ self._prev_num_ref = 0rr.init(application_id, spawn=spawn_viewer)# SAM3D / MHR convention: right-handed, Y-down camera frame.@@ -138,6 +157,12 @@ class RerunVisualizer:h, w = image_rgb.shape[:2]+ # Before the no-detection early-out below: the reference is independent+ # of whether the student found anybody, and seeing it alone on a frame+ # the student missed is exactly the failure worth looking at.+ if self._reference is not None:+ self._log_reference(frame_idx)+# Always log the image — even when no person was detected — so the# viewer keeps showing the camera feed and the timing HUD.render_ms = 0.0@@ -220,6 +245,51 @@ class RerunVisualizer:rr.LineStrips3D(lines, colors=[255, 230, 0], radii=0.004),)+ def _log_reference(self, frame_idx: int) -> None:+ """Log the saved teacher prediction for this frame under ``world/reference``.++ Kept in a sibling entity tree rather than mixed into+ ``world/persons/`` so the viewer can toggle either side on its own.+ """+ rr = self._rr+ persons = self._reference.persons_at(frame_idx)++ for stale in range(len(persons), self._prev_num_ref):+ rr.log(f"world/reference/person_{stale}", rr.Clear(recursive=True))+ self._prev_num_ref = len(persons)++ for idx, ref in enumerate(persons):+ path = f"world/reference/person_{idx}"+ joints = ref.joints_3d_cam+ rr.log(+ f"{path}/joints",+ rr.Points3D(positions=joints, radii=0.012,+ colors=list(REF_JOINT_COLOR)),+ )+ edges = edges_for(joints.shape[0])+ if edges:+ rr.log(+ f"{path}/skeleton",+ rr.LineStrips3D(+ [[joints[i].tolist(), joints[j].tolist()] for i, j in edges],+ colors=list(REF_SKELETON_COLOR), radii=0.004,+ ),+ )+ # The teacher stored parameters, not vertices; decoding them with+ # the student's own rig is what keeps the two meshes comparable.+ if self._mhr_renderer is not None:+ verts = self._mhr_renderer.forward(+ ref.mhr_params, ref.shape_params,+ ) + ref.cam_trans+ rr.log(+ f"{path}/mesh",+ rr.Mesh3D(+ vertex_positions=verts,+ triangle_indices=self._mhr_renderer.faces,+ albedo_factor=self._ref_albedo,+ ),+ )+def _log_person_mesh(self, idx: int, person: HMRPrediction) -> None:"""Run MHR forward pass and log the resulting mesh vertices via Rerun."""assert self._mhr_renderer is not Nonediff --git a/tools/pack_reference.py b/tools/pack_reference.pynew file mode 100644index 0000000..2768fb6--- /dev/null+++ b/tools/pack_reference.py@@ -0,0 +1,84 @@+#!/usr/bin/env python3+"""Pack a SAM 3D Body reference run into one small .npz for ``demo.py --show-reference``.++The reference comes from ``video_to_pose_pipeline/main.py --save_meshes``, which+writes one ``frame_%06d.npz`` per frame carrying the full 18,439-vertex mesh AND+a copy of the constant 36,874-face index array -- ~360 kB per frame, so a few+minutes of video costs a gigabyte.++None of that needs storing. The saved ``person_i_vertices`` are **exactly** what+``instanthmr.mhr_renderer`` reproduces from the saved ``mhr_model_params`` and+``mhr_shape_params`` -- verified at 0.0005 mm on vid4, same rig, same rig-local+frame, faces from the rig itself. So this keeps the parameters and drops the+mesh, which is ~1 kB per frame and has a useful side effect: the reference and+the student are then decoded by the SAME rig, so anything you see differing in+the viewer is the model, never the renderer.++ python tools/pack_reference.py reference/vid4/meshes reference/vid4.npz++Frames with no detection produce no input file and are simply absent from the+output; ``frame_idx`` records which frames are present.+"""++from __future__ import annotations++import argparse+from pathlib import Path++import numpy as np++# Per-person fields copied straight through, with the output key they take.+PERSON_FIELDS = {+ "mhr_model_params": "model_params",+ "mhr_shape_params": "shape_params",+ "cam_translation": "cam_trans",+ "joints_3d": "joints_3d",+ "joints_2d": "joints_2d",+ "bbox": "bbox",+ "cam_focal_length": "focal_length",+ "cam_principal_point": "principal_point",+}+++def main() -> None:+ ap = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("mesh_dir", help="the pipeline's meshes/ directory")+ ap.add_argument("out", help="output .npz")+ a = ap.parse_args()++ files = sorted(Path(a.mesh_dir).glob("frame_*.npz"))+ if not files:+ raise SystemExit(f"no frame_*.npz under {a.mesh_dir}")++ frame_idx, counts = [], []+ cols: dict[str, list] = {k: [] for k in PERSON_FIELDS.values()}+ image_shape = None++ for f in files:+ d = np.load(f)+ n = int(d["num_persons"])+ frame_idx.append(int(d["frame_idx"]))+ counts.append(n)+ if image_shape is None:+ image_shape = d["image_shape"]+ for i in range(n):+ for src, dst in PERSON_FIELDS.items():+ cols[dst].append(d[f"person_{i}_{src}"])++ out = {+ "frame_idx": np.asarray(frame_idx, np.int32),+ "counts": np.asarray(counts, np.int32),+ "image_shape": np.asarray(image_shape, np.int32),+ }+ for dst, vals in cols.items():+ out[dst] = np.stack(vals).astype(np.float32)++ Path(a.out).parent.mkdir(parents=True, exist_ok=True)+ np.savez_compressed(a.out, **out)+ mb = Path(a.out).stat().st_size / 1e6+ print(f"{len(files)} frames, {sum(counts)} persons -> {a.out} ({mb:.2f} MB)")+++if __name__ == "__main__":+ main()