Skip to content
Python

feat: demo.py --show-reference overlays the saved SAM 3D Body teacher, packed as MHR parameters and decoded by the student's own rig

mohamdevd86702f13aa4
diff --git a/.gitignore b/.gitignore
index a40b5c6..f305feb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,4 +50,9 @@ runs/
# Notebook checkpoints
.ipynb_checkpoints/
competitors/
-*.png
\ No newline at end of file
+*.png
+
+# Saved teacher runs for `demo.py --show-reference` (renders + packed .npz).
+# Local and regenerable: tools/pack_reference.py rebuilds the .npz from a
+# video_to_pose_pipeline --save_meshes run. 247 MB here.
+reference/
diff --git a/demo.py b/demo.py
index c9f3a8b..047891a 100755
--- a/demo.py
+++ b/demo.py
@@ -37,6 +37,10 @@ Usage
# Video file
python demo.py --video path/to/clip.mp4
+ # Video file, with the saved SAM 3D Body teacher overlaid for comparison
+ # (bare flag reads reference/<video stem>.npz)
+ python demo.py --video vid1.mp4 --mhr-model --show-reference
+
# Live camera (index 0)
python demo.py --camera 0
"""
@@ -54,6 +58,7 @@ import numpy as np
from instanthmr import PosePipeline, onnx_output_names
from instanthmr.mhr_renderer import DEFAULT_MHR_SCRIPT
+from instanthmr.reference import ReferenceTrack, resolve_reference
from instanthmr.smoothing import TemporalSmoother
from instanthmr.visualizer import RerunVisualizer
@@ -185,6 +190,18 @@ def parse_args() -> argparse.Namespace:
"LOD (18 439 verts) and ignores this."
),
)
+ p.add_argument(
+ "--show-reference", type=str, nargs="?", default=None, const="",
+ metavar="NPZ",
+ help=(
+ "Overlay a saved SAM 3D Body (teacher) prediction for this clip, "
+ "so you can compare it against the student in the same 3D scene. "
+ "Bare flag = reference/<video stem>.npz. The reference stores MHR "
+ "parameters and is decoded by the same rig as the student, so any "
+ "difference you see is the model, not the renderer. Build one with "
+ "tools/pack_reference.py — see instanthmr/reference.py."
+ ),
+ )
p.add_argument(
"--mesh-alpha", type=float, default=0.35, metavar="A",
help=(
@@ -324,10 +341,30 @@ def build_all(args: argparse.Namespace):
save_path=args.save_rrd,
mhr_renderer=mhr,
mesh_alpha=args.mesh_alpha,
+ reference=build_reference(args),
)
return pipeline, viz, mhr
+def build_reference(args: argparse.Namespace):
+ """Load the saved teacher track for ``--show-reference``, or ``None``."""
+ path = resolve_reference(args.show_reference, getattr(args, "video", None))
+ if path is None:
+ return None
+ if not path.exists():
+ sys.exit(
+ f"[error] reference not found: {path}\n"
+ " Build it by running the teacher once, then packing it:\n"
+ " MOMENTUM_ENABLED=0 python main.py --video_path <clip> \\\n"
+ " --output_path reference/<stem> --save_meshes\n"
+ " python tools/pack_reference.py reference/<stem>/meshes "
+ "reference/<stem>.npz"
+ )
+ track = ReferenceTrack(path)
+ print(f"reference: {path} — {len(track)} frames, {track.num_persons} persons")
+ return track
+
+
def _print_timings(
prefix: str,
detector_ms: float,
diff --git a/instanthmr/reference.py b/instanthmr/reference.py
new file mode 100644
index 0000000..1401797
--- /dev/null
+++ b/instanthmr/reference.py
@@ -0,0 +1,104 @@
+"""Saved SAM 3D Body predictions, for side-by-side comparison in the demo.
+
+The teacher's own output on a clip is the most useful thing to hold a student
+against visually: it shares the rig, so the two meshes are directly comparable
+rather than needing a conversion.
+
+A reference file is produced by running the teacher once and packing the result:
+
+ MOMENTUM_ENABLED=0 python main.py --video_path vid1.mp4 \\
+ --output_path reference/vid1 --save_meshes # in video_to_pose_pipeline
+ python tools/pack_reference.py reference/vid1/meshes reference/vid1.npz
+
+It stores MHR **parameters**, not vertices, and ``demo.py`` decodes them with
+the same rig it uses for the student. That is what makes the comparison honest:
+any difference you see in the viewer is the model, never the renderer.
+
+``MOMENTUM_ENABLED=0`` matters -- it makes the teacher's MHR head load the
+TorchScript rig instead of ``mhr.mhr.MHR``, whose pymomentum extension needs
+``torch>=2.8`` and segfaults (not raises) against an older torch.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class ReferencePerson:
+ """One person in one reference frame, in the demo's own conventions."""
+
+ mhr_params: np.ndarray # (204,) float32
+ shape_params: np.ndarray # (45,) float32
+ cam_trans: np.ndarray # (3,) float32, rig-local -> camera space
+ joints_3d_cam: np.ndarray # (70, 3) float32, camera space
+ joints_2d: np.ndarray # (70, 2) float32, pixels
+ bbox: np.ndarray # (4,) float32, xyxy
+
+
+class ReferenceTrack:
+ """Random access by frame index into a packed reference run.
+
+ Frames where the teacher detected nobody were never written, so a lookup
+ for them returns an empty list rather than raising.
+ """
+
+ def __init__(self, path: str | Path):
+ self.path = Path(path)
+ z = np.load(self.path)
+ self.image_shape = z["image_shape"]
+ counts = z["counts"].astype(int)
+ starts = np.concatenate([[0], np.cumsum(counts)[:-1]])
+ # frame index -> (offset, count) into the flat per-person arrays
+ self._span = {int(f): (int(s), int(c))
+ for f, s, c in zip(z["frame_idx"], starts, counts)}
+ self._model = z["model_params"]
+ self._shape = z["shape_params"]
+ self._trans = z["cam_trans"]
+ self._j3d = z["joints_3d"]
+ self._j2d = z["joints_2d"]
+ self._bbox = z["bbox"]
+ self.num_frames = len(counts)
+ self.num_persons = int(counts.sum())
+
+ def __len__(self) -> int:
+ return self.num_frames
+
+ def persons_at(self, frame_idx: int) -> list[ReferencePerson]:
+ span = self._span.get(int(frame_idx))
+ if span is None:
+ return []
+ start, count = span
+ out = []
+ for i in range(start, start + count):
+ # joints_3d as saved by the teacher are rig-local, the same frame
+ # its vertices are in; the demo works in camera space.
+ out.append(ReferencePerson(
+ mhr_params=self._model[i],
+ shape_params=self._shape[i],
+ cam_trans=self._trans[i],
+ joints_3d_cam=self._j3d[i] + self._trans[i],
+ joints_2d=self._j2d[i],
+ bbox=self._bbox[i],
+ ))
+ return out
+
+
+def resolve_reference(arg: str | None, video: str | Path | None) -> Path | None:
+ """Turn ``--show-reference`` into a path.
+
+ A bare flag means "the reference for this clip": ``reference/<stem>.npz``
+ beside the repo root, named after the input video.
+ """
+ if arg is None:
+ return None
+ if arg:
+ return Path(arg)
+ if video is None:
+ raise SystemExit(
+ "[error] --show-reference with no path needs --video to name the "
+ "reference file (reference/<video stem>.npz)")
+ return Path("reference") / f"{Path(video).stem}.npz"
diff --git a/instanthmr/visualizer.py b/instanthmr/visualizer.py
index bbf9f3d..3a37db7 100644
--- a/instanthmr/visualizer.py
+++ b/instanthmr/visualizer.py
@@ -42,6 +42,13 @@ MESH_COLOR = (200, 160, 130)
#: body, high enough that the silhouette is still legible.
MESH_ALPHA = 0.35
+#: Reference (teacher) overlay colours — a cool palette against the student's
+#: warm skin tone and green/yellow skeleton, so the two read apart at a glance
+#: even where the bodies overlap.
+REF_MESH_COLOR = (110, 170, 255)
+REF_JOINT_COLOR = (0, 200, 255)
+REF_SKELETON_COLOR = (255, 110, 255)
+
class RerunVisualizer:
"""Rerun logger for the demo.
@@ -56,6 +63,11 @@ class RerunVisualizer:
mesh_alpha: body-mesh opacity in ``[0, 1]``. Applied through Rerun's
``albedo_factor`` rather than per-vertex alpha, so one scalar
controls the whole surface.
+ reference: optional :class:`instanthmr.reference.ReferenceTrack` whose
+ saved teacher prediction is logged alongside the student's under
+ ``world/reference/``, in the cool palette above. It is decoded by
+ the SAME ``mhr_renderer``, so a visible difference is the model and
+ not the renderer.
"""
def __init__(
@@ -65,6 +77,7 @@ class RerunVisualizer:
save_path: str | None = None,
mhr_renderer: "MHRRenderer | None" = None,
mesh_alpha: float = MESH_ALPHA,
+ reference=None,
):
import rerun as rr
@@ -74,6 +87,12 @@ class RerunVisualizer:
*MESH_COLOR,
int(round(255 * min(max(mesh_alpha, 0.0), 1.0))),
)
+ self._reference = reference
+ self._ref_albedo = (
+ *REF_MESH_COLOR,
+ int(round(255 * min(max(mesh_alpha, 0.0), 1.0))),
+ )
+ self._prev_num_ref = 0
rr.init(application_id, spawn=spawn_viewer)
# SAM3D / MHR convention: right-handed, Y-down camera frame.
@@ -138,6 +157,12 @@ class RerunVisualizer:
h, w = image_rgb.shape[:2]
+ # Before the no-detection early-out below: the reference is independent
+ # of whether the student found anybody, and seeing it alone on a frame
+ # the student missed is exactly the failure worth looking at.
+ if self._reference is not None:
+ self._log_reference(frame_idx)
+
# Always log the image — even when no person was detected — so the
# viewer keeps showing the camera feed and the timing HUD.
render_ms = 0.0
@@ -220,6 +245,51 @@ class RerunVisualizer:
rr.LineStrips3D(lines, colors=[255, 230, 0], radii=0.004),
)
+ def _log_reference(self, frame_idx: int) -> None:
+ """Log the saved teacher prediction for this frame under ``world/reference``.
+
+ Kept in a sibling entity tree rather than mixed into
+ ``world/persons/`` so the viewer can toggle either side on its own.
+ """
+ rr = self._rr
+ persons = self._reference.persons_at(frame_idx)
+
+ for stale in range(len(persons), self._prev_num_ref):
+ rr.log(f"world/reference/person_{stale}", rr.Clear(recursive=True))
+ self._prev_num_ref = len(persons)
+
+ for idx, ref in enumerate(persons):
+ path = f"world/reference/person_{idx}"
+ joints = ref.joints_3d_cam
+ rr.log(
+ f"{path}/joints",
+ rr.Points3D(positions=joints, radii=0.012,
+ colors=list(REF_JOINT_COLOR)),
+ )
+ edges = edges_for(joints.shape[0])
+ if edges:
+ rr.log(
+ f"{path}/skeleton",
+ rr.LineStrips3D(
+ [[joints[i].tolist(), joints[j].tolist()] for i, j in edges],
+ colors=list(REF_SKELETON_COLOR), radii=0.004,
+ ),
+ )
+ # The teacher stored parameters, not vertices; decoding them with
+ # the student's own rig is what keeps the two meshes comparable.
+ if self._mhr_renderer is not None:
+ verts = self._mhr_renderer.forward(
+ ref.mhr_params, ref.shape_params,
+ ) + ref.cam_trans
+ rr.log(
+ f"{path}/mesh",
+ rr.Mesh3D(
+ vertex_positions=verts,
+ triangle_indices=self._mhr_renderer.faces,
+ albedo_factor=self._ref_albedo,
+ ),
+ )
+
def _log_person_mesh(self, idx: int, person: HMRPrediction) -> None:
"""Run MHR forward pass and log the resulting mesh vertices via Rerun."""
assert self._mhr_renderer is not None
diff --git a/tools/pack_reference.py b/tools/pack_reference.py
new file mode 100644
index 0000000..2768fb6
--- /dev/null
+++ b/tools/pack_reference.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Pack a SAM 3D Body reference run into one small .npz for ``demo.py --show-reference``.
+
+The reference comes from ``video_to_pose_pipeline/main.py --save_meshes``, which
+writes one ``frame_%06d.npz`` per frame carrying the full 18,439-vertex mesh AND
+a copy of the constant 36,874-face index array -- ~360 kB per frame, so a few
+minutes of video costs a gigabyte.
+
+None of that needs storing. The saved ``person_i_vertices`` are **exactly** what
+``instanthmr.mhr_renderer`` reproduces from the saved ``mhr_model_params`` and
+``mhr_shape_params`` -- verified at 0.0005 mm on vid4, same rig, same rig-local
+frame, faces from the rig itself. So this keeps the parameters and drops the
+mesh, which is ~1 kB per frame and has a useful side effect: the reference and
+the student are then decoded by the SAME rig, so anything you see differing in
+the viewer is the model, never the renderer.
+
+ python tools/pack_reference.py reference/vid4/meshes reference/vid4.npz
+
+Frames with no detection produce no input file and are simply absent from the
+output; ``frame_idx`` records which frames are present.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import numpy as np
+
+# Per-person fields copied straight through, with the output key they take.
+PERSON_FIELDS = {
+ "mhr_model_params": "model_params",
+ "mhr_shape_params": "shape_params",
+ "cam_translation": "cam_trans",
+ "joints_3d": "joints_3d",
+ "joints_2d": "joints_2d",
+ "bbox": "bbox",
+ "cam_focal_length": "focal_length",
+ "cam_principal_point": "principal_point",
+}
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("mesh_dir", help="the pipeline's meshes/ directory")
+ ap.add_argument("out", help="output .npz")
+ a = ap.parse_args()
+
+ files = sorted(Path(a.mesh_dir).glob("frame_*.npz"))
+ if not files:
+ raise SystemExit(f"no frame_*.npz under {a.mesh_dir}")
+
+ frame_idx, counts = [], []
+ cols: dict[str, list] = {k: [] for k in PERSON_FIELDS.values()}
+ image_shape = None
+
+ for f in files:
+ d = np.load(f)
+ n = int(d["num_persons"])
+ frame_idx.append(int(d["frame_idx"]))
+ counts.append(n)
+ if image_shape is None:
+ image_shape = d["image_shape"]
+ for i in range(n):
+ for src, dst in PERSON_FIELDS.items():
+ cols[dst].append(d[f"person_{i}_{src}"])
+
+ out = {
+ "frame_idx": np.asarray(frame_idx, np.int32),
+ "counts": np.asarray(counts, np.int32),
+ "image_shape": np.asarray(image_shape, np.int32),
+ }
+ for dst, vals in cols.items():
+ out[dst] = np.stack(vals).astype(np.float32)
+
+ Path(a.out).parent.mkdir(parents=True, exist_ok=True)
+ np.savez_compressed(a.out, **out)
+ mb = Path(a.out).stat().st_size / 1e6
+ print(f"{len(files)} frames, {sum(counts)} persons -> {a.out} ({mb:.2f} MB)")
+
+
+if __name__ == "__main__":
+ main()