Python
feat: added multi-person support, corrected freezing bugs due to multiple persons detected. Added rf-detr warmup.
mohamdev6f841fc93331
diff --git a/README.md b/README.mdindex 179bd5e..c9e5c00 100644--- a/README.md+++ b/README.md@@ -197,8 +197,43 @@ per frame than InstantHMR. The two flags below target that bottleneck.| `--detector-stride N` | Run RF-DETR every Nth frame; reuse the previous bbox (slightly expanded) on the in-between frames. Stride 2–3 is the single biggest knob — typically 2–3× end-to-end FPS for slow movement, with negligible quality loss. || `--detector-variant nano` | Use the smallest RF-DETR; biggest win on CPU / Apple Silicon where the detector is doing all the work. || `--device coreml` | On macOS, route the InstantHMR ONNX through `CoreMLExecutionProvider`. |+| `--max-persons N` | Maximum number of persons processed per frame (default: **2**). See below. || `--no-batch-persons` | Disable batched multi-person HMR (one ONNX call per person). The default is batched. |+### `--max-persons` and why it matters++`--max-persons` does two things at once:++1. **Caps detections** — RF-DETR may find more people than you need. Setting `N`+ keeps only the `N` highest-confidence detections per frame.++2. **Fixes the ONNX batch size** — CUDA / ONNX Runtime compiles GPU kernels the+ first time a model runs at a new input shape. Because the batch dimension+ counts as part of the shape, switching from 1 person to 2 persons mid-demo+ would normally trigger a multi-second stall. The pipeline avoids this by+ **always** padding the input to exactly `N` crops (zero-filling unused slots)+ so the session always sees `batch = N`, regardless of how many people are+ actually in frame.++**Startup:** during `PosePipeline.__init__` the pipeline runs two warm-up passes+— `batch = N` and `batch = 1` (the single-person path) — so all CUDA kernels+are compiled before the first real frame. Startup cost is roughly `2 ×`+one-inference time and does **not** scale with `N`.++**Choosing `N`:**++| Scene | Recommended `--max-persons` |+|-------|----------------------------|+| Single person / portrait | 1 |+| Two people, couples, sparring | **2 (default)** |+| Small group, team sport | 4–6 |+| Crowd / many people visible | raise `N`; also consider `--detector-stride` |++Raising `N` does not slow down single-person frames at inference time (the+padded slots run on GPU in parallel), but it does proportionally increase+the memory footprint of the batch buffer and may slightly increase latency+when `N` is large and the GPU can no longer fully parallelise the extra crops.+Measured on RTX 4070 + torch 2.5 cu121 + ORT 1.25 (1080p video, 1 person, 150 frames after warm-up):| `--detector-stride` | RF-DETR ms | HMR ms | total ms | FPS |diff --git a/demo.py b/demo.pyindex a46efe7..3a511bb 100755--- a/demo.py+++ b/demo.py@@ -79,8 +79,8 @@ def parse_args() -> argparse.Namespace:help="RF-DETR detection confidence threshold (default: 0.5).",)p.add_argument(- "--max-persons", type=int, default=5,- help="Max persons processed per frame (default: 5).",+ "--max-persons", type=int, default=2,+ help="Max persons processed per frame (default: 2).",)p.add_argument("--frame-skip", type=int, default=1,@@ -232,9 +232,6 @@ def run_image(args: argparse.Namespace) -> None:mhr = build_mhr_renderer(args)viz = build_visualizer(args, mhr)- # Warm-up: first call includes lazy CUDA / ORT initialisation, so a real- # latency reading is much more useful from the second call onwards.- _ = pipeline.predict(rgb)result = pipeline.predict(rgb)render_ms = viz.log_frame(diff --git a/instanthmr/detector.py b/instanthmr/detector.pyindex c828eb4..7d16364 100644--- a/instanthmr/detector.py+++ b/instanthmr/detector.py@@ -64,6 +64,11 @@ class RFDETRDetector:self._max_persons = max_personsself._variant = variant+ def warmup(self) -> None:+ """Run one silent forward pass to pre-compile PyTorch/CUDA kernels."""+ dummy = np.zeros((224, 224, 3), dtype=np.uint8)+ self.detect(dummy)+@propertydef variant(self) -> str:return self._variantdiff --git a/instanthmr/inference.py b/instanthmr/inference.pyindex a2845c1..0dca31b 100644--- a/instanthmr/inference.py+++ b/instanthmr/inference.py@@ -194,12 +194,17 @@ class InstantHMR:self,image_rgb: np.ndarray,detections: list[dict],+ padded_to: int | None = None,) -> list[HMRPrediction]:"""Run InstantHMR on multiple persons in a single ONNX call.Args:image_rgb: (H, W, 3) uint8 RGB full-frame image.detections: list of ``{"bbox": [x1,y1,x2,y2], "confidence": f}``.+ padded_to: when set, the ONNX batch dimension is always this size+ (zero-padded if fewer detections). Keeping a constant batch+ size prevents CUDA from re-compiling kernels each time the+ person count changes.Returns:One ``HMRPrediction`` per input detection, in the same order.@@ -210,9 +215,11 @@ class InstantHMR:image_rgb = np.ascontiguousarray(image_rgb)h, w = image_rgb.shape[:2]n = len(detections)+ batch = max(padded_to or n, n) # always >= n; zero-pad the rest- crops = np.empty((n, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32)- cliffs = np.empty((n, 3), dtype=np.float32)+ # Use zeros so padded slots produce deterministic (discarded) outputs.+ crops = np.zeros((batch, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32)+ cliffs = np.zeros((batch, 3), dtype=np.float32)sq_meta = [] # per-person (sq_x1, sq_y1, sq_size, bbox)for i, det in enumerate(detections):bbox_arr = np.asarray(det["bbox"], dtype=np.float32).reshape(4)@@ -266,6 +273,28 @@ class InstantHMR:))return results+ def warmup(self, max_batch_size: int = 1) -> None:+ """Pre-warm the ONNX session for the two batch sizes used at runtime.++ With the padded-batch strategy the pipeline only ever calls the session+ with batch=1 (single person) or batch=max_batch_size (multi-person,+ zero-padded to a fixed size). Two runs here compile all CUDA kernels+ upfront so no mid-demo stall occurs when persons first appear.+ """+ dummy_image = np.zeros(+ (max_batch_size, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32+ )+ dummy_cliff = np.zeros((max_batch_size, 3), dtype=np.float32)+ # Large batch first — allocates the most memory; batch=1 reuses the plan.+ self.session.run(+ None, {self._in_image: dummy_image, self._in_cliff: dummy_cliff}+ )+ if max_batch_size > 1:+ self.session.run(+ None,+ {self._in_image: dummy_image[:1], self._in_cliff: dummy_cliff[:1]},+ )+# ------------------------------------------------------------------# Preprocessing# ------------------------------------------------------------------diff --git a/instanthmr/pipeline.py b/instanthmr/pipeline.pyindex a4c23a7..c3e8ddd 100644--- a/instanthmr/pipeline.py+++ b/instanthmr/pipeline.py@@ -83,7 +83,7 @@ class PosePipeline:device: str = "cuda",detector_variant: str = "medium",det_confidence: float = 0.5,- max_persons: int = 5,+ max_persons: int = 2,detector_stride: int = 1,batch_persons: bool = True,):@@ -97,6 +97,18 @@ class PosePipeline:max_persons=max_persons,)+ # Warm up both models before the first real frame.+ # - RF-DETR (PyTorch): first call triggers CUDA JIT for transformer ops.+ # - InstantHMR (ONNX/CUDA EP): first call at a new batch size triggers+ # kernel compilation and GPU memory allocation.+ # With padded-batch inference the session only ever sees two shapes:+ # batch=1 and batch=max_persons — two warm-up calls covers everything.+ print(f"Warming up models (max_persons={max_persons})…", flush=True)+ self.detector.warmup()+ self.hmr.warmup(max_batch_size=max_persons)+ print("Ready.", flush=True)++ self._max_persons = max_personsself._detector_stride = detector_strideself._batch_persons = batch_personsself._frame_idx = 0@@ -138,11 +150,17 @@ class PosePipeline:detections = [{"bbox": fb, "confidence": 0.0}]# ---- InstantHMR ----+ # Hard-cap so we never exceed the pre-warmed batch size.+ detections = detections[: self._max_persons]outputs: list[HMRPrediction] = []t1 = time.perf_counter()if detections:if self._batch_persons and len(detections) > 1:- outputs = self.hmr.predict_batch(image_rgb, detections)+ # Always pad to _max_persons so ONNX sees a constant batch+ # shape and never triggers a mid-demo recompilation.+ outputs = self.hmr.predict_batch(+ image_rgb, detections, padded_to=self._max_persons+ )else:for det in detections:outputs.append(self.hmr.predict(