Skip to content
Python

feat: added multi-person support, corrected freezing bugs due to multiple persons detected. Added rf-detr warmup.

mohamdev6f841fc93331
diff --git a/README.md b/README.md
index 179bd5e..c9e5c00 100644
--- a/README.md
+++ b/README.md
@@ -197,8 +197,43 @@ per frame than InstantHMR. The two flags below target that bottleneck.
| `--detector-stride N` | Run RF-DETR every Nth frame; reuse the previous bbox (slightly expanded) on the in-between frames. Stride 2–3 is the single biggest knob — typically 2–3× end-to-end FPS for slow movement, with negligible quality loss. |
| `--detector-variant nano` | Use the smallest RF-DETR; biggest win on CPU / Apple Silicon where the detector is doing all the work. |
| `--device coreml` | On macOS, route the InstantHMR ONNX through `CoreMLExecutionProvider`. |
+| `--max-persons N` | Maximum number of persons processed per frame (default: **2**). See below. |
| `--no-batch-persons` | Disable batched multi-person HMR (one ONNX call per person). The default is batched. |
+### `--max-persons` and why it matters
+
+`--max-persons` does two things at once:
+
+1. **Caps detections** — RF-DETR may find more people than you need. Setting `N`
+ keeps only the `N` highest-confidence detections per frame.
+
+2. **Fixes the ONNX batch size** — CUDA / ONNX Runtime compiles GPU kernels the
+ first time a model runs at a new input shape. Because the batch dimension
+ counts as part of the shape, switching from 1 person to 2 persons mid-demo
+ would normally trigger a multi-second stall. The pipeline avoids this by
+ **always** padding the input to exactly `N` crops (zero-filling unused slots)
+ so the session always sees `batch = N`, regardless of how many people are
+ actually in frame.
+
+**Startup:** during `PosePipeline.__init__` the pipeline runs two warm-up passes
+— `batch = N` and `batch = 1` (the single-person path) — so all CUDA kernels
+are compiled before the first real frame. Startup cost is roughly `2 ×`
+one-inference time and does **not** scale with `N`.
+
+**Choosing `N`:**
+
+| Scene | Recommended `--max-persons` |
+|-------|----------------------------|
+| Single person / portrait | 1 |
+| Two people, couples, sparring | **2 (default)** |
+| Small group, team sport | 4–6 |
+| Crowd / many people visible | raise `N`; also consider `--detector-stride` |
+
+Raising `N` does not slow down single-person frames at inference time (the
+padded slots run on GPU in parallel), but it does proportionally increase
+the memory footprint of the batch buffer and may slightly increase latency
+when `N` is large and the GPU can no longer fully parallelise the extra crops.
+
Measured on RTX 4070 + torch 2.5 cu121 + ORT 1.25 (1080p video, 1 person, 150 frames after warm-up):
| `--detector-stride` | RF-DETR ms | HMR ms | total ms | FPS |
diff --git a/demo.py b/demo.py
index a46efe7..3a511bb 100755
--- a/demo.py
+++ b/demo.py
@@ -79,8 +79,8 @@ def parse_args() -> argparse.Namespace:
help="RF-DETR detection confidence threshold (default: 0.5).",
)
p.add_argument(
- "--max-persons", type=int, default=5,
- help="Max persons processed per frame (default: 5).",
+ "--max-persons", type=int, default=2,
+ help="Max persons processed per frame (default: 2).",
)
p.add_argument(
"--frame-skip", type=int, default=1,
@@ -232,9 +232,6 @@ def run_image(args: argparse.Namespace) -> None:
mhr = build_mhr_renderer(args)
viz = build_visualizer(args, mhr)
- # Warm-up: first call includes lazy CUDA / ORT initialisation, so a real
- # latency reading is much more useful from the second call onwards.
- _ = pipeline.predict(rgb)
result = pipeline.predict(rgb)
render_ms = viz.log_frame(
diff --git a/instanthmr/detector.py b/instanthmr/detector.py
index c828eb4..7d16364 100644
--- a/instanthmr/detector.py
+++ b/instanthmr/detector.py
@@ -64,6 +64,11 @@ class RFDETRDetector:
self._max_persons = max_persons
self._variant = variant
+ def warmup(self) -> None:
+ """Run one silent forward pass to pre-compile PyTorch/CUDA kernels."""
+ dummy = np.zeros((224, 224, 3), dtype=np.uint8)
+ self.detect(dummy)
+
@property
def variant(self) -> str:
return self._variant
diff --git a/instanthmr/inference.py b/instanthmr/inference.py
index a2845c1..0dca31b 100644
--- a/instanthmr/inference.py
+++ b/instanthmr/inference.py
@@ -194,12 +194,17 @@ class InstantHMR:
self,
image_rgb: np.ndarray,
detections: list[dict],
+ padded_to: int | None = None,
) -> list[HMRPrediction]:
"""Run InstantHMR on multiple persons in a single ONNX call.
Args:
image_rgb: (H, W, 3) uint8 RGB full-frame image.
detections: list of ``{"bbox": [x1,y1,x2,y2], "confidence": f}``.
+ padded_to: when set, the ONNX batch dimension is always this size
+ (zero-padded if fewer detections). Keeping a constant batch
+ size prevents CUDA from re-compiling kernels each time the
+ person count changes.
Returns:
One ``HMRPrediction`` per input detection, in the same order.
@@ -210,9 +215,11 @@ class InstantHMR:
image_rgb = np.ascontiguousarray(image_rgb)
h, w = image_rgb.shape[:2]
n = len(detections)
+ batch = max(padded_to or n, n) # always >= n; zero-pad the rest
- crops = np.empty((n, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32)
- cliffs = np.empty((n, 3), dtype=np.float32)
+ # Use zeros so padded slots produce deterministic (discarded) outputs.
+ crops = np.zeros((batch, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32)
+ cliffs = np.zeros((batch, 3), dtype=np.float32)
sq_meta = [] # per-person (sq_x1, sq_y1, sq_size, bbox)
for i, det in enumerate(detections):
bbox_arr = np.asarray(det["bbox"], dtype=np.float32).reshape(4)
@@ -266,6 +273,28 @@ class InstantHMR:
))
return results
+ def warmup(self, max_batch_size: int = 1) -> None:
+ """Pre-warm the ONNX session for the two batch sizes used at runtime.
+
+ With the padded-batch strategy the pipeline only ever calls the session
+ with batch=1 (single person) or batch=max_batch_size (multi-person,
+ zero-padded to a fixed size). Two runs here compile all CUDA kernels
+ upfront so no mid-demo stall occurs when persons first appear.
+ """
+ dummy_image = np.zeros(
+ (max_batch_size, 3, INPUT_SIZE, INPUT_SIZE), dtype=np.float32
+ )
+ dummy_cliff = np.zeros((max_batch_size, 3), dtype=np.float32)
+ # Large batch first — allocates the most memory; batch=1 reuses the plan.
+ self.session.run(
+ None, {self._in_image: dummy_image, self._in_cliff: dummy_cliff}
+ )
+ if max_batch_size > 1:
+ self.session.run(
+ None,
+ {self._in_image: dummy_image[:1], self._in_cliff: dummy_cliff[:1]},
+ )
+
# ------------------------------------------------------------------
# Preprocessing
# ------------------------------------------------------------------
diff --git a/instanthmr/pipeline.py b/instanthmr/pipeline.py
index a4c23a7..c3e8ddd 100644
--- a/instanthmr/pipeline.py
+++ b/instanthmr/pipeline.py
@@ -83,7 +83,7 @@ class PosePipeline:
device: str = "cuda",
detector_variant: str = "medium",
det_confidence: float = 0.5,
- max_persons: int = 5,
+ max_persons: int = 2,
detector_stride: int = 1,
batch_persons: bool = True,
):
@@ -97,6 +97,18 @@ class PosePipeline:
max_persons=max_persons,
)
+ # Warm up both models before the first real frame.
+ # - RF-DETR (PyTorch): first call triggers CUDA JIT for transformer ops.
+ # - InstantHMR (ONNX/CUDA EP): first call at a new batch size triggers
+ # kernel compilation and GPU memory allocation.
+ # With padded-batch inference the session only ever sees two shapes:
+ # batch=1 and batch=max_persons — two warm-up calls covers everything.
+ print(f"Warming up models (max_persons={max_persons})…", flush=True)
+ self.detector.warmup()
+ self.hmr.warmup(max_batch_size=max_persons)
+ print("Ready.", flush=True)
+
+ self._max_persons = max_persons
self._detector_stride = detector_stride
self._batch_persons = batch_persons
self._frame_idx = 0
@@ -138,11 +150,17 @@ class PosePipeline:
detections = [{"bbox": fb, "confidence": 0.0}]
# ---- InstantHMR ----
+ # Hard-cap so we never exceed the pre-warmed batch size.
+ detections = detections[: self._max_persons]
outputs: list[HMRPrediction] = []
t1 = time.perf_counter()
if detections:
if self._batch_persons and len(detections) > 1:
- outputs = self.hmr.predict_batch(image_rgb, detections)
+ # Always pad to _max_persons so ONNX sees a constant batch
+ # shape and never triggers a mid-demo recompilation.
+ outputs = self.hmr.predict_batch(
+ image_rgb, detections, padded_to=self._max_persons
+ )
else:
for det in detections:
outputs.append(self.hmr.predict(