Python
feat: added smoothing filter and IoU identification for multi-person filtering
mohamdevfa5001616403
diff --git a/datasets_pipeline/backfill_originals.py b/datasets_pipeline/backfill_originals.pynew file mode 100644index 0000000..2bf0962--- /dev/null+++ b/datasets_pipeline/backfill_originals.py@@ -0,0 +1,125 @@+#!/usr/bin/env python3+"""Populate original_images/ for a split that was built with --originals none.++build_split.py skips a row whose crop already exists (`skipped_existing`) BEFORE+it reaches place_original, so re-running the build with a different --originals+backfills nothing. This walks the finished annotations instead: every npz stores+`image_relpath`, the exact source path the crop came from, so the originals can+be placed without re-decoding a single image.++ python backfill_originals.py \+ --dir $DATA_ROOT/sam3d_gt_aic \+ --image-dir $IMG_ROOT/aic \+ --mode hardlink++hardlink is the right default when the split and the source frames sit on the+same filesystem: one directory entry, zero extra data blocks, and the result is+self-contained if the source tree is later deleted. It falls back to a copy+across filesystems. Use link (symlink) when the source is a permanent read-only+mirror such as $DSDIR, and copy when you want the split to stand alone.+"""+from __future__ import annotations++import argparse+import sys+from collections import Counter+from concurrent.futures import ThreadPoolExecutor+from pathlib import Path++import numpy as np+from tqdm import tqdm++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))++from build_split import place_original # noqa: E402+++def relpaths_from_annotations(ann_dir: Path, workers: int) -> set[str]:+ """The unique set of source frames a split's npz files point at."""+ files = sorted(ann_dir.glob("*.npz"))+ if not files:+ raise SystemExit(f"no .npz under {ann_dir}")++ def one(f: Path) -> str | None:+ try:+ with np.load(f, allow_pickle=True) as d:+ if "image_relpath" not in d.files:+ return None+ return str(d["image_relpath"])+ except Exception: # noqa: BLE001+ return None++ out: set[str] = set()+ missing_key = 0+ with ThreadPoolExecutor(workers) as ex:+ for rp in tqdm(ex.map(one, files), total=len(files),+ desc="reading annotations", unit="npz"):+ if rp is None:+ missing_key += 1+ else:+ out.add(rp)+ if missing_key:+ print(f" {missing_key:,} npz had no image_relpath field "+ f"(built by an older script — those frames cannot be resolved)")+ return out+++def main() -> None:+ p = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument("--dir", required=True, type=Path,+ help="a built split, e.g. $DATA_ROOT/sam3d_gt_aic")+ p.add_argument("--image-dir", required=True, type=Path,+ help="root the image_relpath values are relative to")+ p.add_argument("--mode", choices=("copy", "link", "hardlink"),+ default="hardlink")+ p.add_argument("--workers", type=int, default=16)+ p.add_argument("--dry-run", action="store_true",+ help="report what would be placed, write nothing")+ args = p.parse_args()++ ann_dir = args.dir / "annotations"+ dst_root = args.dir / "original_images"+ if not ann_dir.is_dir():+ raise SystemExit(f"{ann_dir} does not exist")++ rels = relpaths_from_annotations(ann_dir, args.workers)+ print(f"\n{len(rels):,} unique source frames referenced")++ st = Counter()++ def place(rel: str) -> str:+ src, dst = args.image_dir / rel, dst_root / rel+ if dst.exists():+ return "already_present"+ if not src.exists():+ return "missing_source"+ if args.dry_run:+ return "would_place"+ try:+ place_original(src, dst, args.mode)+ return "placed"+ except Exception as e: # noqa: BLE001+ tqdm.write(f"[fail] {rel}: {e}")+ return "failed"++ with ThreadPoolExecutor(args.workers) as ex:+ for r in tqdm(ex.map(place, sorted(rels)), total=len(rels),+ desc=f"{args.mode} originals", unit="img"):+ st[r] += 1++ print("\n" + "=" * 62)+ print(f" {args.dir.name} ({args.mode}{', DRY RUN' if args.dry_run else ''})")+ for k in ("placed", "would_place", "already_present",+ "missing_source", "failed"):+ if st[k]:+ print(f" {k:<18}: {st[k]:,}")+ print("=" * 62)+ if st["missing_source"]:+ print(f" {st['missing_source']:,} frames are not under {args.image_dir} —"+ f" check that --image-dir matches the build's --image-dir.")+++if __name__ == "__main__":+ main()diff --git a/datasets_pipeline/build_split.py b/datasets_pipeline/build_split.pyindex 6a908a2..bb9986b 100755--- a/datasets_pipeline/build_split.py+++ b/datasets_pipeline/build_split.py@@ -282,6 +282,15 @@ def main() -> None:name = f"{dataset}_{stem}_p{int(row['person_id'])}"if name in existing:st["skipped_existing"] += 1+ # Still place the original: a resume with a different+ # --originals must be able to backfill original_images/,+ # otherwise a split built with `none` can never gain them.+ if args.originals != "none":+ rp = image_relpath(dataset, image)+ if index is not None:+ rp = index.resolve(image) or rp+ place_original(args.image_dir / rp,+ dirs["original_images"] / rp, args.originals)continueif not args.keep_invalid and not bool(row["mhr_valid"]):st["skipped_invalid"] += 1diff --git a/datasets_pipeline/registry.py b/datasets_pipeline/registry.pyindex 201d6f6..8e25cc3 100755--- a/datasets_pipeline/registry.py+++ b/datasets_pipeline/registry.py@@ -103,14 +103,21 @@ DATASETS: dict[str, Dataset] = {name="aic",splits=("aic_train",),method="opendatalab",- approx_gb=36.4,- notes="AI Challenger 2017. The original challenger.ai host is dead; the "- "working route is OpenDataLab, which is where this corpus came "- "from last time (see ~/sam3d_data/aic_download).\n"- " pip install opendatalab && odl login\n"- " odl get OpenDataLab/AI_Challenger -d $DEST\n"- " tar xzf $DEST/OpenDataLab___AI_Challenger/raw/AI_Challenger.tar.gz\n"- "Needs the train/images/ subtree only.",+ approx_gb=16.0,+ notes="AI Challenger 2017. Only the train/images/ subtree is needed: "+ "210,000 jpgs, 16 GB. The full OpenDataLab archive is 36 GB but "+ "the rest is val, eval scripts and duplicate annotation copies.\n"+ "The original challenger.ai host is dead. If you already hold a "+ "copy, rsync train/ straight in - that is the reliable route.\n"+ "Otherwise OpenDataLab, in an ISOLATED venv: openxlab pins old "+ "requests/urllib3/tqdm and will break huggingface_hub if it "+ "shares an environment with fetch_annotations.py.\n"+ " python -m venv ~/venvs/odl && ~/venvs/odl/bin/pip install opendatalab\n"+ " ~/venvs/odl/bin/odl login\n"+ " ~/venvs/odl/bin/odl get AI_Challenger -d $DEST\n"+ "The dataset name is bare - `odl get OpenDataLab/AI_Challenger` "+ "fails with `No dataset: OpenDataLab`; the org/name form is an "+ "openxlab convention, not an odl one.",),"3dpw": Dataset(name="3dpw",diff --git a/demo.py b/demo.pyindex 82a9995..5a67d25 100755--- a/demo.py+++ b/demo.py@@ -44,6 +44,7 @@ Usagefrom __future__ import annotationsimport argparse+from collections import dequeimport sysfrom pathlib import Path@@ -52,6 +53,7 @@ import numpy as npfrom instanthmr import PosePipeline, onnx_output_namesfrom instanthmr.mhr_renderer import DEFAULT_MHR_SCRIPT+from instanthmr.smoothing import TemporalSmootherfrom instanthmr.visualizer import RerunVisualizer@@ -107,6 +109,23 @@ def parse_args() -> argparse.Namespace:"--no-batch-persons", action="store_true",help="Disable batched multi-person HMR (one ONNX call per person).",)+ p.add_argument(+ "--smoothing-filter", action="store_true",+ help="Temporally smooth per-person predictions with a moving-average "+ "filter (see instanthmr/smoothing.py). Non-learned and free; NLF "+ "reports a 5-frame filter of this kind giving 3DPW 59.0 -> 57.2 mm.",+ )+ p.add_argument(+ "--smoothing-window", type=int, default=5, metavar="N",+ help="Frames averaged by --smoothing-filter (default: 5). Rounded up to "+ "the next odd number in centered mode.",+ )+ p.add_argument(+ "--smoothing-mode", choices=("centered", "causal"), default="centered",+ help="'centered' has no phase lag but delays output by (N-1)//2 frames; "+ "'causal' is a trailing average with zero latency. --camera always "+ "uses causal (default: centered).",+ )p.add_argument("--save-rrd", type=str, default=None,help="Optional path to save the Rerun recording as a .rrd file.",@@ -308,6 +327,17 @@ def _print_timings(# ---------------------------------------------------------------------------+def build_smoother(args: argparse.Namespace, force_causal: bool = False):+ """Construct the temporal filter, or ``None`` when --smoothing-filter is off."""+ if not args.smoothing_filter:+ return None+ mode = "causal" if force_causal else args.smoothing_mode+ sm = TemporalSmoother(window=args.smoothing_window, mode=mode)+ lag = f", {sm.lag} frame lag" if sm.lag else ""+ print(f"temporal smoothing: window={sm.window}, mode={sm.mode}{lag}")+ return sm++def run_image(args: argparse.Namespace) -> None:image_path = Path(args.image)if not image_path.exists():@@ -350,6 +380,7 @@ def run_video(args: argparse.Namespace) -> None:print(f"video: {video_path.name}, {total} frames @ {fps:.1f} fps")pipeline, viz = build_all(args)+ smoother = build_smoother(args)frame_idx = 0sent = 0@@ -357,6 +388,23 @@ def run_video(args: argparse.Namespace) -> None:hmr_times: list[float] = []render_times: list[float] = []tot_times: list[float] = []++ # When smoothing is centred the filter emits frame t only after it has seen+ # t + (window-1)//2, so the frames still in flight wait here with the data+ # the visualizer will need for them.+ pending: deque[tuple[int, np.ndarray, float, float, float]] = deque()++ def emit(persons) -> None:+ f_idx, f_rgb, d_ms, h_ms, t_ms = pending.popleft()+ render_times.append(viz.log_frame(+ f_rgb, persons,+ frame_idx=f_idx,+ timestamp=f_idx / fps,+ detector_ms=d_ms,+ hmr_ms=h_ms,+ total_ms=t_ms,+ ))+try:while True:ret, bgr = cap.read()@@ -374,15 +422,14 @@ def run_video(args: argparse.Namespace) -> None:hmr_times.append(result.hmr_ms)tot_times.append(result.total_ms)- render_ms = viz.log_frame(- rgb, result.persons,- frame_idx=frame_idx,- timestamp=frame_idx / fps,- detector_ms=result.detector_ms,- hmr_ms=result.hmr_ms,- total_ms=result.total_ms,- )- render_times.append(render_ms)+ pending.append((frame_idx, rgb, result.detector_ms,+ result.hmr_ms, result.total_ms))+ if smoother is None:+ emit(result.persons)+ else:+ smoothed = smoother.push(result.persons)+ if smoothed is not None:+ emit(smoothed)sent += 1frame_idx += 1@@ -396,11 +443,19 @@ def run_video(args: argparse.Namespace) -> None:float(np.mean(tot_times[-30:])),len(result.persons),)+ if smoother is not None:+ for smoothed in smoother.flush():+ emit(smoothed)except KeyboardInterrupt:print("\ninterrupted")finally:cap.release()+ if smoother is not None:+ report = smoother.jitter_report()+ if report:+ print("\n" + report)+if tot_times:# Skip the first frame from the average — it bears the warm-up cost.warm = slice(1, None) if len(tot_times) > 1 else slice(None)@@ -428,6 +483,7 @@ def run_camera(args: argparse.Namespace) -> None:print(f"camera {args.camera}: {width}x{height} @ {fps:.1f} fps — Ctrl+C to stop")pipeline, viz = build_all(args)+ smoother = build_smoother(args, force_causal=True)frame_idx = 0det_times: list[float] = []@@ -447,8 +503,12 @@ def run_camera(args: argparse.Namespace) -> None:hmr_times.append(result.hmr_ms)tot_times.append(result.total_ms)+ persons = result.persons+ if smoother is not None:+ persons = smoother.push(persons) or persons+render_ms = viz.log_frame(- rgb, result.persons,+ rgb, persons,frame_idx=frame_idx,timestamp=frame_idx / fps,detector_ms=result.detector_ms,diff --git a/instanthmr/smoothing.py b/instanthmr/smoothing.pynew file mode 100644index 0000000..49c1537--- /dev/null+++ b/instanthmr/smoothing.py@@ -0,0 +1,270 @@+"""Temporal smoothing of per-person HMR predictions.++InstantHMR is a frame-by-frame model: every frame is estimated independently,+so its outputs carry per-frame noise that reads as jitter in a video — most+visibly in ``cam_trans``, which is a free 3-vector from a linear head with+almost nothing tying it to the observed pixels.++A non-learned moving-average filter over a short window removes much of that.+NLF (arXiv:2407.07532, Tables 2-3) reports a 5-frame filter of this kind+improving 3DPW MPJPE 59.0 -> 57.2 mm and EMDB 68.4 -> 66.7 mm, at zero+training cost. This module is that filter.++Two things the filter needs that the pipeline does not provide:++* **Identity across frames.** ``PosePipeline.predict`` returns an unordered+ list of detections per frame, so a person's index is not stable. We attach a+ minimal greedy IoU tracker (:class:`_Tracker`) purely to decide which+ predictions belong to the same person before averaging them.+* **Lookahead**, for ``mode="centered"``. A centred window has no phase lag+ but needs ``(window - 1) // 2`` future frames, so :meth:`TemporalSmoother.push`+ returns results delayed by that many frames and :meth:`flush` drains the+ tail. Use ``mode="causal"`` for live sources, which trades phase lag for+ zero latency.++Caveat on ``mhr_params``: these are MHR joint parameters (angles and scales),+and this filter averages them linearly. That is well-behaved for continuous+motion but is *wrong* across an angle wraparound — a global yaw crossing +/-pi+will be averaged toward zero instead of through the discontinuity, producing a+brief flip. Short windows on real footage rarely hit it; if you see one, drop+``mhr_params`` from ``SMOOTHED_FIELDS`` and accept a mesh that jitters while+the keypoints do not.+"""++from __future__ import annotations++from collections import deque+from dataclasses import replace+from typing import Iterator, Sequence++import numpy as np++from .inference import HMRPrediction++#: Fields averaged across the window. All are Euclidean quantities for which a+#: linear mean is the right operation, except ``mhr_params`` — see the module+#: docstring. ``joints_3d_cam`` is deliberately absent: it is recomputed from+#: the smoothed ``joints_3d_local`` and ``cam_trans`` so the three stay+#: consistent with each other.+SMOOTHED_FIELDS = (+ "joints_3d_local",+ "cam_trans",+ "joints_2d",+ "mhr_params",+ "shape_params",+)+++def _iou(a: np.ndarray, b: np.ndarray) -> float:+ """Intersection-over-union of two ``[x1, y1, x2, y2]`` boxes."""+ x1 = max(float(a[0]), float(b[0]))+ y1 = max(float(a[1]), float(b[1]))+ x2 = min(float(a[2]), float(b[2]))+ y2 = min(float(a[3]), float(b[3]))+ iw = max(0.0, x2 - x1)+ ih = max(0.0, y2 - y1)+ inter = iw * ih+ if inter <= 0.0:+ return 0.0+ area_a = max(0.0, float(a[2] - a[0])) * max(0.0, float(a[3] - a[1]))+ area_b = max(0.0, float(b[2] - b[0])) * max(0.0, float(b[3] - b[1]))+ union = area_a + area_b - inter+ return inter / union if union > 0.0 else 0.0+++class _Tracker:+ """Greedy IoU association of detections to persistent track ids.++ Deliberately minimal — it exists only to group predictions of the same+ person before averaging, not to be a general-purpose tracker. Tracks+ survive ``max_age`` frames without a match so a brief miss does not restart+ the filter (and reset the smoothing) for that person.+ """++ def __init__(self, iou_threshold: float = 0.3, max_age: int = 5):+ self.iou_threshold = iou_threshold+ self.max_age = max_age+ self._boxes: dict[int, np.ndarray] = {}+ self._age: dict[int, int] = {}+ self._next_id = 0++ def assign(self, persons: Sequence[HMRPrediction]) -> list[int]:+ """Return one track id per person, in the order given."""+ ids: list[int] = [-1] * len(persons)+ free = set(self._boxes)++ # Score every (detection, track) pair, then take them best-first. With+ # the handful of people this pipeline handles, an O(n*m) sort is far+ # cheaper than pulling in a Hungarian solver.+ pairs = [+ (_iou(p.bbox, self._boxes[t]), i, t)+ for i, p in enumerate(persons)+ for t in free+ ]+ pairs.sort(reverse=True)+ taken: set[int] = set()+ for score, i, t in pairs:+ if score < self.iou_threshold:+ break+ if ids[i] != -1 or t not in free or i in taken:+ continue+ ids[i] = t+ free.discard(t)+ taken.add(i)++ for i, p in enumerate(persons):+ if ids[i] == -1:+ ids[i] = self._next_id+ self._next_id += 1+ self._boxes[ids[i]] = np.asarray(p.bbox, dtype=np.float64)+ self._age[ids[i]] = 0++ for t in list(free):+ self._age[t] = self._age.get(t, 0) + 1+ if self._age[t] > self.max_age:+ self._boxes.pop(t, None)+ self._age.pop(t, None)+ return ids+++class TemporalSmoother:+ """Moving-average filter over per-person predictions.++ Args:+ window: number of frames averaged. Forced odd in ``"centered"`` mode.+ mode: ``"centered"`` (no phase lag, ``(window-1)//2`` frames of+ latency) or ``"causal"`` (trailing average, zero latency).+ iou_threshold: minimum IoU to associate a detection with a track.+ max_age: frames a track survives without a detection.++ Usage mirrors a delay line::++ for persons in stream:+ out = smoother.push(persons)+ if out is not None:+ render(out) # a frame (window-1)//2 back+ for out in smoother.flush():+ render(out) # the tail+ """++ def __init__(+ self,+ window: int = 5,+ mode: str = "centered",+ iou_threshold: float = 0.3,+ max_age: int = 5,+ ):+ if mode not in ("centered", "causal"):+ raise ValueError(f"mode must be 'centered' or 'causal', got {mode!r}")+ if window < 1:+ raise ValueError(f"window must be >= 1, got {window}")+ if mode == "centered" and window % 2 == 0:+ window += 1+ self.window = window+ self.mode = mode+ self.lag = (window - 1) // 2 if mode == "centered" else 0++ self._tracker = _Tracker(iou_threshold=iou_threshold, max_age=max_age)+ self._buf: deque[dict[int, HMRPrediction]] = deque(maxlen=window)+ # Frames in and frames out. Every pushed frame is emitted exactly once,+ # so a caller holding its own delay line can pop one entry per returned+ # frame and stay in step with the video.+ self._n_pushed = 0+ self._n_emitted = 0++ # Jitter accounting, so the effect is measurable rather than only+ # visible. Both are mean frame-to-frame deltas over the same tracks.+ self._prev_raw: dict[int, HMRPrediction] = {}+ self._prev_out: dict[int, HMRPrediction] = {}+ self._raw_cam: list[float] = []+ self._raw_j3d: list[float] = []+ self._out_cam: list[float] = []+ self._out_j3d: list[float] = []++ # ------------------------------------------------------------------+ def push(self, persons: Sequence[HMRPrediction]) -> list[HMRPrediction] | None:+ """Feed one frame. Returns a smoothed frame, or ``None`` while filling."""+ ids = self._tracker.assign(persons)+ frame = {t: p for t, p in zip(ids, persons)}+ self._accumulate(frame, self._prev_raw, self._raw_cam, self._raw_j3d)+ self._prev_raw = frame++ self._buf.append(frame)+ self._n_pushed += 1++ # Emit frame ``_n_emitted`` as soon as ``lag`` frames after it have+ # arrived. Near the start of the stream that window is one-sided rather+ # than centred — the usual boundary treatment for a moving average, and+ # what keeps frames-in equal to frames-out.+ if self._n_pushed - 1 < self._n_emitted + self.lag:+ return None+ return self._emit_next()++ def flush(self) -> Iterator[list[HMRPrediction]]:+ """Drain the frames still held in the delay line at end of stream."""+ while self._n_emitted < self._n_pushed:+ yield self._emit_next()++ def _emit_next(self) -> list[HMRPrediction]:+ buf_start = self._n_pushed - len(self._buf)+ out = self._emit(self._n_emitted - buf_start)+ self._n_emitted += 1+ return out++ # ------------------------------------------------------------------+ def _emit(self, centre: int) -> list[HMRPrediction]:+ """Average every track present in frame ``centre`` over the buffer."""+ out: list[HMRPrediction] = []+ emitted: dict[int, HMRPrediction] = {}+ for track_id, anchor in self._buf[centre].items():+ samples = [f[track_id] for f in self._buf if track_id in f]+ fields = {+ name: np.mean(+ np.stack([np.asarray(getattr(s, name), dtype=np.float64)+ for s in samples]),+ axis=0,+ ).astype(np.float32)+ for name in SMOOTHED_FIELDS+ }+ fields["joints_3d_cam"] = (+ fields["joints_3d_local"] + fields["cam_trans"][None, :]+ ).astype(np.float32)+ smoothed = replace(anchor, **fields)+ out.append(smoothed)+ emitted[track_id] = smoothed++ self._accumulate(emitted, self._prev_out, self._out_cam, self._out_j3d)+ self._prev_out = emitted+ return out++ @staticmethod+ def _accumulate(cur, prev, cam_acc: list[float], j3d_acc: list[float]) -> None:+ """Record mean frame-to-frame movement for tracks seen in both frames."""+ for track_id, p in cur.items():+ q = prev.get(track_id)+ if q is None:+ continue+ cam_acc.append(float(np.linalg.norm(p.cam_trans - q.cam_trans)))+ j3d_acc.append(+ float(np.linalg.norm(p.joints_3d_local - q.joints_3d_local, axis=-1).mean())+ )++ # ------------------------------------------------------------------+ def jitter_report(self) -> str | None:+ """One-line before/after summary, or ``None`` if nothing was measured.++ These are mean per-frame *movement* magnitudes, not errors: real motion+ contributes to both columns. The filter is doing its job when the+ numbers drop without the pose visibly lagging the video.+ """+ if not self._raw_cam or not self._out_cam:+ return None+ rc, oc = float(np.mean(self._raw_cam)), float(np.mean(self._out_cam))+ rj, oj = float(np.mean(self._raw_j3d)), float(np.mean(self._out_j3d))+ return (+ f"smoothing (window={self.window}, {self.mode}) — mean frame-to-frame delta\n"+ f" cam_trans {rc * 1000:7.2f} -> {oc * 1000:7.2f} mm "+ f"({100.0 * (1.0 - oc / rc) if rc > 0 else 0.0:+.0f}%)\n"+ f" joints_3d_local {rj * 1000:7.2f} -> {oj * 1000:7.2f} mm "+ f"({100.0 * (1.0 - oj / rj) if rj > 0 else 0.0:+.0f}%)"+ )