Skip to content
Python

feat: --preset v4 trains on precomputed windows cut from the originals like the runtime crop (build_split --windows), with baseline losses; also --v3-lanczos.

mohamdev5e4c1792af72
diff --git a/datasets_pipeline/build_split.py b/datasets_pipeline/build_split.py
index dcff254..2805fda 100755
--- a/datasets_pipeline/build_split.py
+++ b/datasets_pipeline/build_split.py
@@ -53,6 +53,26 @@ byte-identical to HEAD; the 1.2x window cut back out of a 2.0x / 448 crop
matches today's 224 crop at 42.2 dB mean (sigma-1 blur, padding masked) with
a 0.09 px shift. At 2.0x, 195 of 200 AIC squares reach past the frame edge.
+`--windows K --window-sizes 224,288` stores PRECOMPUTED WINDOWS instead (the
+v4 preset): K square windows per person, each cut from the full-resolution
+photo with ONE resize -- the same operation as the runtime crop, so the training
+texture is the deployment texture (a 448 px context crop plus a second warp
+was not: STATUS.md 2026-10-21). Window 0 is the 1.2x square of the tight box
+with the RUNTIME's rounding (floor / ceil, instanthmr/inference.py), i.e. the
+crop the phone cuts for a perfect detector box -- within 1 px of bbox_square,
+whose int() edges the runtime does not use; windows 1..K-1 are v3 framings (side log-uniform in
+0.6-2.0 x the tight box, centre shifted up to +/-15% of the side), padded black
+only where they leave the photo. Each size is written to its own folder
+(--output-dir with a `{size}` placeholder) as one tile per person, K windows
+side by side (K*S x S px). One key is added:
+
+ win_boxes (K, 4) the windows, full-frame px, integer; rows 1.. are
+ exact squares, row 0 is the runtime box (which its
+ floor / ceil can leave 1 px off square)
+
+The boxes are seeded by the person's name, so every size, rerun and resume
+draws the same ones.
+
Hand crops are stored UNFLIPPED. SAM3D trains a single right-hand model and
mirrors the left at inference; flipping a loaded crop is a free numpy slice, so
baking the mirror into disk would only remove a choice.
@@ -65,10 +85,12 @@ baking the mirror into disk would only remove a choice.
from __future__ import annotations
import argparse
+import math
import os
import re
import shutil
import sys
+import zlib
from pathlib import Path
import cv2
@@ -163,6 +185,44 @@ def enough_inside(sq: np.ndarray, h: int, w: int, min_frac: float) -> bool:
return ((ix2 - ix1) * (iy2 - iy1)) / max(area, 1e-6) >= min_frac
+# v3's framing range, as multiples of the tight box (1.2 = the standard crop),
+# and its centre shift as a fraction of the window side.
+WINDOW_BOX = (0.6, 2.0)
+WINDOW_SHIFT = 0.15
+
+
+def window_boxes(tight: np.ndarray, k: int, name: str) -> np.ndarray:
+ """Row 0 is the runtime's 1.2x box of `tight`; rows 1..k-1 are random v3
+ framings, integer and exactly square, in frame px. Seeded by `name`, so
+ reruns agree."""
+ rng = np.random.default_rng(zlib.crc32(name.encode()))
+ x1, y1, x2, y2 = (float(v) for v in tight)
+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0
+ b = max(x2 - x1, y2 - y1)
+ # instanthmr.inference.InstantHMR._preprocess, line for line
+ sq_size = b * 1.2
+ sx, sy = cx - sq_size / 2.0, cy - sq_size / 2.0
+ out = [np.array([math.floor(sx), math.floor(sy), math.ceil(sx + sq_size),
+ math.ceil(sy + sq_size)], dtype=np.float32)]
+ for _ in range(k - 1):
+ m = math.exp(rng.uniform(math.log(WINDOW_BOX[0]), math.log(WINDOW_BOX[1])))
+ side = max(8, int(round(m * b)))
+ wx = cx + rng.uniform(-WINDOW_SHIFT, WINDOW_SHIFT) * side
+ wy = cy + rng.uniform(-WINDOW_SHIFT, WINDOW_SHIFT) * side
+ x1, y1 = int(round(wx - side / 2.0)), int(round(wy - side / 2.0))
+ out.append(np.array([x1, y1, x1 + side, y1 + side], dtype=np.float32))
+ return np.stack(out)
+
+
+def cut_box(img_rgb: np.ndarray, box: np.ndarray) -> np.ndarray:
+ """The integer box `box` of the photo, black-padded where it leaves it."""
+ h, w = img_rgb.shape[:2]
+ x1, y1, x2, y2 = (int(v) for v in box)
+ crop = img_rgb[max(0, y1):min(h, y2), max(0, x1):min(w, x2)]
+ return cv2.copyMakeBorder(crop, max(0, -y1), max(0, y2 - h), max(0, -x1),
+ max(0, x2 - w), cv2.BORDER_CONSTANT, value=(0, 0, 0))
+
+
def write_crop(img_rgb, box, size, path: Path) -> None:
crop, _ = get_square_crop_padded(img_rgb, box, expand=1.0)
crop = cv2.resize(crop, (size, size), interpolation=cv2.INTER_LINEAR)
@@ -203,6 +263,13 @@ def parse_args() -> argparse.Namespace:
"as bbox_context (full-frame px) plus context_padded; "
"bbox_square stays the 1.2x square, so the standard window "
"can always be cut back out.")
+ p.add_argument("--windows", type=int, default=0, metavar="K",
+ help="store K precomputed windows per person (window 0 = the "
+ "standard 1.2x crop, 1..K-1 = v3 framings), each cut from "
+ "the photo with one resize. Needs --window-sizes, "
+ "--no-hands and an --output-dir containing {size}.")
+ p.add_argument("--window-sizes", default="224,288",
+ help="with --windows: comma-separated tile sizes, one folder each")
p.add_argument("--crop-format", choices=("png", "jpg"), default="png",
help="png keeps body_crops byte-identical to the existing "
"data/ tree; jpg is ~5x smaller and ~4x faster to decode")
@@ -257,30 +324,44 @@ def main() -> None:
if args.context != 1.2 and args.output_dir is None:
sys.exit(f"--context {args.context} needs an explicit --output-dir, so the "
f"context crops never land in the standard {out_dir.name}")
- first = next((out_dir / "annotations").glob("*.npz"), None)
- if first is not None:
- with np.load(first) as z:
- was_context = "bbox_context" in z.files
- if was_context != (args.context != 1.2):
- sys.exit(f"{out_dir} already holds {'context' if was_context else 'standard'} "
- f"crops; refusing to add {'standard' if was_context else 'context'} ones")
+ win_sizes = [int(v) for v in args.window_sizes.split(",")] if args.windows else []
+ if args.windows:
+ if args.windows < 2 or args.context != 1.2 or args.hands:
+ sys.exit("--windows needs K >= 2, the default --context and --no-hands")
+ if args.output_dir is None or "{size}" not in str(args.output_dir):
+ sys.exit("--windows needs an --output-dir containing {size}, e.g. "
+ "$SCRATCH/instanthmr/data_win{size}/sam3d_gt_aic")
+ out_dirs = [Path(str(args.output_dir).format(size=s)) for s in win_sizes]
+ else:
+ out_dirs = [out_dir]
+ out_dir = out_dirs[0]
+ kind = "windows" if args.windows else "context" if args.context != 1.2 else "standard"
+ for d in out_dirs:
+ first = next((d / "annotations").glob("*.npz"), None)
+ if first is not None:
+ with np.load(first) as z:
+ was = ("windows" if "win_boxes" in z.files else
+ "context" if "bbox_context" in z.files else "standard")
+ if was != kind:
+ sys.exit(f"{d} already holds {was} crops; refusing to add {kind} ones")
+
+ for d in out_dirs:
+ for k in ("original_images", "annotations", "body_crops", "hands_crops"):
+ if k == "original_images" and (args.originals == "none" or d != out_dir):
+ continue
+ if k == "hands_crops" and not args.hands:
+ continue
+ (d / k).mkdir(parents=True, exist_ok=True)
+ # Let the unmodified training script discover this folder.
+ compat = d / "images"
+ if not compat.exists():
+ try:
+ os.symlink("body_crops", compat)
+ except FileExistsError:
+ pass # another array task won the race
dirs = {k: out_dir / k for k in
("original_images", "annotations", "body_crops", "hands_crops")}
- for k, d in dirs.items():
- if k == "original_images" and args.originals == "none":
- continue
- if k == "hands_crops" and not args.hands:
- continue
- d.mkdir(parents=True, exist_ok=True)
-
- # Let the unmodified training script discover this folder.
- compat = out_dir / "images"
- if not compat.exists():
- try:
- os.symlink("body_crops", compat)
- except FileExistsError:
- pass # another array task won the race
parquets = sorted(args.annotation_dir.glob("*.parquet"))
if not parquets:
@@ -299,7 +380,8 @@ def main() -> None:
ext = args.crop_format
imwrite_params = ([int(cv2.IMWRITE_JPEG_QUALITY), args.jpg_quality]
if ext == "jpg" else [])
- existing = {p.stem for p in dirs["annotations"].glob("*.npz")}
+ # The last folder's npz is written last, so it marks a finished person.
+ existing = {p.stem for p in (out_dirs[-1] / "annotations").glob("*.npz")}
total = len(existing)
st = {k: 0 for k in ("crops", "hands_l", "hands_r", "hand_too_small",
"hand_out_of_frame", "hand_occluded", "skipped_existing",
@@ -378,11 +460,18 @@ def main() -> None:
ann["context_padded"] = np.array(
bool(ctx_bbox[0] < 0 or ctx_bbox[1] < 0
or ctx_bbox[2] > w or ctx_bbox[3] > h))
- body_crop = cv2.resize(body_crop, (args.body_size, args.body_size),
- interpolation=cv2.INTER_LINEAR)
- cv2.imwrite(str(dirs["body_crops"] / f"{name}.{ext}"),
- cv2.cvtColor(body_crop, cv2.COLOR_RGB2BGR),
- imwrite_params)
+ if args.windows:
+ ann["win_boxes"] = window_boxes(ann["bbox"], args.windows, name)
+ crops = [cut_box(img_rgb, b) for b in ann["win_boxes"]]
+ tiles = [np.hstack([cv2.resize(c, (s, s), interpolation=cv2.INTER_LINEAR)
+ for c in crops]) for s in win_sizes]
+ else:
+ tiles = [cv2.resize(body_crop, (args.body_size, args.body_size),
+ interpolation=cv2.INTER_LINEAR)]
+ for d, tile in zip(out_dirs, tiles):
+ cv2.imwrite(str(d / "body_crops" / f"{name}.{ext}"),
+ cv2.cvtColor(tile, cv2.COLOR_RGB2BGR),
+ imwrite_params)
ann["bbox_square"] = sq_bbox
ann["image_relpath"] = relpath
@@ -419,7 +508,8 @@ def main() -> None:
ann[f"hand_valid_{side}"] = np.array(True)
st[f"hands_{side}"] += 1
- np.savez(dirs["annotations"] / f"{name}.npz", **ann)
+ for d in out_dirs:
+ np.savez(d / "annotations" / f"{name}.npz", **ann)
if args.originals != "none":
place_original(args.image_dir / relpath,
dirs["original_images"] / relpath,
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.py
index 49f55f2..a47285a 100755
--- a/instanthmr_distill_train/train_distill_jz.py
+++ b/instanthmr_distill_train/train_distill_jz.py
@@ -397,7 +397,8 @@ def build_jz_loaders(cfg, args, rank: int, world: int):
jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality,
image_size=cfg.image_size,
context_crops=cfg.context_crops, v3_box_min=cfg.v3_box_min,
- v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift)
+ v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift,
+ v3_lanczos=cfg.v3_lanczos, crop_windows=cfg.crop_windows)
# Rank 0 builds the index (or reads the cache) while the others wait, then
# everyone loads the same cache. Building on all ranks at once would put
@@ -710,6 +711,7 @@ def train(args, cfg, rank, local_rank, world):
"backbone": cfg.backbone, "w_reproj": cfg.w_reproj,
"image_size": cfg.image_size, "context_crops": cfg.context_crops,
"v3_box": [cfg.v3_box_min, cfg.v3_box_max], "v3_shift": cfg.v3_shift,
+ "v3_lanczos": cfg.v3_lanczos, "crop_windows": cfg.crop_windows,
"cont_head": cfg.cont_head, "root_rot_loss": cfg.root_rot_loss,
"exact_landmarks": cfg.exact_landmarks,
"geom_scale_max": cfg.geom_scale_max, "occl_p": cfg.occl_p,
@@ -1144,6 +1146,10 @@ def parse_args():
p.add_argument("--w_simcc", type=float, default=None)
p.add_argument("--weight_decay", type=float, default=None)
p.add_argument("--backbone", default=None)
+ p.add_argument("--v3-lanczos", dest="v3_lanczos", action="store_true",
+ help="--preset v3: cut the training window with Lanczos instead of "
+ "bilinear, so training crops are as sharp as val/phone crops. "
+ "Use for every new v3 run.")
p.add_argument("--image-size", dest="image_size", type=int, default=None,
help="Network input side in px (default 224). A multiple of "
"32: the decoder's positional grid is image_size // 32. "
@@ -1162,13 +1168,17 @@ def parse_args():
help="Skip the timm download. Compute nodes have no internet, "
"so run --warm-cache once on a login node instead.")
p.add_argument("--log-every", type=int, default=100)
- p.add_argument("--preset", choices=["baseline", "v2", "v3"], default="baseline",
+ p.add_argument("--preset", choices=["baseline", "v2", "v3", "v4"], default="baseline",
help="'v2' enables the wave-1 bundle: truncation, occlusion "
"and JPEG augmentation with the SimCC-mask and CLIFF "
"prerequisites, plus absolute-pose supervision. "
"'v3' is v2 with every crop sampled from a context-crop "
"split over a wide range of framings (needs --data_root "
"of build_split.py --context splits). "
+ "'v4' gives v3's framings from precomputed windows cut "
+ "like the runtime crop (needs a build_split.py --windows "
+ "corpus at --image-size), v2's occlusion/JPEG, baseline "
+ "geometry and baseline loss weights. "
"'baseline' reproduces the pre-v2 trainer exactly.")
p.add_argument("--w_reproj", type=float, default=None)
p.add_argument("--crop-centre-fix", dest="crop_centre_fix", action="store_true",
@@ -1263,6 +1273,7 @@ def build_cfg(args):
if args.w_simcc is not None: cfg.w_simcc = args.w_simcc
if args.weight_decay is not None: cfg.weight_decay = args.weight_decay
if args.ema_decay is not None: cfg.ema_decay = args.ema_decay
+ if args.v3_lanczos: cfg.v3_lanczos = True
if args.image_size is not None:
if args.image_size % 32:
raise SystemExit(f"--image-size {args.image_size} is not a multiple of 32")
@@ -1272,6 +1283,8 @@ def build_cfg(args):
apply_v2_preset(cfg)
if args.preset == "v3":
apply_v3_preset(cfg)
+ if args.preset == "v4":
+ apply_v4_preset(cfg)
# Orthogonal axis: --preset is the input pipeline + absolute-pose terms,
# --losses is the loss budget. Applied second so it owns w_keypoints3d.
if args.losses == "rebalanced":
@@ -1333,6 +1346,29 @@ def apply_v3_preset(cfg) -> None:
cfg.cliff_focal = True
+def apply_v4_preset(cfg) -> None:
+ """v3's framings without v3's texture, on baseline loss weights.
+
+ Each crop is one of K windows build_split.py --windows cut from the
+ full-resolution photo with a single resize, like the runtime crop: window 0
+ is the standard 1.2x crop, the others v3 framings. v3 cut its windows from a
+ 448 px context crop with a second warp, and its models learned that texture
+ (3DPW val J14 PA, g9_b4_224 ep314: 49.3 on crops cut from the frame vs 43.6
+ on crops built its own way; g8h 40.0 vs 39.4, STATUS.md 2026-10-21).
+
+ On top: v2's occlusion and JPEG with the two prerequisites truncated
+ framings need, and baseline geometry (the windows already supply the zoom
+ and shift). The loss weights stay the baseline ones -- no v2 item 3."""
+ cfg.crop_windows = True
+ cfg.cliff_follows_aug = True
+ cfg.mask_oob_2d = True
+ cfg.occl_p = 0.7
+ cfg.occl_scale = (0.02, 0.30)
+ cfg.jpeg_p = 0.3
+ cfg.jpeg_quality = (30, 90)
+ cfg.cliff_focal = True
+
+
def main():
args = parse_args()
diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.py
index 5a846e4..2af8922 100644
--- a/instanthmr_distill_train/train_distill_mhr_only.py
+++ b/instanthmr_distill_train/train_distill_mhr_only.py
@@ -178,6 +178,19 @@ class DistillConfig:
v3_box_min: float = 0.6
v3_box_max: float = 2.0
v3_shift: float = 0.15
+ # Lanczos instead of bilinear for the v3 window cut. Bilinear after the
+ # stored crop's own resize left v3 training crops at 0.73 (224) / 0.67 (288)
+ # of the sharpness of a crop cut straight from the photo (val3dpw and the
+ # phone do the latter); Lanczos gives 1.00 / 0.94. Measured 2026-10-20 on 300
+ # AIC people. Bilinear-trained g9 models lost 3-5 mm (224) and up to 17 mm
+ # (288) on sharp tight-box crops. Off by default so in-flight g9 runs resume
+ # unchanged.
+ v3_lanczos: bool = False
+ # --preset v4: each image is a tile of K windows that build_split.py
+ # --windows cut from the full-resolution photo with ONE resize, the same
+ # operation as the runtime crop; one window is drawn per sample (window 0,
+ # the standard 1.2x crop, when not augmenting).
+ crop_windows: bool = False
max_images: int | None = None
per_dataset_caps: dict = field(default_factory=lambda: {"sam3d_gt_harmony4d": 300_000})
val_split: float = 0.1
@@ -616,8 +629,11 @@ class SAM3DStudentDataset(Dataset):
occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),
jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90),
context_crops: bool = False, v3_box_min: float = 0.6,
- v3_box_max: float = 2.0, v3_shift: float = 0.15):
+ v3_box_max: float = 2.0, v3_shift: float = 0.15,
+ v3_lanczos: bool = False, crop_windows: bool = False):
super().__init__()
+ self.crop_windows = crop_windows
+ self.v3_interp = cv2.INTER_LANCZOS4 if v3_lanczos else cv2.INTER_LINEAR
self.image_size = image_size
self.context_crops = context_crops
self.v3_box_min, self.v3_box_max, self.v3_shift = v3_box_min, v3_box_max, v3_shift
@@ -722,7 +738,7 @@ class SAM3DStudentDataset(Dataset):
"""v3 samples from context crops. A standard 224 split would still
'work' -- with the 1.2x crop read as if it were the context, i.e. every
box silently 1.67x too large -- so refuse it up front."""
- if not self.context_crops or len(self.pairs) == 0:
+ if not (self.context_crops or self.crop_windows) or len(self.pairs) == 0:
return
# One probe per sub-folder, so a --data_root mixing context and
# standard splits fails here and not hours in, inside a worker.
@@ -737,6 +753,19 @@ class SAM3DStudentDataset(Dataset):
for k in probes:
npz = Path(self.pairs[k][1])
with np.load(npz) as z:
+ if self.crop_windows:
+ if "win_boxes" not in z.files:
+ raise ValueError(
+ f"{npz.parent.parent.name}: {npz.name} has no win_boxes. "
+ f"--preset v4 needs a window split, built with "
+ f"build_split.py --windows 4 --window-sizes 224,288 ...")
+ tw, th = Image.open(self.pairs[k][0]).size
+ if th != self.image_size or tw != th * len(z["win_boxes"]):
+ raise ValueError(
+ f"{npz.parent.parent.name}: tile {tw}x{th} does not hold "
+ f"{len(z['win_boxes'])} windows of {self.image_size} px; "
+ f"use the data_win{self.image_size} corpus")
+ continue
if "bbox_context" not in z.files:
raise ValueError(
f"{npz.parent.parent.name}: {npz.name} has no bbox_context. "
@@ -836,7 +865,7 @@ class SAM3DStudentDataset(Dataset):
[0.0, ay * ky, (0.5 * ay + ctx[1] - sq[1]) * ky - 0.5]])
A, t = M_total[:, :2], M_total[:, 2]
M_img = np.hstack([A @ C[:, :2], (A @ C[:, 2] + t)[:, None]])
- img_np = cv2.warpAffine(ctx_np, M_img, (W, H), flags=cv2.INTER_LINEAR,
+ img_np = cv2.warpAffine(ctx_np, M_img, (W, H), flags=self.v3_interp,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
return Image.fromarray(img_np), M_total, aug_active, aug_flip, angle, centre, bpx
@@ -875,6 +904,13 @@ class SAM3DStudentDataset(Dataset):
orig_h, orig_w = ann["orig_shape"]
sq_bbox = ann["bbox_square"]
+ win = 0
+ if self.crop_windows:
+ # The tile is K windows side by side, each already at image_size;
+ # the chosen one's box replaces bbox_square everywhere below.
+ win = random.randrange(len(ann["win_boxes"])) if self.augment else 0
+ img_pil = img_pil.crop((win * W, 0, (win + 1) * W, H))
+ sq_bbox = ann["win_boxes"][win]
sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]
orig_crop_size = sq_bbox[2] - sq_bbox[0]
true_scale = self.image_size / max(orig_crop_size, 1.0)
@@ -909,6 +945,14 @@ class SAM3DStudentDataset(Dataset):
cy = (tight_bbox[1] + tight_bbox[3]) / 2.0
b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])
b_scale = b_size / max(orig_w, orig_h)
+ if win:
+ # CLIFF describes the window: its centre, and the tight box that
+ # gives it at the standard 1.2x expansion -- what a detector box
+ # framing the person this way would produce at runtime.
+ cx = (sq_bbox[0] + sq_bbox[2]) / 2.0
+ cy = (sq_bbox[1] + sq_bbox[3]) / 2.0
+ b_size = (sq_bbox[2] - sq_bbox[0]) / 1.2
+ b_scale = b_size / max(orig_w, orig_h)
aug_flip, cos_t, sin_t = 0.0, 1.0, 0.0
M_total = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=np.float64)
@@ -1058,7 +1102,8 @@ def build_dataloaders(cfg):
jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality,
image_size=cfg.image_size,
context_crops=cfg.context_crops, v3_box_min=cfg.v3_box_min,
- v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift)
+ v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift,
+ v3_lanczos=cfg.v3_lanczos, crop_windows=cfg.crop_windows)
full_dataset = SAM3DStudentDataset(
cfg.data_root,
augment=cfg.augment,
diff --git a/tools/ddp_smoke.py b/tools/ddp_smoke.py
index acd8b2f..1ef3295 100644
--- a/tools/ddp_smoke.py
+++ b/tools/ddp_smoke.py
@@ -55,6 +55,9 @@ def main() -> int:
p.add_argument("--context-crops", dest="context_crops", action="store_true",
help="--preset v3's dataset; --data_root must be a context-crop "
"split (build_split.py --context 2.0 --body-size 448)")
+ p.add_argument("--crop-windows", dest="crop_windows", action="store_true",
+ help="--preset v4's dataset; --data_root must be a window "
+ "split (build_split.py --windows 4) at --image-size")
p.add_argument("--image-size", dest="image_size", type=int, default=None,
help="network input side; the positional grid and every crop follow it")
p.add_argument("--data_root", default="data")
@@ -113,7 +116,8 @@ def main() -> int:
per_dataset_caps={}, cliff_focal=cfg.cliff_focal,
crop_centre_fix=cfg.crop_centre_fix,
image_size=cfg.image_size,
- context_crops=args.context_crops)
+ context_crops=args.context_crops,
+ crop_windows=args.crop_windows)
loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, num_workers=0)
batch = {k: (v.to(T.device) if torch.is_tensor(v) else v)
for k, v in next(iter(loader)).items()}
diff --git a/tools/verify_windows.py b/tools/verify_windows.py
new file mode 100644
index 0000000..9497a38
--- /dev/null
+++ b/tools/verify_windows.py
@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""Verify the precomputed-window corpus (build_split.py --windows) and the
+--preset v4 dataset on real rows.
+
+ python tools/verify_windows.py --split aic_train \
+ --annotation-dir ~/sam3d_data/annotations/aic_train \
+ --image-dir ~/sam3d_data/images/aic \
+ --ref <train_distill_mhr_only.py before the change> --n 64
+
+Checks:
+
+1. Builder: the default and --context 2.0 outputs are byte-identical to HEAD's.
+2. Window build: both size folders hold the same people and the same npz, which
+ is the standard npz plus win_boxes; row 0 is within 1 px of bbox_square,
+ rows 1.. are integer exact squares; each tile is K*S x S.
+3. Every window is the RUNTIME crop: instanthmr.inference's own _preprocess,
+ given the detector box that window stands for (the tight box itself for
+ window 0), returns the same pixels (PNG build: exact) and the same CLIFF
+ vector, and for windows 1.. the labels land where the runtime crop puts the
+ annotated keypoints (window 0's box can be 1 px off square, and the label
+ normalisation, unchanged since g8, uses its width for both axes). Also reports PSNR against the
+ runtime crop for a JPEG q95 build (the cluster corpus).
+4. Augmented v4 (baseline geometry on top): the loss's reprojection chain lands
+ on the joints_2d labels.
+5. The window framings follow the v3 range: side log-uniform 0.6-2.0x the
+ tight box, centre shift within +/-15% of the side.
+6. With crop_windows off, baseline and v2 samples are bit-identical to --ref.
+7. A standard split, and tiles of the wrong size, are refused.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import math
+import os
+import random
+import subprocess
+import sys
+import tempfile
+import types
+from pathlib import Path
+
+import cv2
+import numpy as np
+import torch
+
+REPO = Path(__file__).resolve().parent.parent
+BUILDER = REPO / "datasets_pipeline" / "build_split.py"
+sys.path.insert(0, str(REPO / "instanthmr_distill_train"))
+sys.path.insert(0, str(REPO))
+sys.path.insert(0, str(REPO / "tools"))
+import train_distill_mhr_only as T # noqa: E402
+import train_distill_jz as JZ # noqa: E402
+from instanthmr.inference import InstantHMR # noqa: E402
+from verify_v3_preset import reproject, same, seeded # noqa: E402
+
+MEAN = np.array([0.485, 0.456, 0.406])
+STD = np.array([0.229, 0.224, 0.225])
+K = 4
+SIZES = (224, 288)
+fails: list[str] = []
+
+
+def check(ok: bool, msg: str) -> None:
+ print(f" {'OK ' if ok else 'BAD'} {msg}")
+ if not ok:
+ fails.append(msg)
+
+
+def build(builder: Path, out: Path, args, extra: list[str], fmt: str = "png") -> None:
+ env = dict(os.environ, PYTHONPATH=f"{REPO / 'datasets_pipeline'}:{REPO}")
+ cmd = [sys.executable, str(builder), "--split", args.split,
+ "--annotation-dir", str(args.annotation_dir),
+ "--image-dir", str(args.image_dir), "--output-dir", str(out),
+ "--max-samples", str(args.n), "--originals", "none", "--crop-format", fmt,
+ *extra]
+ subprocess.run(cmd, check=True, env=env, stdout=subprocess.DEVNULL)
+
+
+def tree_diff(a: Path, b: Path) -> int:
+ fa = {p.relative_to(a) for p in a.rglob("*") if p.is_file()}
+ fb = {p.relative_to(b) for p in b.rglob("*") if p.is_file()}
+ if fa != fb or not fa:
+ return max(1, len(fa ^ fb))
+ return sum((a / f).read_bytes() != (b / f).read_bytes() for f in fa)
+
+
+def load(d: Path, name: str) -> dict:
+ with np.load(d / "annotations" / f"{name}.npz", allow_pickle=True) as z:
+ return {k: z[k] for k in z.files}
+
+
+def psnr(a, b) -> float:
+ mse = np.mean((np.asarray(a, np.float64) - np.asarray(b, np.float64)) ** 2)
+ return float("inf") if mse == 0 else 10 * math.log10(255.0 ** 2 / mse)
+
+
+def runtime(img_rgb, box, size, focal):
+ """instanthmr.inference's crop + CLIFF for detector box `box`."""
+ stub = types.SimpleNamespace(cliff_focal=True, input_size=size,
+ _focal_px=lambda h, w, f: f)
+ h, w = img_rgb.shape[:2]
+ crop, x1, y1, side, cliff = InstantHMR._preprocess(stub, img_rgb, box, h, w, focal)
+ u8 = np.clip((crop.transpose(1, 2, 0) * STD + MEAN) * 255.0, 0, 255)
+ return u8, np.array([x1, y1, side]), cliff
+
+
+def detector_box(win: np.ndarray, size: int, img_rgb, focal):
+ """The detector box whose 1.2x square is exactly `win` at the runtime:
+ side / 1.2 in float lands a hair off the integer and the runtime's floor /
+ ceil then cut one pixel more, so nudge it by ulps until it does not."""
+ c, half = (win[:2] + win[2:]) / 2, (win[2] - win[0]) / 2.4
+ for t in range(64):
+ det = np.concatenate([c - half, c + half])
+ out = runtime(img_rgb, det, size, focal)
+ x1, y1, side = out[1] # the runtime cuts floor(x1) .. ceil(x1 + side)
+ if [math.floor(x1), math.floor(y1), math.ceil(x1 + side), math.ceil(y1 + side)] \
+ == [int(v) for v in win]:
+ return det, out
+ half = np.nextafter(half, np.inf if t % 2 else -np.inf)
+ raise RuntimeError(f"no detector box reproduces {win}")
+
+
+def to_uint8(img: torch.Tensor) -> np.ndarray:
+ return np.clip((img.numpy().transpose(1, 2, 0) * STD + MEAN) * 255.0, 0, 255)
+
+
+def cfg_for(preset: str, size: int = 224):
+ cfg = T.DistillConfig()
+ cfg.cliff_focal = True
+ cfg.crop_centre_fix = True
+ cfg.image_size = size
+ if preset == "v2":
+ JZ.apply_v2_preset(cfg)
+ if preset == "v4":
+ JZ.apply_v4_preset(cfg)
+ return cfg
+
+
+def dataset(mod, root, cfg, augment=True, **over):
+ kw = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,
+ geom_scale_range=cfg.geom_scale_range, geom_trans=cfg.geom_trans,
+ geom_flip_p=cfg.geom_flip_p, geom_scale_max=cfg.geom_scale_max,
+ cliff_follows_aug=cfg.cliff_follows_aug, crop_centre_fix=cfg.crop_centre_fix,
+ cliff_focal=cfg.cliff_focal, occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,
+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality, image_size=cfg.image_size)
+ if cfg.crop_windows:
+ kw["crop_windows"] = True
+ kw.update(over)
+ return mod.SAM3DStudentDataset(str(root), augment=augment, **kw)
+
+
+def main() -> None: # noqa: C901
+ p = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--split", default="aic_train")
+ p.add_argument("--annotation-dir", type=Path, required=True)
+ p.add_argument("--image-dir", type=Path, required=True)
+ p.add_argument("--ref", type=Path, required=True)
+ p.add_argument("--n", type=int, default=64)
+ args = p.parse_args()
+
+ tmp = Path(tempfile.mkdtemp(prefix="verify_windows_"))
+ head = tmp / "build_split_head.py"
+ head.write_text(subprocess.run(
+ ["git", "-C", str(REPO), "show", "HEAD:datasets_pipeline/build_split.py"],
+ check=True, capture_output=True, text=True).stdout)
+
+ print("[1] builder defaults vs HEAD")
+ build(head, tmp / "head" / "sam3d_gt_aic", args, [])
+ build(BUILDER, tmp / "std" / "sam3d_gt_aic", args, [])
+ build(BUILDER, tmp / "std_nh" / "sam3d_gt_aic", args, ["--no-hands"])
+ check(tree_diff(tmp / "head", tmp / "std") == 0, "default build byte-identical to HEAD")
+ ctx = ["--context", "2.0", "--body-size", "448", "--no-hands"]
+ build(head, tmp / "head_ctx", args, ctx)
+ build(BUILDER, tmp / "ctx", args, ctx)
+ check(tree_diff(tmp / "head_ctx", tmp / "ctx") == 0, "--context 2.0 build byte-identical to HEAD")
+
+ win = ["--windows", str(K), "--window-sizes", ",".join(map(str, SIZES)), "--no-hands"]
+ build(BUILDER, tmp / "win{size}" / "sam3d_gt_aic", args, win)
+ build(BUILDER, tmp / "winjpg{size}" / "sam3d_gt_aic", args, win, fmt="jpg")
+ std = tmp / "std" / "sam3d_gt_aic"
+ wd = {s: tmp / f"win{s}" / "sam3d_gt_aic" for s in SIZES}
+ wj = {s: tmp / f"winjpg{s}" / "sam3d_gt_aic" for s in SIZES}
+
+ print("[2] window build layout")
+ names = sorted(q.stem for q in (std / "annotations").glob("*.npz"))
+ sets = [sorted(q.stem for q in (wd[s] / "annotations").glob("*.npz")) for s in SIZES]
+ check(all(x == names for x in sets) and names, f"{len(names)} people in every folder")
+ bad_npz = bad_box = bad_tile = 0
+ for nm in names:
+ a, b, c = (load(tmp / "std_nh" / "sam3d_gt_aic", nm), load(wd[SIZES[0]], nm),
+ load(wd[SIZES[1]], nm))
+ wb = b.pop("win_boxes")
+ bad_npz += (b.keys() != a.keys() or not all(same(a[k], b[k]) for k in a)
+ or not np.array_equal(wb, c.pop("win_boxes"))
+ or not all(same(b[k], c[k]) for k in b))
+ sides = wb[1:, 2:] - wb[1:, :2]
+ bad_box += (wb.shape != (K, 4) or np.abs(wb[0] - a["bbox_square"]).max() > 1
+ or not np.array_equal(wb[1:], np.round(wb[1:]))
+ or not np.array_equal(sides[:, 0], sides[:, 1]))
+ for s in SIZES:
+ tile = cv2.imread(str(wd[s] / "body_crops" / f"{nm}.png"))
+ bad_tile += tile.shape != (s, K * s, 3)
+ check(bad_npz == 0, f"npz = standard --no-hands npz + win_boxes, same in both folders "
+ f"({bad_npz} bad)")
+ check(bad_box == 0, f"win_boxes: row 0 within 1 px of bbox_square, rows 1.. integer "
+ f"squares ({bad_box} bad)")
+ check(bad_tile == 0, f"tiles K*S x S ({bad_tile} bad)")
+
+ print("[3] each window is the runtime crop")
+ for s in SIZES:
+ cfg = cfg_for("v4", s)
+ plain = dataset(T, wd[s].parent, cfg, augment=False).transform
+ ds = dataset(T, wd[s].parent, cfg, augment=True, geom_p=0.0)
+ ds.transform = plain # no colour / occlusion noise
+ dsj = dataset(T, wj[s].parent, cfg, augment=True, geom_p=0.0)
+ dsj.transform = plain
+ exact = cl = lab = 0.0
+ ps = {k: [] for k in range(K)}
+ n_k = np.zeros(K, int)
+ for i in range(len(ds)):
+ nm = Path(ds.pairs[i][1]).stem
+ a = load(wd[s], nm)
+ img = cv2.cvtColor(cv2.imread(str(args.image_dir / a["image_relpath"].item())),
+ cv2.COLOR_BGR2RGB)
+ f = float(a["cam_focal_length"][1])
+ for k in range(K):
+ # the draw inside the dataset is random.randrange(K): pick k
+ for seed in range(10_000):
+ random.seed(seed)
+ if random.randrange(K) == k:
+ break
+ smp = seeded(ds, i, seed)
+ smj = seeded(dsj, i, seed)
+ box = a["win_boxes"][k].astype(np.float64)
+ if k == 0:
+ rt, (x1, y1, side), cliff = runtime(img, a["bbox"].astype(np.float64), s, f)
+ else:
+ _, (rt, (x1, y1, side), cliff) = detector_box(box, s, img, f)
+ exact = max(exact, float(np.abs(to_uint8(smp["image"]) - rt).max()))
+ ps[k].append(psnr(to_uint8(smj["image"]), rt))
+ cl = max(cl, float(np.abs(smp["cliff_cond"].numpy() - cliff).max()))
+ j = a["joints_2d"].astype(np.float64)
+ want = np.stack([(j[:, 0] - x1) / side, (j[:, 1] - y1) / side], 1) * 2 - 1
+ vis = smp["joints_2d_vis"].numpy() > 0
+ err = np.abs(smp["joints_2d"].numpy() - want)[vis] * s / 2
+ if k:
+ lab = max(lab, float(err.max()))
+ n_k[k] += 1
+ check(exact < 0.01, f"{s} px: every window pixel-equal to the runtime crop "
+ f"(max |diff| {exact:.4f} /255, PNG build)")
+ check(cl < 1e-5, f"{s} px: CLIFF = runtime's for that box (max diff {cl:.1e})")
+ check(lab < 1e-3, f"{s} px: labels = keypoints in the runtime crop (max {lab:.1e} px)")
+ msg = ", ".join(f"w{k} {np.mean(ps[k]):.1f}" for k in range(K))
+ check(min(np.mean(ps[k]) for k in range(K)) > 36.0,
+ f"{s} px JPEG q95 build vs runtime crop, PSNR dB: {msg}")
+
+ print("[4] augmented v4: reprojection chain")
+ for s in SIZES:
+ ds = dataset(T, wd[s].parent, cfg_for("v4", s), augment=True, geom_p=1.0)
+ rep, shapes, aug = 0.0, set(), 0
+ for t in range(200):
+ smp = seeded(ds, t % len(ds), 500 + t)
+ vis = smp["joints_2d_vis"].numpy() > 0
+ err = np.abs(reproject(smp, s) - smp["joints_2d"].numpy())[vis] * s / 2
+ rep = max(rep, float(err.max()))
+ shapes.add(tuple(smp["image"].shape))
+ aug += int(float(smp["aug_active"]) > 0.5)
+ check(rep < 1e-3 and aug == 200, f"{s} px: max {rep:.2e} px over 200 augmented samples")
+ check(shapes == {(3, s, s)}, f"{s} px: output shapes {shapes}")
+
+ print("[5] framing range")
+ ms, sh = [], []
+ for nm in names:
+ a = load(wd[224], nm)
+ t = a["bbox"].astype(np.float64)
+ b = max(t[2] - t[0], t[3] - t[1])
+ for w in a["win_boxes"][1:].astype(np.float64):
+ side = w[2] - w[0]
+ ms.append(side / b)
+ sh.append(np.abs((w[:2] + w[2:]) / 2 - (t[:2] + t[2:]) / 2).max() / side)
+ ms, sh = np.array(ms), np.array(sh)
+ expect = math.log(1.1 / 0.6) / math.log(2.0 / 0.6)
+ check(ms.min() > 0.59 and ms.max() < 2.01 and abs(np.mean(ms < 1.1) - expect) < 0.08,
+ f"side {ms.min():.2f}-{ms.max():.2f}x, P(<1.1x) {np.mean(ms < 1.1):.2f} vs "
+ f"log-uniform {expect:.2f} ({len(ms)} windows)")
+ check(sh.max() < 0.15 + 0.01, f"centre shift max {sh.max():.3f} of the side")
+
+ print("[6] crop_windows off: bit-identical to --ref")
+ spec = importlib.util.spec_from_file_location("ref_trainer", args.ref)
+ R = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(R)
+ for preset in ("baseline", "v2"):
+ cfg = cfg_for(preset)
+ new, old = dataset(T, std.parent, cfg), dataset(R, std.parent, cfg)
+ diff = 0
+ for t in range(24):
+ for _ in range(2): # alternating calls
+ a, b = seeded(new, t, 7 + t), seeded(old, t, 7 + t)
+ diff += not all(same(a[x], b[x]) for x in a)
+ check(diff == 0, f"{preset}: {diff} of 48 samples differ")
+
+ print("[7] refusals")
+ for root, size, want in ((std.parent, 224, "win_boxes"), (wd[224].parent, 288, "tile")):
+ try:
+ dataset(T, root, cfg_for("v4", size))
+ check(False, f"{root.name} accepted at {size} px")
+ except ValueError as e:
+ check(want in str(e), f"refused: {str(e)[:90]}")
+
+ if fails:
+ print(f"FAIL ({len(fails)})")
+ sys.exit(1)
+ print("PASS")
+
+
+if __name__ == "__main__":
+ main()