Python
feat: --preset v4 trains on precomputed windows cut from the originals like the runtime crop (build_split --windows), with baseline losses; also --v3-lanczos.
mohamdev5e4c1792af72
diff --git a/datasets_pipeline/build_split.py b/datasets_pipeline/build_split.pyindex dcff254..2805fda 100755--- a/datasets_pipeline/build_split.py+++ b/datasets_pipeline/build_split.py@@ -53,6 +53,26 @@ byte-identical to HEAD; the 1.2x window cut back out of a 2.0x / 448 cropmatches today's 224 crop at 42.2 dB mean (sigma-1 blur, padding masked) witha 0.09 px shift. At 2.0x, 195 of 200 AIC squares reach past the frame edge.+`--windows K --window-sizes 224,288` stores PRECOMPUTED WINDOWS instead (the+v4 preset): K square windows per person, each cut from the full-resolution+photo with ONE resize -- the same operation as the runtime crop, so the training+texture is the deployment texture (a 448 px context crop plus a second warp+was not: STATUS.md 2026-10-21). Window 0 is the 1.2x square of the tight box+with the RUNTIME's rounding (floor / ceil, instanthmr/inference.py), i.e. the+crop the phone cuts for a perfect detector box -- within 1 px of bbox_square,+whose int() edges the runtime does not use; windows 1..K-1 are v3 framings (side log-uniform in+0.6-2.0 x the tight box, centre shifted up to +/-15% of the side), padded black+only where they leave the photo. Each size is written to its own folder+(--output-dir with a `{size}` placeholder) as one tile per person, K windows+side by side (K*S x S px). One key is added:++ win_boxes (K, 4) the windows, full-frame px, integer; rows 1.. are+ exact squares, row 0 is the runtime box (which its+ floor / ceil can leave 1 px off square)++The boxes are seeded by the person's name, so every size, rerun and resume+draws the same ones.+Hand crops are stored UNFLIPPED. SAM3D trains a single right-hand model andmirrors the left at inference; flipping a loaded crop is a free numpy slice, sobaking the mirror into disk would only remove a choice.@@ -65,10 +85,12 @@ baking the mirror into disk would only remove a choice.from __future__ import annotationsimport argparse+import mathimport osimport reimport shutilimport sys+import zlibfrom pathlib import Pathimport cv2@@ -163,6 +185,44 @@ def enough_inside(sq: np.ndarray, h: int, w: int, min_frac: float) -> bool:return ((ix2 - ix1) * (iy2 - iy1)) / max(area, 1e-6) >= min_frac+# v3's framing range, as multiples of the tight box (1.2 = the standard crop),+# and its centre shift as a fraction of the window side.+WINDOW_BOX = (0.6, 2.0)+WINDOW_SHIFT = 0.15+++def window_boxes(tight: np.ndarray, k: int, name: str) -> np.ndarray:+ """Row 0 is the runtime's 1.2x box of `tight`; rows 1..k-1 are random v3+ framings, integer and exactly square, in frame px. Seeded by `name`, so+ reruns agree."""+ rng = np.random.default_rng(zlib.crc32(name.encode()))+ x1, y1, x2, y2 = (float(v) for v in tight)+ cx, cy = (x1 + x2) / 2.0, (y1 + y2) / 2.0+ b = max(x2 - x1, y2 - y1)+ # instanthmr.inference.InstantHMR._preprocess, line for line+ sq_size = b * 1.2+ sx, sy = cx - sq_size / 2.0, cy - sq_size / 2.0+ out = [np.array([math.floor(sx), math.floor(sy), math.ceil(sx + sq_size),+ math.ceil(sy + sq_size)], dtype=np.float32)]+ for _ in range(k - 1):+ m = math.exp(rng.uniform(math.log(WINDOW_BOX[0]), math.log(WINDOW_BOX[1])))+ side = max(8, int(round(m * b)))+ wx = cx + rng.uniform(-WINDOW_SHIFT, WINDOW_SHIFT) * side+ wy = cy + rng.uniform(-WINDOW_SHIFT, WINDOW_SHIFT) * side+ x1, y1 = int(round(wx - side / 2.0)), int(round(wy - side / 2.0))+ out.append(np.array([x1, y1, x1 + side, y1 + side], dtype=np.float32))+ return np.stack(out)+++def cut_box(img_rgb: np.ndarray, box: np.ndarray) -> np.ndarray:+ """The integer box `box` of the photo, black-padded where it leaves it."""+ h, w = img_rgb.shape[:2]+ x1, y1, x2, y2 = (int(v) for v in box)+ crop = img_rgb[max(0, y1):min(h, y2), max(0, x1):min(w, x2)]+ return cv2.copyMakeBorder(crop, max(0, -y1), max(0, y2 - h), max(0, -x1),+ max(0, x2 - w), cv2.BORDER_CONSTANT, value=(0, 0, 0))++def write_crop(img_rgb, box, size, path: Path) -> None:crop, _ = get_square_crop_padded(img_rgb, box, expand=1.0)crop = cv2.resize(crop, (size, size), interpolation=cv2.INTER_LINEAR)@@ -203,6 +263,13 @@ def parse_args() -> argparse.Namespace:"as bbox_context (full-frame px) plus context_padded; ""bbox_square stays the 1.2x square, so the standard window ""can always be cut back out.")+ p.add_argument("--windows", type=int, default=0, metavar="K",+ help="store K precomputed windows per person (window 0 = the "+ "standard 1.2x crop, 1..K-1 = v3 framings), each cut from "+ "the photo with one resize. Needs --window-sizes, "+ "--no-hands and an --output-dir containing {size}.")+ p.add_argument("--window-sizes", default="224,288",+ help="with --windows: comma-separated tile sizes, one folder each")p.add_argument("--crop-format", choices=("png", "jpg"), default="png",help="png keeps body_crops byte-identical to the existing ""data/ tree; jpg is ~5x smaller and ~4x faster to decode")@@ -257,30 +324,44 @@ def main() -> None:if args.context != 1.2 and args.output_dir is None:sys.exit(f"--context {args.context} needs an explicit --output-dir, so the "f"context crops never land in the standard {out_dir.name}")- first = next((out_dir / "annotations").glob("*.npz"), None)- if first is not None:- with np.load(first) as z:- was_context = "bbox_context" in z.files- if was_context != (args.context != 1.2):- sys.exit(f"{out_dir} already holds {'context' if was_context else 'standard'} "- f"crops; refusing to add {'standard' if was_context else 'context'} ones")+ win_sizes = [int(v) for v in args.window_sizes.split(",")] if args.windows else []+ if args.windows:+ if args.windows < 2 or args.context != 1.2 or args.hands:+ sys.exit("--windows needs K >= 2, the default --context and --no-hands")+ if args.output_dir is None or "{size}" not in str(args.output_dir):+ sys.exit("--windows needs an --output-dir containing {size}, e.g. "+ "$SCRATCH/instanthmr/data_win{size}/sam3d_gt_aic")+ out_dirs = [Path(str(args.output_dir).format(size=s)) for s in win_sizes]+ else:+ out_dirs = [out_dir]+ out_dir = out_dirs[0]+ kind = "windows" if args.windows else "context" if args.context != 1.2 else "standard"+ for d in out_dirs:+ first = next((d / "annotations").glob("*.npz"), None)+ if first is not None:+ with np.load(first) as z:+ was = ("windows" if "win_boxes" in z.files else+ "context" if "bbox_context" in z.files else "standard")+ if was != kind:+ sys.exit(f"{d} already holds {was} crops; refusing to add {kind} ones")++ for d in out_dirs:+ for k in ("original_images", "annotations", "body_crops", "hands_crops"):+ if k == "original_images" and (args.originals == "none" or d != out_dir):+ continue+ if k == "hands_crops" and not args.hands:+ continue+ (d / k).mkdir(parents=True, exist_ok=True)+ # Let the unmodified training script discover this folder.+ compat = d / "images"+ if not compat.exists():+ try:+ os.symlink("body_crops", compat)+ except FileExistsError:+ pass # another array task won the racedirs = {k: out_dir / k for k in("original_images", "annotations", "body_crops", "hands_crops")}- for k, d in dirs.items():- if k == "original_images" and args.originals == "none":- continue- if k == "hands_crops" and not args.hands:- continue- d.mkdir(parents=True, exist_ok=True)-- # Let the unmodified training script discover this folder.- compat = out_dir / "images"- if not compat.exists():- try:- os.symlink("body_crops", compat)- except FileExistsError:- pass # another array task won the raceparquets = sorted(args.annotation_dir.glob("*.parquet"))if not parquets:@@ -299,7 +380,8 @@ def main() -> None:ext = args.crop_formatimwrite_params = ([int(cv2.IMWRITE_JPEG_QUALITY), args.jpg_quality]if ext == "jpg" else [])- existing = {p.stem for p in dirs["annotations"].glob("*.npz")}+ # The last folder's npz is written last, so it marks a finished person.+ existing = {p.stem for p in (out_dirs[-1] / "annotations").glob("*.npz")}total = len(existing)st = {k: 0 for k in ("crops", "hands_l", "hands_r", "hand_too_small","hand_out_of_frame", "hand_occluded", "skipped_existing",@@ -378,11 +460,18 @@ def main() -> None:ann["context_padded"] = np.array(bool(ctx_bbox[0] < 0 or ctx_bbox[1] < 0or ctx_bbox[2] > w or ctx_bbox[3] > h))- body_crop = cv2.resize(body_crop, (args.body_size, args.body_size),- interpolation=cv2.INTER_LINEAR)- cv2.imwrite(str(dirs["body_crops"] / f"{name}.{ext}"),- cv2.cvtColor(body_crop, cv2.COLOR_RGB2BGR),- imwrite_params)+ if args.windows:+ ann["win_boxes"] = window_boxes(ann["bbox"], args.windows, name)+ crops = [cut_box(img_rgb, b) for b in ann["win_boxes"]]+ tiles = [np.hstack([cv2.resize(c, (s, s), interpolation=cv2.INTER_LINEAR)+ for c in crops]) for s in win_sizes]+ else:+ tiles = [cv2.resize(body_crop, (args.body_size, args.body_size),+ interpolation=cv2.INTER_LINEAR)]+ for d, tile in zip(out_dirs, tiles):+ cv2.imwrite(str(d / "body_crops" / f"{name}.{ext}"),+ cv2.cvtColor(tile, cv2.COLOR_RGB2BGR),+ imwrite_params)ann["bbox_square"] = sq_bboxann["image_relpath"] = relpath@@ -419,7 +508,8 @@ def main() -> None:ann[f"hand_valid_{side}"] = np.array(True)st[f"hands_{side}"] += 1- np.savez(dirs["annotations"] / f"{name}.npz", **ann)+ for d in out_dirs:+ np.savez(d / "annotations" / f"{name}.npz", **ann)if args.originals != "none":place_original(args.image_dir / relpath,dirs["original_images"] / relpath,diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.pyindex 49f55f2..a47285a 100755--- a/instanthmr_distill_train/train_distill_jz.py+++ b/instanthmr_distill_train/train_distill_jz.py@@ -397,7 +397,8 @@ def build_jz_loaders(cfg, args, rank: int, world: int):jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality,image_size=cfg.image_size,context_crops=cfg.context_crops, v3_box_min=cfg.v3_box_min,- v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift)+ v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift,+ v3_lanczos=cfg.v3_lanczos, crop_windows=cfg.crop_windows)# Rank 0 builds the index (or reads the cache) while the others wait, then# everyone loads the same cache. Building on all ranks at once would put@@ -710,6 +711,7 @@ def train(args, cfg, rank, local_rank, world):"backbone": cfg.backbone, "w_reproj": cfg.w_reproj,"image_size": cfg.image_size, "context_crops": cfg.context_crops,"v3_box": [cfg.v3_box_min, cfg.v3_box_max], "v3_shift": cfg.v3_shift,+ "v3_lanczos": cfg.v3_lanczos, "crop_windows": cfg.crop_windows,"cont_head": cfg.cont_head, "root_rot_loss": cfg.root_rot_loss,"exact_landmarks": cfg.exact_landmarks,"geom_scale_max": cfg.geom_scale_max, "occl_p": cfg.occl_p,@@ -1144,6 +1146,10 @@ def parse_args():p.add_argument("--w_simcc", type=float, default=None)p.add_argument("--weight_decay", type=float, default=None)p.add_argument("--backbone", default=None)+ p.add_argument("--v3-lanczos", dest="v3_lanczos", action="store_true",+ help="--preset v3: cut the training window with Lanczos instead of "+ "bilinear, so training crops are as sharp as val/phone crops. "+ "Use for every new v3 run.")p.add_argument("--image-size", dest="image_size", type=int, default=None,help="Network input side in px (default 224). A multiple of ""32: the decoder's positional grid is image_size // 32. "@@ -1162,13 +1168,17 @@ def parse_args():help="Skip the timm download. Compute nodes have no internet, ""so run --warm-cache once on a login node instead.")p.add_argument("--log-every", type=int, default=100)- p.add_argument("--preset", choices=["baseline", "v2", "v3"], default="baseline",+ p.add_argument("--preset", choices=["baseline", "v2", "v3", "v4"], default="baseline",help="'v2' enables the wave-1 bundle: truncation, occlusion ""and JPEG augmentation with the SimCC-mask and CLIFF ""prerequisites, plus absolute-pose supervision. ""'v3' is v2 with every crop sampled from a context-crop ""split over a wide range of framings (needs --data_root ""of build_split.py --context splits). "+ "'v4' gives v3's framings from precomputed windows cut "+ "like the runtime crop (needs a build_split.py --windows "+ "corpus at --image-size), v2's occlusion/JPEG, baseline "+ "geometry and baseline loss weights. ""'baseline' reproduces the pre-v2 trainer exactly.")p.add_argument("--w_reproj", type=float, default=None)p.add_argument("--crop-centre-fix", dest="crop_centre_fix", action="store_true",@@ -1263,6 +1273,7 @@ def build_cfg(args):if args.w_simcc is not None: cfg.w_simcc = args.w_simccif args.weight_decay is not None: cfg.weight_decay = args.weight_decayif args.ema_decay is not None: cfg.ema_decay = args.ema_decay+ if args.v3_lanczos: cfg.v3_lanczos = Trueif args.image_size is not None:if args.image_size % 32:raise SystemExit(f"--image-size {args.image_size} is not a multiple of 32")@@ -1272,6 +1283,8 @@ def build_cfg(args):apply_v2_preset(cfg)if args.preset == "v3":apply_v3_preset(cfg)+ if args.preset == "v4":+ apply_v4_preset(cfg)# Orthogonal axis: --preset is the input pipeline + absolute-pose terms,# --losses is the loss budget. Applied second so it owns w_keypoints3d.if args.losses == "rebalanced":@@ -1333,6 +1346,29 @@ def apply_v3_preset(cfg) -> None:cfg.cliff_focal = True+def apply_v4_preset(cfg) -> None:+ """v3's framings without v3's texture, on baseline loss weights.++ Each crop is one of K windows build_split.py --windows cut from the+ full-resolution photo with a single resize, like the runtime crop: window 0+ is the standard 1.2x crop, the others v3 framings. v3 cut its windows from a+ 448 px context crop with a second warp, and its models learned that texture+ (3DPW val J14 PA, g9_b4_224 ep314: 49.3 on crops cut from the frame vs 43.6+ on crops built its own way; g8h 40.0 vs 39.4, STATUS.md 2026-10-21).++ On top: v2's occlusion and JPEG with the two prerequisites truncated+ framings need, and baseline geometry (the windows already supply the zoom+ and shift). The loss weights stay the baseline ones -- no v2 item 3."""+ cfg.crop_windows = True+ cfg.cliff_follows_aug = True+ cfg.mask_oob_2d = True+ cfg.occl_p = 0.7+ cfg.occl_scale = (0.02, 0.30)+ cfg.jpeg_p = 0.3+ cfg.jpeg_quality = (30, 90)+ cfg.cliff_focal = True++def main():args = parse_args()diff --git a/instanthmr_distill_train/train_distill_mhr_only.py b/instanthmr_distill_train/train_distill_mhr_only.pyindex 5a846e4..2af8922 100644--- a/instanthmr_distill_train/train_distill_mhr_only.py+++ b/instanthmr_distill_train/train_distill_mhr_only.py@@ -178,6 +178,19 @@ class DistillConfig:v3_box_min: float = 0.6v3_box_max: float = 2.0v3_shift: float = 0.15+ # Lanczos instead of bilinear for the v3 window cut. Bilinear after the+ # stored crop's own resize left v3 training crops at 0.73 (224) / 0.67 (288)+ # of the sharpness of a crop cut straight from the photo (val3dpw and the+ # phone do the latter); Lanczos gives 1.00 / 0.94. Measured 2026-10-20 on 300+ # AIC people. Bilinear-trained g9 models lost 3-5 mm (224) and up to 17 mm+ # (288) on sharp tight-box crops. Off by default so in-flight g9 runs resume+ # unchanged.+ v3_lanczos: bool = False+ # --preset v4: each image is a tile of K windows that build_split.py+ # --windows cut from the full-resolution photo with ONE resize, the same+ # operation as the runtime crop; one window is drawn per sample (window 0,+ # the standard 1.2x crop, when not augmenting).+ crop_windows: bool = Falsemax_images: int | None = Noneper_dataset_caps: dict = field(default_factory=lambda: {"sam3d_gt_harmony4d": 300_000})val_split: float = 0.1@@ -616,8 +629,11 @@ class SAM3DStudentDataset(Dataset):occl_p: float = 0.2, occl_scale: tuple = (0.02, 0.15),jpeg_p: float = 0.0, jpeg_quality: tuple = (30, 90),context_crops: bool = False, v3_box_min: float = 0.6,- v3_box_max: float = 2.0, v3_shift: float = 0.15):+ v3_box_max: float = 2.0, v3_shift: float = 0.15,+ v3_lanczos: bool = False, crop_windows: bool = False):super().__init__()+ self.crop_windows = crop_windows+ self.v3_interp = cv2.INTER_LANCZOS4 if v3_lanczos else cv2.INTER_LINEARself.image_size = image_sizeself.context_crops = context_cropsself.v3_box_min, self.v3_box_max, self.v3_shift = v3_box_min, v3_box_max, v3_shift@@ -722,7 +738,7 @@ class SAM3DStudentDataset(Dataset):"""v3 samples from context crops. A standard 224 split would still'work' -- with the 1.2x crop read as if it were the context, i.e. everybox silently 1.67x too large -- so refuse it up front."""- if not self.context_crops or len(self.pairs) == 0:+ if not (self.context_crops or self.crop_windows) or len(self.pairs) == 0:return# One probe per sub-folder, so a --data_root mixing context and# standard splits fails here and not hours in, inside a worker.@@ -737,6 +753,19 @@ class SAM3DStudentDataset(Dataset):for k in probes:npz = Path(self.pairs[k][1])with np.load(npz) as z:+ if self.crop_windows:+ if "win_boxes" not in z.files:+ raise ValueError(+ f"{npz.parent.parent.name}: {npz.name} has no win_boxes. "+ f"--preset v4 needs a window split, built with "+ f"build_split.py --windows 4 --window-sizes 224,288 ...")+ tw, th = Image.open(self.pairs[k][0]).size+ if th != self.image_size or tw != th * len(z["win_boxes"]):+ raise ValueError(+ f"{npz.parent.parent.name}: tile {tw}x{th} does not hold "+ f"{len(z['win_boxes'])} windows of {self.image_size} px; "+ f"use the data_win{self.image_size} corpus")+ continueif "bbox_context" not in z.files:raise ValueError(f"{npz.parent.parent.name}: {npz.name} has no bbox_context. "@@ -836,7 +865,7 @@ class SAM3DStudentDataset(Dataset):[0.0, ay * ky, (0.5 * ay + ctx[1] - sq[1]) * ky - 0.5]])A, t = M_total[:, :2], M_total[:, 2]M_img = np.hstack([A @ C[:, :2], (A @ C[:, 2] + t)[:, None]])- img_np = cv2.warpAffine(ctx_np, M_img, (W, H), flags=cv2.INTER_LINEAR,+ img_np = cv2.warpAffine(ctx_np, M_img, (W, H), flags=self.v3_interp,borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))return Image.fromarray(img_np), M_total, aug_active, aug_flip, angle, centre, bpx@@ -875,6 +904,13 @@ class SAM3DStudentDataset(Dataset):orig_h, orig_w = ann["orig_shape"]sq_bbox = ann["bbox_square"]+ win = 0+ if self.crop_windows:+ # The tile is K windows side by side, each already at image_size;+ # the chosen one's box replaces bbox_square everywhere below.+ win = random.randrange(len(ann["win_boxes"])) if self.augment else 0+ img_pil = img_pil.crop((win * W, 0, (win + 1) * W, H))+ sq_bbox = ann["win_boxes"][win]sq_x1, sq_y1 = sq_bbox[0], sq_bbox[1]orig_crop_size = sq_bbox[2] - sq_bbox[0]true_scale = self.image_size / max(orig_crop_size, 1.0)@@ -909,6 +945,14 @@ class SAM3DStudentDataset(Dataset):cy = (tight_bbox[1] + tight_bbox[3]) / 2.0b_size = max(tight_bbox[2] - tight_bbox[0], tight_bbox[3] - tight_bbox[1])b_scale = b_size / max(orig_w, orig_h)+ if win:+ # CLIFF describes the window: its centre, and the tight box that+ # gives it at the standard 1.2x expansion -- what a detector box+ # framing the person this way would produce at runtime.+ cx = (sq_bbox[0] + sq_bbox[2]) / 2.0+ cy = (sq_bbox[1] + sq_bbox[3]) / 2.0+ b_size = (sq_bbox[2] - sq_bbox[0]) / 1.2+ b_scale = b_size / max(orig_w, orig_h)aug_flip, cos_t, sin_t = 0.0, 1.0, 0.0M_total = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=np.float64)@@ -1058,7 +1102,8 @@ def build_dataloaders(cfg):jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality,image_size=cfg.image_size,context_crops=cfg.context_crops, v3_box_min=cfg.v3_box_min,- v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift)+ v3_box_max=cfg.v3_box_max, v3_shift=cfg.v3_shift,+ v3_lanczos=cfg.v3_lanczos, crop_windows=cfg.crop_windows)full_dataset = SAM3DStudentDataset(cfg.data_root,augment=cfg.augment,diff --git a/tools/ddp_smoke.py b/tools/ddp_smoke.pyindex acd8b2f..1ef3295 100644--- a/tools/ddp_smoke.py+++ b/tools/ddp_smoke.py@@ -55,6 +55,9 @@ def main() -> int:p.add_argument("--context-crops", dest="context_crops", action="store_true",help="--preset v3's dataset; --data_root must be a context-crop ""split (build_split.py --context 2.0 --body-size 448)")+ p.add_argument("--crop-windows", dest="crop_windows", action="store_true",+ help="--preset v4's dataset; --data_root must be a window "+ "split (build_split.py --windows 4) at --image-size")p.add_argument("--image-size", dest="image_size", type=int, default=None,help="network input side; the positional grid and every crop follow it")p.add_argument("--data_root", default="data")@@ -113,7 +116,8 @@ def main() -> int:per_dataset_caps={}, cliff_focal=cfg.cliff_focal,crop_centre_fix=cfg.crop_centre_fix,image_size=cfg.image_size,- context_crops=args.context_crops)+ context_crops=args.context_crops,+ crop_windows=args.crop_windows)loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, num_workers=0)batch = {k: (v.to(T.device) if torch.is_tensor(v) else v)for k, v in next(iter(loader)).items()}diff --git a/tools/verify_windows.py b/tools/verify_windows.pynew file mode 100644index 0000000..9497a38--- /dev/null+++ b/tools/verify_windows.py@@ -0,0 +1,320 @@+#!/usr/bin/env python3+"""Verify the precomputed-window corpus (build_split.py --windows) and the+--preset v4 dataset on real rows.++ python tools/verify_windows.py --split aic_train \+ --annotation-dir ~/sam3d_data/annotations/aic_train \+ --image-dir ~/sam3d_data/images/aic \+ --ref <train_distill_mhr_only.py before the change> --n 64++Checks:++1. Builder: the default and --context 2.0 outputs are byte-identical to HEAD's.+2. Window build: both size folders hold the same people and the same npz, which+ is the standard npz plus win_boxes; row 0 is within 1 px of bbox_square,+ rows 1.. are integer exact squares; each tile is K*S x S.+3. Every window is the RUNTIME crop: instanthmr.inference's own _preprocess,+ given the detector box that window stands for (the tight box itself for+ window 0), returns the same pixels (PNG build: exact) and the same CLIFF+ vector, and for windows 1.. the labels land where the runtime crop puts the+ annotated keypoints (window 0's box can be 1 px off square, and the label+ normalisation, unchanged since g8, uses its width for both axes). Also reports PSNR against the+ runtime crop for a JPEG q95 build (the cluster corpus).+4. Augmented v4 (baseline geometry on top): the loss's reprojection chain lands+ on the joints_2d labels.+5. The window framings follow the v3 range: side log-uniform 0.6-2.0x the+ tight box, centre shift within +/-15% of the side.+6. With crop_windows off, baseline and v2 samples are bit-identical to --ref.+7. A standard split, and tiles of the wrong size, are refused.+"""++from __future__ import annotations++import argparse+import importlib.util+import math+import os+import random+import subprocess+import sys+import tempfile+import types+from pathlib import Path++import cv2+import numpy as np+import torch++REPO = Path(__file__).resolve().parent.parent+BUILDER = REPO / "datasets_pipeline" / "build_split.py"+sys.path.insert(0, str(REPO / "instanthmr_distill_train"))+sys.path.insert(0, str(REPO))+sys.path.insert(0, str(REPO / "tools"))+import train_distill_mhr_only as T # noqa: E402+import train_distill_jz as JZ # noqa: E402+from instanthmr.inference import InstantHMR # noqa: E402+from verify_v3_preset import reproject, same, seeded # noqa: E402++MEAN = np.array([0.485, 0.456, 0.406])+STD = np.array([0.229, 0.224, 0.225])+K = 4+SIZES = (224, 288)+fails: list[str] = []+++def check(ok: bool, msg: str) -> None:+ print(f" {'OK ' if ok else 'BAD'} {msg}")+ if not ok:+ fails.append(msg)+++def build(builder: Path, out: Path, args, extra: list[str], fmt: str = "png") -> None:+ env = dict(os.environ, PYTHONPATH=f"{REPO / 'datasets_pipeline'}:{REPO}")+ cmd = [sys.executable, str(builder), "--split", args.split,+ "--annotation-dir", str(args.annotation_dir),+ "--image-dir", str(args.image_dir), "--output-dir", str(out),+ "--max-samples", str(args.n), "--originals", "none", "--crop-format", fmt,+ *extra]+ subprocess.run(cmd, check=True, env=env, stdout=subprocess.DEVNULL)+++def tree_diff(a: Path, b: Path) -> int:+ fa = {p.relative_to(a) for p in a.rglob("*") if p.is_file()}+ fb = {p.relative_to(b) for p in b.rglob("*") if p.is_file()}+ if fa != fb or not fa:+ return max(1, len(fa ^ fb))+ return sum((a / f).read_bytes() != (b / f).read_bytes() for f in fa)+++def load(d: Path, name: str) -> dict:+ with np.load(d / "annotations" / f"{name}.npz", allow_pickle=True) as z:+ return {k: z[k] for k in z.files}+++def psnr(a, b) -> float:+ mse = np.mean((np.asarray(a, np.float64) - np.asarray(b, np.float64)) ** 2)+ return float("inf") if mse == 0 else 10 * math.log10(255.0 ** 2 / mse)+++def runtime(img_rgb, box, size, focal):+ """instanthmr.inference's crop + CLIFF for detector box `box`."""+ stub = types.SimpleNamespace(cliff_focal=True, input_size=size,+ _focal_px=lambda h, w, f: f)+ h, w = img_rgb.shape[:2]+ crop, x1, y1, side, cliff = InstantHMR._preprocess(stub, img_rgb, box, h, w, focal)+ u8 = np.clip((crop.transpose(1, 2, 0) * STD + MEAN) * 255.0, 0, 255)+ return u8, np.array([x1, y1, side]), cliff+++def detector_box(win: np.ndarray, size: int, img_rgb, focal):+ """The detector box whose 1.2x square is exactly `win` at the runtime:+ side / 1.2 in float lands a hair off the integer and the runtime's floor /+ ceil then cut one pixel more, so nudge it by ulps until it does not."""+ c, half = (win[:2] + win[2:]) / 2, (win[2] - win[0]) / 2.4+ for t in range(64):+ det = np.concatenate([c - half, c + half])+ out = runtime(img_rgb, det, size, focal)+ x1, y1, side = out[1] # the runtime cuts floor(x1) .. ceil(x1 + side)+ if [math.floor(x1), math.floor(y1), math.ceil(x1 + side), math.ceil(y1 + side)] \+ == [int(v) for v in win]:+ return det, out+ half = np.nextafter(half, np.inf if t % 2 else -np.inf)+ raise RuntimeError(f"no detector box reproduces {win}")+++def to_uint8(img: torch.Tensor) -> np.ndarray:+ return np.clip((img.numpy().transpose(1, 2, 0) * STD + MEAN) * 255.0, 0, 255)+++def cfg_for(preset: str, size: int = 224):+ cfg = T.DistillConfig()+ cfg.cliff_focal = True+ cfg.crop_centre_fix = True+ cfg.image_size = size+ if preset == "v2":+ JZ.apply_v2_preset(cfg)+ if preset == "v4":+ JZ.apply_v4_preset(cfg)+ return cfg+++def dataset(mod, root, cfg, augment=True, **over):+ kw = dict(geom_p=cfg.geom_p, geom_rot_deg=cfg.geom_rot_deg,+ geom_scale_range=cfg.geom_scale_range, geom_trans=cfg.geom_trans,+ geom_flip_p=cfg.geom_flip_p, geom_scale_max=cfg.geom_scale_max,+ cliff_follows_aug=cfg.cliff_follows_aug, crop_centre_fix=cfg.crop_centre_fix,+ cliff_focal=cfg.cliff_focal, occl_p=cfg.occl_p, occl_scale=cfg.occl_scale,+ jpeg_p=cfg.jpeg_p, jpeg_quality=cfg.jpeg_quality, image_size=cfg.image_size)+ if cfg.crop_windows:+ kw["crop_windows"] = True+ kw.update(over)+ return mod.SAM3DStudentDataset(str(root), augment=augment, **kw)+++def main() -> None: # noqa: C901+ p = argparse.ArgumentParser(description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument("--split", default="aic_train")+ p.add_argument("--annotation-dir", type=Path, required=True)+ p.add_argument("--image-dir", type=Path, required=True)+ p.add_argument("--ref", type=Path, required=True)+ p.add_argument("--n", type=int, default=64)+ args = p.parse_args()++ tmp = Path(tempfile.mkdtemp(prefix="verify_windows_"))+ head = tmp / "build_split_head.py"+ head.write_text(subprocess.run(+ ["git", "-C", str(REPO), "show", "HEAD:datasets_pipeline/build_split.py"],+ check=True, capture_output=True, text=True).stdout)++ print("[1] builder defaults vs HEAD")+ build(head, tmp / "head" / "sam3d_gt_aic", args, [])+ build(BUILDER, tmp / "std" / "sam3d_gt_aic", args, [])+ build(BUILDER, tmp / "std_nh" / "sam3d_gt_aic", args, ["--no-hands"])+ check(tree_diff(tmp / "head", tmp / "std") == 0, "default build byte-identical to HEAD")+ ctx = ["--context", "2.0", "--body-size", "448", "--no-hands"]+ build(head, tmp / "head_ctx", args, ctx)+ build(BUILDER, tmp / "ctx", args, ctx)+ check(tree_diff(tmp / "head_ctx", tmp / "ctx") == 0, "--context 2.0 build byte-identical to HEAD")++ win = ["--windows", str(K), "--window-sizes", ",".join(map(str, SIZES)), "--no-hands"]+ build(BUILDER, tmp / "win{size}" / "sam3d_gt_aic", args, win)+ build(BUILDER, tmp / "winjpg{size}" / "sam3d_gt_aic", args, win, fmt="jpg")+ std = tmp / "std" / "sam3d_gt_aic"+ wd = {s: tmp / f"win{s}" / "sam3d_gt_aic" for s in SIZES}+ wj = {s: tmp / f"winjpg{s}" / "sam3d_gt_aic" for s in SIZES}++ print("[2] window build layout")+ names = sorted(q.stem for q in (std / "annotations").glob("*.npz"))+ sets = [sorted(q.stem for q in (wd[s] / "annotations").glob("*.npz")) for s in SIZES]+ check(all(x == names for x in sets) and names, f"{len(names)} people in every folder")+ bad_npz = bad_box = bad_tile = 0+ for nm in names:+ a, b, c = (load(tmp / "std_nh" / "sam3d_gt_aic", nm), load(wd[SIZES[0]], nm),+ load(wd[SIZES[1]], nm))+ wb = b.pop("win_boxes")+ bad_npz += (b.keys() != a.keys() or not all(same(a[k], b[k]) for k in a)+ or not np.array_equal(wb, c.pop("win_boxes"))+ or not all(same(b[k], c[k]) for k in b))+ sides = wb[1:, 2:] - wb[1:, :2]+ bad_box += (wb.shape != (K, 4) or np.abs(wb[0] - a["bbox_square"]).max() > 1+ or not np.array_equal(wb[1:], np.round(wb[1:]))+ or not np.array_equal(sides[:, 0], sides[:, 1]))+ for s in SIZES:+ tile = cv2.imread(str(wd[s] / "body_crops" / f"{nm}.png"))+ bad_tile += tile.shape != (s, K * s, 3)+ check(bad_npz == 0, f"npz = standard --no-hands npz + win_boxes, same in both folders "+ f"({bad_npz} bad)")+ check(bad_box == 0, f"win_boxes: row 0 within 1 px of bbox_square, rows 1.. integer "+ f"squares ({bad_box} bad)")+ check(bad_tile == 0, f"tiles K*S x S ({bad_tile} bad)")++ print("[3] each window is the runtime crop")+ for s in SIZES:+ cfg = cfg_for("v4", s)+ plain = dataset(T, wd[s].parent, cfg, augment=False).transform+ ds = dataset(T, wd[s].parent, cfg, augment=True, geom_p=0.0)+ ds.transform = plain # no colour / occlusion noise+ dsj = dataset(T, wj[s].parent, cfg, augment=True, geom_p=0.0)+ dsj.transform = plain+ exact = cl = lab = 0.0+ ps = {k: [] for k in range(K)}+ n_k = np.zeros(K, int)+ for i in range(len(ds)):+ nm = Path(ds.pairs[i][1]).stem+ a = load(wd[s], nm)+ img = cv2.cvtColor(cv2.imread(str(args.image_dir / a["image_relpath"].item())),+ cv2.COLOR_BGR2RGB)+ f = float(a["cam_focal_length"][1])+ for k in range(K):+ # the draw inside the dataset is random.randrange(K): pick k+ for seed in range(10_000):+ random.seed(seed)+ if random.randrange(K) == k:+ break+ smp = seeded(ds, i, seed)+ smj = seeded(dsj, i, seed)+ box = a["win_boxes"][k].astype(np.float64)+ if k == 0:+ rt, (x1, y1, side), cliff = runtime(img, a["bbox"].astype(np.float64), s, f)+ else:+ _, (rt, (x1, y1, side), cliff) = detector_box(box, s, img, f)+ exact = max(exact, float(np.abs(to_uint8(smp["image"]) - rt).max()))+ ps[k].append(psnr(to_uint8(smj["image"]), rt))+ cl = max(cl, float(np.abs(smp["cliff_cond"].numpy() - cliff).max()))+ j = a["joints_2d"].astype(np.float64)+ want = np.stack([(j[:, 0] - x1) / side, (j[:, 1] - y1) / side], 1) * 2 - 1+ vis = smp["joints_2d_vis"].numpy() > 0+ err = np.abs(smp["joints_2d"].numpy() - want)[vis] * s / 2+ if k:+ lab = max(lab, float(err.max()))+ n_k[k] += 1+ check(exact < 0.01, f"{s} px: every window pixel-equal to the runtime crop "+ f"(max |diff| {exact:.4f} /255, PNG build)")+ check(cl < 1e-5, f"{s} px: CLIFF = runtime's for that box (max diff {cl:.1e})")+ check(lab < 1e-3, f"{s} px: labels = keypoints in the runtime crop (max {lab:.1e} px)")+ msg = ", ".join(f"w{k} {np.mean(ps[k]):.1f}" for k in range(K))+ check(min(np.mean(ps[k]) for k in range(K)) > 36.0,+ f"{s} px JPEG q95 build vs runtime crop, PSNR dB: {msg}")++ print("[4] augmented v4: reprojection chain")+ for s in SIZES:+ ds = dataset(T, wd[s].parent, cfg_for("v4", s), augment=True, geom_p=1.0)+ rep, shapes, aug = 0.0, set(), 0+ for t in range(200):+ smp = seeded(ds, t % len(ds), 500 + t)+ vis = smp["joints_2d_vis"].numpy() > 0+ err = np.abs(reproject(smp, s) - smp["joints_2d"].numpy())[vis] * s / 2+ rep = max(rep, float(err.max()))+ shapes.add(tuple(smp["image"].shape))+ aug += int(float(smp["aug_active"]) > 0.5)+ check(rep < 1e-3 and aug == 200, f"{s} px: max {rep:.2e} px over 200 augmented samples")+ check(shapes == {(3, s, s)}, f"{s} px: output shapes {shapes}")++ print("[5] framing range")+ ms, sh = [], []+ for nm in names:+ a = load(wd[224], nm)+ t = a["bbox"].astype(np.float64)+ b = max(t[2] - t[0], t[3] - t[1])+ for w in a["win_boxes"][1:].astype(np.float64):+ side = w[2] - w[0]+ ms.append(side / b)+ sh.append(np.abs((w[:2] + w[2:]) / 2 - (t[:2] + t[2:]) / 2).max() / side)+ ms, sh = np.array(ms), np.array(sh)+ expect = math.log(1.1 / 0.6) / math.log(2.0 / 0.6)+ check(ms.min() > 0.59 and ms.max() < 2.01 and abs(np.mean(ms < 1.1) - expect) < 0.08,+ f"side {ms.min():.2f}-{ms.max():.2f}x, P(<1.1x) {np.mean(ms < 1.1):.2f} vs "+ f"log-uniform {expect:.2f} ({len(ms)} windows)")+ check(sh.max() < 0.15 + 0.01, f"centre shift max {sh.max():.3f} of the side")++ print("[6] crop_windows off: bit-identical to --ref")+ spec = importlib.util.spec_from_file_location("ref_trainer", args.ref)+ R = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(R)+ for preset in ("baseline", "v2"):+ cfg = cfg_for(preset)+ new, old = dataset(T, std.parent, cfg), dataset(R, std.parent, cfg)+ diff = 0+ for t in range(24):+ for _ in range(2): # alternating calls+ a, b = seeded(new, t, 7 + t), seeded(old, t, 7 + t)+ diff += not all(same(a[x], b[x]) for x in a)+ check(diff == 0, f"{preset}: {diff} of 48 samples differ")++ print("[7] refusals")+ for root, size, want in ((std.parent, 224, "win_boxes"), (wd[224].parent, 288, "tile")):+ try:+ dataset(T, root, cfg_for("v4", size))+ check(False, f"{root.name} accepted at {size} px")+ except ValueError as e:+ check(want in str(e), f"refused: {str(e)[:90]}")++ if fails:+ print(f"FAIL ({len(fails)})")+ sys.exit(1)+ print("PASS")+++if __name__ == "__main__":+ main()