Python
fix: drop non-finite 3DPW predictions before the Procrustes SVD (killed v3_s0 at epoch 1)
mohamdevabd6a26ef9c1
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.pyindex c99a919..ba40515 100755--- a/instanthmr_distill_train/train_distill_jz.py+++ b/instanthmr_distill_train/train_distill_jz.py@@ -756,6 +756,14 @@ def train(args, cfg, rank, local_rank, world):f"MPJPE {dpw_raw['J12_MPJPE']:.1f} | J14 PA {dpw_raw['J14_PA_MPJPE']:.1f} mm")log(f" 3DPW-{args.val_3dpw_split} EMA J12 PA {dpw_em['J12_PA_MPJPE']:.1f} "f"MPJPE {dpw_em['J12_MPJPE']:.1f} | J14 PA {dpw_em['J14_PA_MPJPE']:.1f} mm")+ # Non-finite predictions are dropped rather than crashing the metric+ # (see val3dpw.evaluate). Say so — a handful at epoch 1 is the raw+ # branch still settling, a rising count is a real instability.+ nd = dpw_raw.get("n_dropped", 0.0) + dpw_em.get("n_dropped", 0.0)+ if nd > 0:+ log(f" ⚠️ 3DPW: {nd:.0f} non-finite prediction(s) dropped "+ f"(raw {dpw_raw.get('n_dropped', 0.0):.0f} / "+ f"ema {dpw_em.get('n_dropped', 0.0):.0f})")if dpw_test_raw is not None:log(f" 3DPW-TEST (report only, not used for selection)")log(f" RAW J14 MPJPE {dpw_test_raw['J14_MPJPE']:.1f} | "diff --git a/instanthmr_distill_train/val3dpw.py b/instanthmr_distill_train/val3dpw.pyindex 63c80ec..4966e24 100644--- a/instanthmr_distill_train/val3dpw.py+++ b/instanthmr_distill_train/val3dpw.py@@ -137,6 +137,7 @@ def evaluate(model, loader, mhr_module, device, use_amp: bool = True) -> dict:"""model.eval()preds, gts = [], []+ n_dropped = 0for b in loader:if float(b["ok"].sum()) == 0:continue@@ -146,16 +147,38 @@ def evaluate(model, loader, mhr_module, device, use_amp: bool = True) -> dict:out = model(img, cc)kp = mhr_module.get_native_keypoints(out["mhr_params"].float(),out["shape_params"].float())- keep = b["ok"].bool()+ # Drop non-finite predictions BEFORE they reach the metric. The forward+ # runs under fp16 autocast, so an early-training head output above+ # 65504 is already inf by the time .float() sees it, and the MHR forward+ # kinematics turn that into inf/NaN keypoints. numpy's SVD inside+ # pa_mpjpe's Procrustes alignment then raises "SVD did not converge",+ # which killed job 1767242 (v3_s0) at the epoch-1 validation with a+ # perfectly healthy training loop behind it -- 0 skipped steps, no+ # divergence. One bad frame out of a few hundred must not cost a+ # 20-hour job.+ finite = torch.isfinite(kp).flatten(1).all(dim=1).cpu()+ keep = b["ok"].bool() & finite+ n_dropped += int((b["ok"].bool() & ~finite).sum())+ if not bool(keep.any()):+ continuepreds.append(kp[keep.to(kp.device)].float().cpu().numpy())gts.append(b["gt"][keep].numpy())+ # Always return the SAME key set. train_distill_jz.all_reduce_mean builds a+ # tensor from sorted(keys) and all-reduces it, so a rank returning a short+ # dict while the others return a full one is a size mismatch -> NCCL hang.+ res: dict[str, float] = {"n": 0.0, "n_dropped": float(n_dropped)}+ for name in ("J12", "J14"):+ res[f"{name}_MPJPE"] = float("inf")+ res[f"{name}_PA_MPJPE"] = float("inf")if not preds:- return {"n": 0.0}+ # inf, not 0: this epoch's 3DPW number must never look like a new best.+ return res+pred = np.concatenate(preds) * MM # (N, 70, 3) rig-local, mmgt = np.concatenate(gts) * MM # (N, 24, 3) camera space, mm- res: dict[str, float] = {"n": float(len(pred))}+ res["n"] = float(len(pred))for name in ("J12", "J14"):spec = J.JOINT_SETS[name]p, g = pred[:, spec["mhr"], :], gt[:, spec["smpl"], :]