Skip to content
Python

fix: drop non-finite 3DPW predictions before the Procrustes SVD (killed v3_s0 at epoch 1)

mohamdevabd6a26ef9c1
diff --git a/instanthmr_distill_train/train_distill_jz.py b/instanthmr_distill_train/train_distill_jz.py
index c99a919..ba40515 100755
--- a/instanthmr_distill_train/train_distill_jz.py
+++ b/instanthmr_distill_train/train_distill_jz.py
@@ -756,6 +756,14 @@ def train(args, cfg, rank, local_rank, world):
f"MPJPE {dpw_raw['J12_MPJPE']:.1f} | J14 PA {dpw_raw['J14_PA_MPJPE']:.1f} mm")
log(f" 3DPW-{args.val_3dpw_split} EMA J12 PA {dpw_em['J12_PA_MPJPE']:.1f} "
f"MPJPE {dpw_em['J12_MPJPE']:.1f} | J14 PA {dpw_em['J14_PA_MPJPE']:.1f} mm")
+ # Non-finite predictions are dropped rather than crashing the metric
+ # (see val3dpw.evaluate). Say so — a handful at epoch 1 is the raw
+ # branch still settling, a rising count is a real instability.
+ nd = dpw_raw.get("n_dropped", 0.0) + dpw_em.get("n_dropped", 0.0)
+ if nd > 0:
+ log(f" ⚠️ 3DPW: {nd:.0f} non-finite prediction(s) dropped "
+ f"(raw {dpw_raw.get('n_dropped', 0.0):.0f} / "
+ f"ema {dpw_em.get('n_dropped', 0.0):.0f})")
if dpw_test_raw is not None:
log(f" 3DPW-TEST (report only, not used for selection)")
log(f" RAW J14 MPJPE {dpw_test_raw['J14_MPJPE']:.1f} | "
diff --git a/instanthmr_distill_train/val3dpw.py b/instanthmr_distill_train/val3dpw.py
index 63c80ec..4966e24 100644
--- a/instanthmr_distill_train/val3dpw.py
+++ b/instanthmr_distill_train/val3dpw.py
@@ -137,6 +137,7 @@ def evaluate(model, loader, mhr_module, device, use_amp: bool = True) -> dict:
"""
model.eval()
preds, gts = [], []
+ n_dropped = 0
for b in loader:
if float(b["ok"].sum()) == 0:
continue
@@ -146,16 +147,38 @@ def evaluate(model, loader, mhr_module, device, use_amp: bool = True) -> dict:
out = model(img, cc)
kp = mhr_module.get_native_keypoints(out["mhr_params"].float(),
out["shape_params"].float())
- keep = b["ok"].bool()
+ # Drop non-finite predictions BEFORE they reach the metric. The forward
+ # runs under fp16 autocast, so an early-training head output above
+ # 65504 is already inf by the time .float() sees it, and the MHR forward
+ # kinematics turn that into inf/NaN keypoints. numpy's SVD inside
+ # pa_mpjpe's Procrustes alignment then raises "SVD did not converge",
+ # which killed job 1767242 (v3_s0) at the epoch-1 validation with a
+ # perfectly healthy training loop behind it -- 0 skipped steps, no
+ # divergence. One bad frame out of a few hundred must not cost a
+ # 20-hour job.
+ finite = torch.isfinite(kp).flatten(1).all(dim=1).cpu()
+ keep = b["ok"].bool() & finite
+ n_dropped += int((b["ok"].bool() & ~finite).sum())
+ if not bool(keep.any()):
+ continue
preds.append(kp[keep.to(kp.device)].float().cpu().numpy())
gts.append(b["gt"][keep].numpy())
+ # Always return the SAME key set. train_distill_jz.all_reduce_mean builds a
+ # tensor from sorted(keys) and all-reduces it, so a rank returning a short
+ # dict while the others return a full one is a size mismatch -> NCCL hang.
+ res: dict[str, float] = {"n": 0.0, "n_dropped": float(n_dropped)}
+ for name in ("J12", "J14"):
+ res[f"{name}_MPJPE"] = float("inf")
+ res[f"{name}_PA_MPJPE"] = float("inf")
if not preds:
- return {"n": 0.0}
+ # inf, not 0: this epoch's 3DPW number must never look like a new best.
+ return res
+
pred = np.concatenate(preds) * MM # (N, 70, 3) rig-local, mm
gt = np.concatenate(gts) * MM # (N, 24, 3) camera space, mm
- res: dict[str, float] = {"n": float(len(pred))}
+ res["n"] = float(len(pred))
for name in ("J12", "J14"):
spec = J.JOINT_SETS[name]
p, g = pred[:, spec["mhr"], :], gt[:, spec["smpl"], :]