Skip to content
Python
6.2 KB
Raw Blame
#!/usr/bin/env python3
"""Build `training/assets/mhr_cont_head.npz` for `--cont-head`.

    python tools/build_cont_head_assets.py --data_root data

Two sources, both fixed:

* **The teacher's own buffers**, lifted from `model.ckpt` of
  `facebook/sam-3d-body-dinov3` (`head_pose.*`). These are the bone-scale PCA
  basis and the hand pose basis that the teacher's head uses to expand its 28
  scale coefficients into 68 bone scales and its 2x54 hand coefficients into
  2x27 finger angles. `head_pose` and `head_pose_hand` carry byte-identical
  copies, so which one is read does not matter.

* **Bounds on the 28 scale coefficients, measured from the corpus.**
  `--bound-scales` clamps the 68 expanded bone scales coordinatewise with a
  tanh, which is exactly what a PCA head must NOT do: an arbitrary point in the
  68-dim box is generally not in the 28-dim column space of `scale_comps`, so
  clamping after expansion silently leaves the subspace the basis defines. The
  bound therefore has to live on the coefficients, and the honest limits are
  the ones the ground truth actually occupies.

The projection uses a rank-truncated pseudo-inverse. `scale_comps` is (28, 68)
but its float32 rank is only 24, so four directions are numerically null;
`scale_comps.T` is NOT its inverse and a plain `pinv` amplifies the null
directions without bound.
"""

from __future__ import annotations

import argparse
import glob
from pathlib import Path

import numpy as np
import torch

# Below this fraction of the largest singular value a direction of scale_comps
# is treated as null. The gap in the spectrum is wide, so the exact cut does not
# matter -- the script prints the spectrum so you can check.
RCOND = 1e-5
# Widen the observed coefficient range by this fraction of its own span, so the
# bound is not exactly the corpus extremes.
MARGIN_FRAC = 0.25


def teacher_buffers(ckpt: Path | None) -> dict[str, np.ndarray]:
    if ckpt is None:
        hits = glob.glob(str(Path.home() / ".cache/huggingface/hub/"
                             "models--facebook--sam-3d-body-dinov3/snapshots/*/model.ckpt"))
        if not hits:
            raise SystemExit("teacher model.ckpt not found — pass --teacher-ckpt")
        ckpt = Path(hits[0])
    sd = torch.load(ckpt, map_location="cpu", weights_only=False)
    sd = sd.get("state_dict", sd)
    keys = ("scale_mean", "scale_comps", "hand_pose_mean", "hand_pose_comps",
            "hand_joint_idxs_left", "hand_joint_idxs_right")
    out = {k: sd[f"head_pose.{k}"].numpy() for k in keys}
    for k in keys:                       # the hand head's copies must agree
        assert np.array_equal(out[k], sd[f"head_pose_hand.{k}"].numpy()), k
    print(f"teacher buffers from {ckpt}")
    return out


def gt_scales(data_root: Path, limit: int) -> np.ndarray:
    """The 68 bone scales (`model_params[136:204]`) from real annotations."""
    files = sorted(data_root.glob("*/annotations/*.npz"))[:limit]
    if not files:
        files = sorted(data_root.glob("annotations/*.npz"))[:limit]
    if not files:
        raise SystemExit(f"no annotations under {data_root}")
    rows = [np.load(f)["mhr_model_params"][136:204] for f in files]
    print(f"{len(rows):,} GT samples from {data_root}")
    return np.stack(rows).astype(np.float64)


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--data_root", default="data")
    ap.add_argument("--teacher-ckpt", type=Path, default=None)
    ap.add_argument("--limit", type=int, default=20000)
    ap.add_argument("--out", type=Path,
                    default=Path("training/assets/mhr_cont_head.npz"))
    args = ap.parse_args()

    buf = teacher_buffers(args.teacher_ckpt)
    comps = torch.from_numpy(buf["scale_comps"]).double()      # (28, 68)
    mean = torch.from_numpy(buf["scale_mean"]).double()        # (68,)

    U, S, Vt = torch.linalg.svd(comps, full_matrices=False)
    rank = int((S > RCOND * S[0]).sum())
    print(f"scale_comps singular values: {S[0]:.4g} ... {S[rank-1]:.4g} "
          f"| {S[rank]:.3g} ... {S[-1]:.3g}   -> rank {rank}/28")
    Sinv = torch.where(S > RCOND * S[0], 1.0 / S, torch.zeros_like(S))
    pinv = (Vt.T * Sinv) @ U.T                                 # (68, 28)

    scales = torch.from_numpy(gt_scales(Path(args.data_root), args.limit))
    coeffs = (scales - mean) @ pinv                            # (N, 28)
    recon = mean + coeffs @ comps
    resid = (recon - scales).abs().max()
    print(f"GT projection residual (max abs, 68 scales): {resid:.3e}")
    if resid > 1e-4:
        raise SystemExit("GT scales do not lie in the teacher's scale subspace — stop")

    lo, hi = coeffs.min(0).values, coeffs.max(0).values
    m = MARGIN_FRAC * (hi - lo)
    lo, hi = lo - m, hi + m
    # Symmetrise about zero. `ContMHRHead` maps the head output into [lo, hi]
    # with a tanh, so a freshly initialised head sits at the MIDPOINT -- and the
    # observed coefficient box is not centred (midpoint up to 3.01), which would
    # start every run at an arbitrary body size instead of `scale_mean`.
    # Widening to +-max(|lo|, |hi|) keeps every GT coefficient reachable, puts
    # tanh(0) exactly on the mean body, and still confines the 68 expanded
    # scales to the 24-dimensional subspace, which is what the bound is for.
    r = torch.maximum(lo.abs(), hi.abs())
    lo, hi = -r, r
    print(f"coefficient bounds: symmetric, widest +-{r.max():.3f}, "
          f"narrowest +-{r.min():.3f}")

    args.out.parent.mkdir(parents=True, exist_ok=True)
    np.savez(args.out,
             scale_mean=buf["scale_mean"].astype(np.float32),
             scale_comps=buf["scale_comps"].astype(np.float32),
             scale_pinv=pinv.numpy().astype(np.float32),
             scale_rank=np.int64(rank),
             coeff_lo=lo.numpy().astype(np.float32),
             coeff_hi=hi.numpy().astype(np.float32),
             hand_pose_mean=buf["hand_pose_mean"].astype(np.float32),
             hand_pose_comps=buf["hand_pose_comps"].astype(np.float32),
             hand_joint_idxs_left=buf["hand_joint_idxs_left"].astype(np.int64),
             hand_joint_idxs_right=buf["hand_joint_idxs_right"].astype(np.int64),
             n_gt=np.int64(len(scales)))
    print(f"wrote {args.out}")


if __name__ == "__main__":
    main()