Skip to content
Python
9.1 KB
Raw Blame
#!/usr/bin/env python3
"""Run the student through a real DDP Reducer before submitting to the cluster.

    python tools/ddp_smoke.py --backbone hgnetv2_b4 --w-verts 0.35
    python tools/ddp_smoke.py --backbone repvit_m2_3        # the control
    python tools/ddp_smoke.py --image-size 288               # a non-224 input

WHY THIS EXISTS. A plain single-process smoke run -- `train_ddp.py` on a
laptop, `--self-test`, `--overfit-test` -- builds no DDP Reducer at all, so it
cannot see the one class of failure that only exists in a multi-GPU run: a
parameter that never receives a gradient. `train_ddp.py` under torchrun runs
`DistributedDataParallel` with the default `find_unused_parameters=False`, which
aborts on the SECOND optimiser step with

    Expected to have finished reduction in the prior iteration before starting
    a new one ... Parameter indices which did not receive grad for rank N: 241

and nothing before that step warns you. On 2026-09-11 that cost all four
generation-7 `hgnetv2_b4` jobs: `forward_features()` stops before the timm
classifier head, and `num_classes=0` empties that head for `repvit_m2_3` but
leaves `hgnetv2_b4` a 2048x2048 `last_conv` -- 4,194,304 parameters with no
gradient. The repvit arms ran for nine hours while every hgnet arm died at
step 2.

World size 1 is enough: the Reducer and its unused-parameter check are built and
run identically at any world size, so this reproduces the cluster error exactly,
on one GPU, in under a minute. The batch size is small for the same reason --
see --batch_size; on a login node a large one is killed for memory, not for
anything this is testing. It does NOT test NCCL, multi-node bring-up or
gradient bucketing across ranks -- only that every parameter participates.

Exit status is 0 when every parameter received a gradient, 1 otherwise, so it
can gate a submission.
"""

from __future__ import annotations

import argparse
import os
import sys
from pathlib import Path

import numpy as np
import torch
import torch.distributed as dist

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "training"))
import train as T  # noqa: E402


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__,
                                formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--backbone", default=None, help="timm name; default is the config's")
    p.add_argument("--context-crops", dest="context_crops", action="store_true",
                   help="--preset v3's dataset; --data_root must be a context-crop "
                        "split (build_split.py --context 2.0 --body-size 448)")
    p.add_argument("--crop-windows", dest="crop_windows", action="store_true",
                   help="--preset v4's dataset; --data_root must be a window "
                        "split (build_split.py --windows 4) at --image-size")
    p.add_argument("--image-size", dest="image_size", type=int, default=None,
                   help="network input side; the positional grid and every crop follow it")
    p.add_argument("--data_root", default="data")
    p.add_argument("--batch_size", type=int, default=2,
                   help="Small on purpose. Which parameters receive a gradient "
                        "does not depend on the batch size, and a shared "
                        "login node may kill a process around 3-4 GB: measured peak "
                        "RSS is 4.49 GB for repvit_m2_3 at batch 8 (killed) "
                        "against 2.45 GB at batch 2. Raise it only if you are on "
                        "a compute node.")
    p.add_argument("--losses", choices=("legacy", "rebalanced"), default="rebalanced")
    p.add_argument("--w-verts", dest="w_verts", type=float, default=0.0)
    p.add_argument("--bound-scales", dest="bound_scales", action="store_true", default=True)
    p.add_argument("--cliff-focal", dest="cliff_focal", action="store_true", default=True)
    p.add_argument("--crop-centre-fix", dest="crop_centre_fix", action="store_true", default=True)
    p.add_argument("--exact-landmarks", dest="exact_landmarks", action="store_true",
                   help="Use the teacher's exact landmark readout, which widens "
                        "the skinned vertex subset.")
    p.add_argument("--cont-head", dest="cont_head", action="store_true",
                   help="Build the teacher's continuous regression head instead "
                        "of the linear 204-vector head. A changed head shape is "
                        "exactly the class of change this exists to gate.")
    p.add_argument("--prev-token", dest="prev_token", action="store_true",
                   help="Foot lock: the previous-pose query, the contact head and "
                        "the three foot-lock losses, all at weight 1.")
    p.add_argument("--contact-floor", dest="contact_floor", default="",
                   help="With --prev-token: Harmony4D contact by height (tools/build_contact_floor.py).")
    p.add_argument("--contact-flight-weight", dest="contact_flight_weight", type=float, default=1.0)
    p.add_argument("--pretrained", action="store_true",
                   help="Load the timm weights too. Off by default: the head "
                        "geometry this checks does not depend on them.")
    args = p.parse_args()

    os.environ.setdefault("MASTER_ADDR", "127.0.0.1")
    os.environ.setdefault("MASTER_PORT", "29517")
    # gloo, not nccl: this needs no GPU-to-GPU transport and must run on a
    # laptop with one card (or none).
    dist.init_process_group(backend="gloo", rank=0, world_size=1)

    cfg = T.DistillConfig()
    T.cfg = cfg
    if args.losses == "rebalanced":
        T.apply_rebalanced_losses(cfg)
    cfg.data_root = args.data_root
    cfg.bound_scales = args.bound_scales
    cfg.cliff_focal = args.cliff_focal
    cfg.crop_centre_fix = args.crop_centre_fix
    cfg.w_verts = args.w_verts
    if args.cont_head:
        cfg.cont_head = True
        cfg.root_rot_loss = True
    if args.exact_landmarks:
        cfg.exact_landmarks = True
    if args.prev_token:
        cfg.prev_token = cfg.contact_head = True
        cfg.w_contact = cfg.w_zero_vel = cfg.w_foot_mocap = 1.0
        cfg.contact_floor = args.contact_floor
        cfg.contact_flight_weight = args.contact_flight_weight
    if args.backbone:
        cfg.backbone = args.backbone
    if args.image_size is not None:
        if args.image_size % 32:
            raise SystemExit(f"--image-size {args.image_size} is not a multiple of 32")
        cfg.image_size = args.image_size

    ds = T.SAM3DStudentDataset(cfg.data_root, augment=True, max_images=args.batch_size,
                               per_dataset_caps={}, cliff_focal=cfg.cliff_focal,
                               crop_centre_fix=cfg.crop_centre_fix,
                               image_size=cfg.image_size,
                               context_crops=args.context_crops,
                               crop_windows=args.crop_windows,
                               temporal=T.temporal_kwargs(cfg))
    loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, num_workers=0)
    batch = {k: (v.to(T.device) if torch.is_tensor(v) else v)
             for k, v in next(iter(loader)).items()}

    mhr = T.MHRForwardPass(
        cfg.mhr_model_path, T.device, kp_regressor=np.load(cfg.kp_regressor_path),
        n_verts=cfg.n_verts if cfg.w_verts > 0 else 0,
        landmark_assets=cfg.landmark_assets_path if cfg.exact_landmarks else None)
    T.mhr_module = mhr

    model = T.InstantHMRStudent(cfg, pretrained=args.pretrained).to(T.device)
    n = sum(q.numel() for q in model.parameters())
    print(f"{cfg.backbone}: {n/1e6:.2f} M parameters, "
          f"head={'continuous' if cfg.cont_head else 'linear'}, "
          f"landmarks={'exact' if cfg.exact_landmarks else 'fitted'}, "
          f"w_verts={cfg.w_verts}, batch {args.batch_size}")

    ddp = torch.nn.parallel.DistributedDataParallel(model)   # find_unused_parameters=False
    crit = T.DistillationLoss(cfg, mhr)
    crit.warmup_steps = 0                                    # full-strength FK terms

    # TWO iterations. The Reducer only reports unused parameters when a new
    # forward starts before the previous reduction completed, so one step passes
    # even on a broken model -- which is how this was missed the first time.
    try:
        for _ in range(2):
            out = ddp(batch["image"], batch["cliff_cond"].to(T.device),
                      batch.get("prev_pose"), batch.get("has_prev"))
            crit(out, batch)["total_loss"].backward()
    except RuntimeError as e:
        print(f"\n❌ DDP rejected {cfg.backbone}:\n{e}")
        print("\nThe named index is a position in list(model.parameters()); "
              "list(model.named_parameters())[i] gives the parameter.")
        return 1
    finally:
        dist.destroy_process_group()

    missing = [k for k, q in model.named_parameters() if q.grad is None]
    if missing:
        print(f"\n❌ {len(missing)} parameter(s) never received a gradient:")
        for k in missing:
            print(f"   {k}")
        return 1

    print(f"✅ every parameter received a gradient over 2 DDP steps")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())