Skip to content

Type a motion, watch a rigged character perform it: fast video model + GVHMR + live Mixamo retargeting

JavaScript
15.4 KB
Raw Blame
"""Prompt2Rig HQ worker — keeps GVHMR warm on the GPU and turns a clip URL into SMPL motion.

POST /process {"video_url": "..."} → {n, fps, global_orient, body_pose, transl, leg_length, timings}
GET  /health                        → {ok, device, error?}

Run:  hq/.venv/Scripts/python hq/worker.py      (listens on 127.0.0.1:5178; setup in README)
"""

from __future__ import annotations

import hashlib
import os
import shutil
import subprocess
import time
import traceback
import urllib.request
from pathlib import Path

HQ = Path(__file__).resolve().parent
os.environ.setdefault("GVHMR_CHECKPOINTS", str(HQ / "weights"))
os.environ.setdefault("GVHMR_BODY_MODELS", str(HQ / "body_models"))
CACHE = HQ / "cache"
SMPLX = Path(os.environ["GVHMR_BODY_MODELS"]) / "smplx" / "SMPLX_NEUTRAL.npz"
DETECTOR = os.environ.get("HQ_DETECTOR", "fast")
TRACKER = os.environ.get("HQ_TRACKER", "bytetrack.yaml")
POSE2D = os.environ.get("HQ_POSE2D", "vitpose")
# ViTPose flip-test runs every crop twice (mirrored) for a small accuracy gain.
POSE_FLIP = os.environ.get("HQ_POSE_FLIP", "0") == "1"

import numpy as np  # noqa: E402
import torch  # noqa: E402

try:  # let onnxruntime-gpu (RTMPose) use the CUDA/cuDNN DLLs that ship with torch
    import onnxruntime

    onnxruntime.preload_dlls()
except Exception:
    pass

from fastapi import FastAPI, HTTPException  # noqa: E402
from pydantic import BaseModel  # noqa: E402

app = FastAPI()
state: dict = {"pipe": None, "error": None, "device": None}
# All GPU work runs on ONE thread: CUDA/cuBLAS/cuDNN handles are per-thread, so a fresh request
# thread would otherwise pay their setup (seconds) and miss the warm-up.
from concurrent.futures import ThreadPoolExecutor  # noqa: E402

GPU = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gpu")


def log(msg: str) -> None:
    print(f"[hq {time.strftime('%H:%M:%S')}] {msg}", flush=True)


FRAME_CACHE: dict[str, list] = {}


def fast_read_video_np(video_path, start_frame=0, end_frame=-1, scale=1.0):
    """Drop-in for gvhmr's PyAV reader: reuse the detector's decoded frames, resize with OpenCV.

    Same output contract (N, H, W, 3) RGB uint8; bicubic like ffmpeg's default scale filter.
    """
    import cv2

    frames = FRAME_CACHE.get(str(Path(video_path).resolve()))
    if frames is None:
        cap = cv2.VideoCapture(str(video_path))
        frames = []
        while True:
            ok, img = cap.read()
            if not ok:
                break
            frames.append(img)
        cap.release()
    frames = frames[start_frame:(None if end_frame == -1 else end_frame)]
    if scale != 1.0:
        h, w = frames[0].shape[:2]
        size = (int(w * scale), int(h * scale))
        frames = [cv2.resize(f, size, interpolation=cv2.INTER_CUBIC) for f in frames]
    return np.ascontiguousarray(np.stack(frames)[..., ::-1])


class FastSingleDetector:
    """Batched YOLO detection keeping the largest person per frame — replaces per-frame tracking.

    Generated clips contain exactly one person, so identity tracking buys nothing; batching the
    detector is ~5x faster than ultralytics' frame-by-frame `track()`.
    """

    _inst = None

    @classmethod
    def get(cls):
        cls._inst = cls._inst or cls()
        return cls._inst

    def __init__(self, ckpt: Path = HQ / "weights" / "yolo11m.pt", batch: int = 32):
        from ultralytics import YOLO
        from ultralytics.utils.downloads import attempt_download_asset

        ckpt.parent.mkdir(parents=True, exist_ok=True)
        self.yolo, self.batch = YOLO(attempt_download_asset(str(ckpt))), batch

    def get_one_track(self, video_path) -> torch.Tensor:
        import cv2
        from gvhmr.utils.net_utils import moving_average_smooth

        cap = cv2.VideoCapture(str(video_path))
        frames = []
        while True:
            ok, img = cap.read()
            if not ok:
                break
            frames.append(img)
        cap.release()
        FRAME_CACHE.clear()
        FRAME_CACHE[str(Path(video_path).resolve())] = frames
        boxes = np.zeros((len(frames), 4), np.float32)
        found = np.zeros(len(frames), bool)
        for i in range(0, len(frames), self.batch):
            chunk = frames[i:i + self.batch]
            # Pad to a fixed batch size: every new batch shape costs a slow first CUDA call.
            pad = [chunk[-1]] * (self.batch - len(chunk))
            results = self.yolo.predict(chunk + pad, classes=[0], conf=0.3, half=True, device=0, verbose=False)[:len(chunk)]
            for j, r in enumerate(results):
                if len(r.boxes):
                    xyxy = r.boxes.xyxy.cpu().numpy()
                    area = (xyxy[:, 2] - xyxy[:, 0]) * (xyxy[:, 3] - xyxy[:, 1])
                    boxes[i + j], found[i + j] = xyxy[area.argmax()], True
        if not found.any():
            raise RuntimeError("no person detected in the clip")
        idx = np.arange(len(frames))
        for k in range(4):  # fill frames with no detection
            boxes[:, k] = np.interp(idx, idx[found], boxes[found, k])
        return moving_average_smooth(torch.from_numpy(boxes), window_size=5, dim=0)


def load_pipeline() -> None:
    if not SMPLX.exists():
        state["error"] = f"SMPL-X missing: put SMPLX_NEUTRAL.npz in {SMPLX.parent}"
        log(state["error"])
        return
    try:
        import gvhmr
        import gvhmr.cli.demo as demo

        if DETECTOR == "fast":
            demo.make_detector = lambda *a, **kw: FastSingleDetector.get()  # cfg still names a valid YOLO preset
            import gvhmr.utils.imgcrop as imgcrop

            imgcrop.read_video_np = fast_read_video_np

        t0 = time.perf_counter()
        state["pipe"] = gvhmr.pipeline("human-motion-recovery", device="cuda")
        state["device"] = torch.cuda.get_device_name(0)
        state["error"] = None
        log(f"GVHMR loaded on {state['device']} in {time.perf_counter() - t0:.1f}s")
        warmup()
    except Exception as e:  # noqa: BLE001
        state["error"] = f"{type(e).__name__}: {e}"
        traceback.print_exc()


def warmup() -> None:
    """Run one tiny clip so CUDA kernels / ONNX sessions are initialized before the first real request."""
    clip = CACHE / "_warmup.mp4"
    if not clip.exists():
        return
    try:
        t0 = time.perf_counter()
        shutil.rmtree(CACHE / "_warmup", ignore_errors=True)  # cached stages would skip the warm-up
        result, _ = recover(clip, CACHE / "_warmup")
        leg_length(result.smpl_params_world)
        method = hands_method(None)
        if method:  # load + warm the hand model too (WiLoR's checkpoint is 2.5 GB)
            import hands as handmod

            frames, fps = handmod.read_frames(clip)
            HANDS_EXEC.submit(lambda: handmod.backend(method)(frames[:32], fps)).result()
        if DETECTOR == "fast":  # first YOLO call per input shape is slow: cover both orientations
            det = FastSingleDetector.get()
            for h, w in ((480, 832), (832, 480), (470, 854), (854, 470)):  # H3 480P, FLUX 3 720p (downscaled)
                for n in (det.batch,):
                    det.yolo.predict([np.zeros((h, w, 3), np.uint8)] * n, classes=[0], half=True, device=0, verbose=False)
        log(f"warmup done in {time.perf_counter() - t0:.1f}s")
    except Exception as e:  # noqa: BLE001
        log(f"warmup failed: {e}")


def recover(video: Path, out_dir: Path):
    from motion2mixamorig.extract import _stabilize_pose_cache

    phases: dict[str, float] = {}
    last = {"name": None, "t": time.perf_counter()}

    def on_progress(_frac: float, desc: str) -> None:
        now = time.perf_counter()
        if desc != last["name"]:
            if last["name"]:
                phases[last["name"]] = phases.get(last["name"], 0) + now - last["t"]
            last.update(name=desc, t=now)

    # Generated clips are single-person, plain background, static camera: a mid-size YOLO with ByteTrack
    # (no camera-motion compensation) tracks as well as YOLOv8x+BoT-SORT at a fraction of the cost, and
    # ViTPose shares one decode+crop pass with the HMR2 feature backbone.
    kwargs = dict(static_camera=True, detector="yolo11m" if DETECTOR == "fast" else DETECTOR, pose2d=POSE2D, set_overrides=[f"++detector.tracker={TRACKER}"] + ([] if POSE_FLIP else ["++pose2d.flip_test=false"]),
                  output_dir=str(out_dir), output_root=str(out_dir.parent), progress=False, progress_callback=on_progress)
    t0 = time.perf_counter()
    result = state["pipe"](str(video), **kwargs)
    t1 = time.perf_counter()
    if last["name"]:
        phases[last["name"]] = phases.get(last["name"], 0) + t1 - last["t"]
    log("phases: " + "  ".join(f"{k} {v:.2f}s" for k, v in phases.items()))
    # RTMPose can swap left/right per frame when the subject turns; fix the 2D cache and redo the
    # (cheap) 3D lift once if anything changed — same trick Motion2MixamoRig uses.
    pose_cache = out_dir / "preprocess" / "vitpose.pt"
    if pose_cache.exists() and _stabilize_pose_cache(pose_cache):
        (out_dir / "hmr4d_results.pt").unlink(missing_ok=True)
        result = state["pipe"](str(video), **kwargs)
    return result, {"recover": t1 - t0, "stabilize": time.perf_counter() - t1}


_BODY: dict = {}


def cv2_size(path) -> tuple[int, int]:
    import cv2

    cap = cv2.VideoCapture(str(path))
    w, h = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    cap.release()
    return w, h


def leg_length(params) -> float:
    """Mean hip→ankle length (m) of the recovered body; models cached across calls.

    SMPL-X → SMPL joint regression adapted from Motion2MixamoRig (Apache-2.0), see NOTICE.
    """
    from einops import einsum
    from gvhmr import PROJ_ROOT
    from gvhmr.utils.smplx_utils import make_smplx

    if not _BODY:
        dev = torch.device("cuda")
        _BODY["smplx"] = make_smplx("supermotion").to(dev)
        _BODY["x2s"] = torch.load(PROJ_ROOT / "gvhmr/utils/body_model/smplx2smpl_sparse.pt", weights_only=False).to(dev)
        _BODY["jreg"] = torch.load(PROJ_ROOT / "gvhmr/utils/body_model/smpl_neutral_J_regressor.pt", weights_only=False).to(dev)
    p = {k: v[:1].cuda() for k, v in params.items() if hasattr(v, "cuda")}
    with torch.no_grad():
        verts = _BODY["smplx"](**p).vertices
        verts = torch.stack([torch.matmul(_BODY["x2s"], v) for v in verts])
        j = einsum(_BODY["jreg"], verts, "j v, l v i -> l j i")[0].cpu().numpy()
    seg = lambda a, b: float(np.linalg.norm(j[a] - j[b]))  # noqa: E731
    return (seg(1, 4) + seg(4, 7) + seg(2, 5) + seg(5, 8)) / 2


class ProcessIn(BaseModel):
    video_url: str
    hands: str | None = None  # "wilor" | "mediapipe" | "off" (default: HQ_HANDS / auto)


HANDS_EXEC = ThreadPoolExecutor(max_workers=1, thread_name_prefix="hands")


def hands_method(requested: str | None) -> str | None:
    m = (requested or os.environ.get("HQ_HANDS", "auto")).lower()
    if m == "off":
        return None
    if m == "auto":
        return "wilor" if (HQ / "weights" / "wilor" / "pretrained_models" / "MANO_RIGHT.pkl").exists() else "mediapipe"
    return m


@app.get("/health")
def health():
    return {"ok": state["pipe"] is not None, "device": state["device"], "error": state["error"]}


@app.post("/process")
def process(body: ProcessIn):
    if state["pipe"] is None:
        GPU.submit(load_pipeline).result()
        if state["pipe"] is None:
            raise HTTPException(503, state["error"] or "GVHMR not loaded")

    t0 = time.perf_counter()
    key = hashlib.sha1(body.video_url.encode()).hexdigest()[:12]
    out_dir = CACHE / key
    out_dir.mkdir(parents=True, exist_ok=True)
    video = out_dir / "input.mp4"
    if not video.exists():
        src = out_dir / "source.mp4"
        with urllib.request.urlopen(body.video_url, timeout=30) as r, open(src, "wb") as f:
            shutil.copyfileobj(r, f)
        # GVHMR runs at 30fps and would re-encode a 24fps clip with a slow preset; do it fast here.
        # It only ever sees 256px person crops, so cap the long side at 854px (H3 480P size): 720p+
        # sources (FLUX 3) otherwise cost ~2x in decode/crop for no accuracy gain.
        fit = "scale='if(gt(iw,ih),min(854,iw),-2)':'if(gt(iw,ih),-2,min(854,ih))'"
        subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", str(src), "-vf", f"fps=30,{fit}", "-an",
                        "-c:v", "libx264", "-preset", "ultrafast", "-crf", "16", str(video)], check=True)
    t_dl = time.perf_counter() - t0
    if not (CACHE / "_warmup.mp4").exists():  # reused to warm CUDA kernels on the next start
        shutil.copyfile(video, CACHE / "_warmup.mp4")

    # Hands only need the full-res clip: detect them while GVHMR runs, assign to wrists afterwards.
    import hands as handmod

    method = hands_method(body.hands)
    src = out_dir / "source.mp4"
    t_h0 = time.perf_counter()
    hand_job = HANDS_EXEC.submit(handmod.detect, src if src.exists() else video, method) if method else None

    to_np = lambda x: (x.detach().cpu().numpy() if hasattr(x, "detach") else np.asarray(x)).astype(np.float32)  # noqa: E731
    def run():
        result, t = recover(video, out_dir)
        world = result.smpl_params_world
        t_ll = time.perf_counter()
        ll = leg_length(world)
        t["leg_length"] = time.perf_counter() - t_ll
        return result, t, world, ll, to_np(result.smpl_params_camera["global_orient"])

    result, t, world, ll, incam_go = GPU.submit(run).result()
    incam_go0 = incam_go[0]

    go, bp, tr = to_np(world["global_orient"]), to_np(world["body_pose"]), to_np(world["transl"])
    n = int(go.shape[0])

    from motion2mixamorig.extract import playback_fps

    fps = playback_fps(video, n, fallback=30.0)

    hands_out, t_hands = None, None
    if hand_job is not None:
        try:
            import torch

            dets, src_fps, hw = hand_job.result()
            vit = np.asarray(torch.load(out_dir / "preprocess" / "vitpose.pt", weights_only=False))
            cap = cv2_size(video)
            hands_out = handmod.finish(dets, src_fps, hw, vit, max(cap), go, incam_go, n, fps)
            hands_out["method"] = method
            t_hands = time.perf_counter() - t_h0
        except Exception as e:  # noqa: BLE001 — hands are a bonus; never fail the body motion
            log(f"hands failed: {type(e).__name__}: {e}")
    total = time.perf_counter() - t0
    hand_note = (f"  hands[{method}] {t_hands:.2f}s R {hands_out['R']['coverage']:.0%} L {hands_out['L']['coverage']:.0%}"
                 if hands_out else "")
    log(f"{key}: {n} frames @ {fps:.1f}fps  download {t_dl:.2f}s  gvhmr {t['recover']:.2f}s  "
        f"stabilize {t['stabilize']:.2f}s  legs {t['leg_length']:.2f}s{hand_note}  total {total:.2f}s")
    return {
        "n": n, "fps": fps, "leg_length": ll,
        "global_orient": go.reshape(-1).round(5).tolist(),
        "body_pose": bp.reshape(n, -1)[:, :63].reshape(-1).round(5).tolist(),
        "transl": tr.reshape(-1).round(5).tolist(),
        # Frame-0 orientation relative to the camera (OpenCV axes): lets the client face the character
        # the way the person faces the camera, so chained clips keep a consistent heading.
        "incam_global_orient0": incam_go0.round(5).tolist(),
        # Finger tracking: per hand 21 wrist-relative keypoints/frame in world axes (see hq/hands.py).
        "hands": hands_out,
        "elapsed": total, "timings": {"download": t_dl, **t},
    }


if __name__ == "__main__":
    import uvicorn

    CACHE.mkdir(exist_ok=True)
    GPU.submit(load_pipeline)
    uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("HQ_PORT", 5178)), log_level="warning")