Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.1 KB
Raw Blame
"""MDM (Motion Diffusion Model) sampler adapter.

Thin wrapper around GuyTevet/motion-diffusion-model (MIT) that turns a text
prompt into SMPL-skeleton motion: per-frame local joint rotations (axis-angle,
24 joints) plus a root translation. Everything torch/MDM-specific is imported
lazily inside `sample()` so this module imports cleanly in environments without
the GPU stack (the worker's pure conversion path + tests). Loaded once per
container by main.py's `_load_model()`.

The MDM repo is cloned into the image (see Dockerfile) and put on PYTHONPATH;
checkpoints are mounted from the weights bucket at `model_dir`.
"""

from __future__ import annotations

import logging

import numpy as np

from motion_decode import (
    _kabsch_quat,
    _parents_from_kinematic_chain,
    _positions_to_local_quats,
    _quat_conjugate,
    _quat_multiply,
    _quat_rotate,
    _quat_to_axis_angle,
    _resample,
    _rotation_matrix_to_quat,
)

log = logging.getLogger("text2motion.mdm")

# HumanML3D motion is sampled at 20 fps; we resample to the requested fps in the
# worker via the clip times. MDM's max horizon for the HumanML3D checkpoint.
HUMANML_FPS = 20
MAX_MOTION_LEN = 196


class MdmSampler:
    def __init__(self, model_dir: str, device: str = "cuda"):
        self.model_dir = model_dir
        self.device = device
        self._model = None
        self._diffusion = None
        self._load()

    def _load(self) -> None:
        # Imports are GPU-only: kept here so the module imports without torch.
        import json
        import os
        from types import SimpleNamespace

        import torch
        import torch.nn as nn
        import model.mdm as mdm_module
        from model.cfg_sampler import ClassifierFreeSampleModel
        from utils.model_util import create_model_and_diffusion, load_model_wo_clip

        # MDM.__init__ unconditionally builds `self.rot2xyz = Rotation2xyz(...)`,
        # whose constructor loads the real SMPL body model (a .pkl gated behind
        # SMPL's own registration/license at smpl.is.tue.mpg.de, not something
        # an automated pipeline can fetch). This worker never calls
        # `rot2xyz(...)` (the thing that would actually need real SMPL data):
        # motion comes back from the diffusion sampler as HumanML3D feature
        # vectors, decoded via `recover_from_ric` + this module's own
        # direction-alignment IK (`_positions_to_local_quats`), never via
        # rotation2xyz. But `MDM._apply`/`MDM.train` (invoked by the
        # `.to(device)`/`.eval()` calls below, which is how the model actually
        # loads onto the GPU) reach into `self.rot2xyz.smpl_model._apply`/
        # `.train`, so it still needs to be a real `nn.Module`, just one that
        # never had to load a license-gated file to exist. Patched on the
        # already-imported `model.mdm` module object so it takes effect
        # regardless of import order.
        class _StubSmplModel(nn.Module):
            def forward(self, *a, **kw):
                raise RuntimeError("SMPL rotation2xyz output is not used by this worker")

        class _StubRotation2xyz:
            def __init__(self, device, dataset="amass"):
                self.device = device
                self.dataset = dataset
                self.smpl_model = _StubSmplModel().to(device)

            def __call__(self, *a, **kw):
                raise RuntimeError("Rotation2xyz.__call__ is not used by this worker")

        mdm_module.Rotation2xyz = _StubRotation2xyz

        # utils.parser_util.generate_args() takes no parameters and parses
        # sys.argv (a CLI-only entry point, not usable from a server process:
        # a prior version of this file called it as `generate_args(model_path=...)`,
        # which doesn't match its signature at all). The released checkpoints
        # ship an `args.json` sibling to the `.pt` file with every field the
        # model/diffusion constructors need (see MDM's own
        # `load_args_from_model`); load that directly into a plain namespace.
        with open(os.path.join(self.model_dir, "args.json")) as fh:
            args = SimpleNamespace(**json.load(fh))
        self._args = args
        self._guidance_param = getattr(args, "guidance_param", 2.5)

        # create_model_and_diffusion()'s `data` param is only consulted for
        # `data.dataset.num_actions` (action-conditioned datasets); passing
        # None crashes on that attribute access before the humanml/text-cond
        # branch (which doesn't need it) is ever reached.
        dummy_data = SimpleNamespace(dataset=SimpleNamespace())
        model, diffusion = create_model_and_diffusion(args, dummy_data)
        state = torch.load(os.path.join(self.model_dir, "model.pt"), map_location="cpu", weights_only=True)
        load_model_wo_clip(model, state)
        # NOT `model.to(device).eval()`: torch's Module.to() returns
        # `self._apply(convert)` (verified against torch 2.3.1's own source),
        # but MDM's `_apply` override (`def _apply(self, fn): super()._apply(fn);
        # self.rot2xyz.smpl_model._apply(fn)`) never returns `self`, an
        # upstream bug that makes `.to()` return None regardless of which
        # smpl_model is attached. `model` itself is still correctly mutated
        # in place, so call `.to()`/`.eval()` for their side effects only and
        # keep using the original reference.
        model.to(self.device)
        model.eval()
        # Classifier-free guidance sampling wrapper: matches sample/generate.py's
        # own gating (skip the wrapper entirely when guidance_param == 1).
        if self._guidance_param != 1:
            model = ClassifierFreeSampleModel(model)
        self._model = model
        self._diffusion = diffusion
        log.info(
            "MDM model + diffusion ready on %s (arch=%s, diffusion_steps=%s, guidance=%s)",
            self.device, args.arch, args.diffusion_steps, self._guidance_param,
        )

    def sample(self, prompt: str, n_frames: int):
        """Text → (poses (T,24,3) axis-angle, trans (T,3)).

        Samples HumanML3D motion for `prompt`, recovers SMPL joint rotations and
        root translation, and resamples to `n_frames`.
        """
        import torch

        horizon = min(MAX_MOTION_LEN, max(2, int(round(n_frames * HUMANML_FPS / max(n_frames, 1))) or n_frames))
        horizon = min(MAX_MOTION_LEN, max(2, n_frames))

        # MDM.forward reads `y['mask'].shape[-1]` unconditionally (see model/mdm.py:
        # `is_valid_mask = y['mask'].shape[-1] > 1`) even though this deployment's
        # checkpoint has mask_frames=False, so the key must exist regardless. A
        # single unpadded prompt at exactly `horizon` frames has no padding, so
        # the honest mask a real (non-padded) batch's collate would produce is
        # all-ones over the full length: shape (bs, 1, 1, horizon), matching
        # sample/generate.py's own `model_kwargs['y']['mask']` convention.
        mask = torch.ones((1, 1, 1, horizon), dtype=torch.bool, device=self.device)
        model_kwargs = {
            "y": {"text": [prompt], "lengths": torch.tensor([horizon], device=self.device), "mask": mask}
        }
        if self._guidance_param != 1:
            model_kwargs["y"]["scale"] = torch.ones(1, device=self.device) * self._guidance_param
        sample_fn = self._diffusion.p_sample_loop
        with torch.no_grad():
            sample = sample_fn(
                self._model,
                (1, self._model.njoints, self._model.nfeats, horizon),
                clip_denoised=False,
                model_kwargs=model_kwargs,
                progress=False,
            )

        # Decode the HumanML3D vector representation back to joint rotations +
        # root translation, then to SMPL-indexed axis-angle.
        poses, trans = _decode_to_smpl(sample)
        poses = np.asarray(poses, dtype=np.float64)
        trans = np.asarray(trans, dtype=np.float64)
        poses, trans = _resample(poses, trans, n_frames)
        return poses, trans


def _decode_to_smpl(sample):
    """Decode an MDM sample tensor to (poses (T,22,3) axis-angle, trans (T,3)).

    MDM's HumanML3D checkpoint emits a 263-dim feature vector per frame.
    `recover_from_ric` (upstream, real) turns that back into global joint
    *positions* on the 22-joint HumanML3D skeleton: the SMPL body joints 0-21
    under the same kinematic tree (paramUtil.t2m_kinematic_chain).

    There is no upstream `positions_to_smpl_poses` in the MDM repo (a prior
    version of this file imported one that does not exist, verified against
    github.com/GuyTevet/motion-diffusion-model). Converting positions to local
    joint *rotations* is done here directly by `_positions_to_local_quats`: a
    joint's local rotation is recovered from where ITS OWN children ended up
    (not its own position, which under standard FK only depends on ancestor
    rotations), via a best-fit (Kabsch) rotation from each child's rest
    direction (`paramUtil.t2m_raw_offsets`, upstream data) to its measured
    direction. This carries no twist signal (same limitation HumanML3D's
    output has generally): `smpl_to_clip`'s `rest_offsets` param is the
    intended place to calibrate the residual SMPL-rest vs. Wolf3D-rest
    orientation offset.
    """
    import torch
    from data_loaders.humanml.scripts.motion_process import recover_from_ric
    from data_loaders.humanml.utils import paramUtil

    n_joints = 22

    # sample: (1, njoints=263, nfeats=1, T) → (T, 263). NOT
    # `.squeeze(0).permute(2,0,1)` (a prior version of this code did that): that
    # reorders to (T, 263, 1) and leaves the size-1 nfeats axis trailing, so
    # `recover_from_ric`'s `data[..., 0]` etc. (which assume the LAST dim is the
    # 263-wide feature vector) silently slice the wrong axis instead of failing
    # loudly. `.squeeze(1)` first removes nfeats, then permute puts T first.
    feats = sample.squeeze(0).squeeze(1).permute(1, 0).float()  # (T, 263)

    # The diffusion model samples in HumanML3D's NORMALIZED feature space
    # (zero mean / unit std over the training set): recover_from_ric expects
    # real units. Denormalize with the checkpoint-independent HumanML3D dataset
    # stats MDM ships in its own repo (`dataset/t2m_{mean,std}.npy`, small
    # precomputed arrays, not a license-gated model file), exactly the
    # `inv_transform` upstream's own sample/generate.py applies before calling
    # this same recover_from_ric.
    # PYTHONPATH=/opt/mdm is fixed by the Dockerfile's `git clone ... /opt/mdm`.
    mean = torch.from_numpy(np.load("/opt/mdm/dataset/t2m_mean.npy")).float().to(feats.device)
    std = torch.from_numpy(np.load("/opt/mdm/dataset/t2m_std.npy")).float().to(feats.device)
    feats = feats * std + mean

    positions = recover_from_ric(feats, n_joints)  # (T, 22, 3)
    positions = positions.cpu().numpy().astype(np.float64)

    parents = _parents_from_kinematic_chain(paramUtil.t2m_kinematic_chain, n_joints)
    rest_dirs = paramUtil.t2m_raw_offsets.astype(np.float64)  # (22, 3), joint 0 unused

    local_quat = _positions_to_local_quats(positions, parents, rest_dirs)  # (T, 22, 4) xyzw
    poses = _quat_to_axis_angle(local_quat)  # (T, 22, 3)
    trans = positions[:, 0, :]  # root joint IS the world root position
    return poses, trans