Skip to content

Official implementation of MotionBricks: Scalable Real-Time Motions with Modular Latent Generative Model and Smart Primitives (SIGGRAPH 2026)

Python
22.7 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"Normalization and conversion between local and global motion features."

from __future__ import annotations

from types import SimpleNamespace
from typing import Optional, Tuple

import numpy as np
import torch


def quat_conjugate(a):
    """Return the conjugates of WXYZ quaternions."""
    shape = a.shape
    a = a.reshape(-1, 4)
    return torch.cat((a[:, 0:1], -a[:, 1:]), dim=-1).view(shape)


def quat_apply(a, b):
    """Rotate vectors by WXYZ quaternions."""
    shape = b.shape
    a = a.reshape(-1, 4)
    b = b.reshape(-1, 3)
    xyz = a[:, 1:].clone()
    t = xyz.cross(b, dim=-1) * 2
    return (b + a[:, 0:1].clone() * t + xyz.cross(t, dim=-1)).view(shape)


def diff_angles(angles, fps: float):
    """Return the angular velocity between neighboring angles, by the shortest turn."""
    cos = torch.cos(angles)
    sin = torch.sin(angles)
    cos_diff = cos[..., 1:] * cos[..., :-1] + sin[..., 1:] * sin[..., :-1]
    sin_diff = sin[..., 1:] * cos[..., :-1] - cos[..., 1:] * sin[..., :-1]
    angles_diff = fps * torch.arctan2(sin_diff, cos_diff)
    return angles_diff


class Stats(torch.nn.Module):
    """Mean and standard deviation used to normalize motion features."""

    def __init__(self, eps=1e-05, legacy=False):
        super().__init__()
        self.legacy = legacy
        self.eps = eps

    def register_from_tensors(self, mean: torch.Tensor, std: torch.Tensor):
        """Store the mean and standard deviation as buffers."""
        self.register_buffer("mean", mean, persistent=False)
        self.register_buffer("std", std, persistent=False)

    def normalize(self, data: torch.Tensor, index=None) -> torch.Tensor:
        """Normalize data, optionally for a subset of feature indices."""
        mean = self.mean.to(device=data.device, dtype=data.dtype)
        std = self.std.to(device=data.device, dtype=data.dtype)
        if index is not None:
            mean = mean[..., index]
            std = std[..., index]
        if self.legacy:
            return (data - mean) / torch.clip(std, 0.001)
        return (data - mean) / torch.sqrt(std**2 + self.eps)

    def unnormalize(self, data: torch.Tensor, index=None) -> torch.Tensor:
        """Undo normalize()."""
        mean = self.mean.to(device=data.device, dtype=data.dtype)
        std = self.std.to(device=data.device, dtype=data.dtype)
        if index is not None:
            mean = mean[..., index]
            std = std[..., index]
        if self.legacy:
            return data * torch.clip(std, 0.001) + mean
        return data * torch.sqrt(std**2 + self.eps) + mean

    def is_loaded(self):
        """Return whether the statistics are set."""
        return hasattr(self, "mean")

    def get_dim(self):
        """Return the number of features the statistics cover."""
        return self.mean.shape[0]


class MotionRepBase(torch.nn.Module):
    """Base for motion feature layouts, with their sample rate, skeleton, and statistics."""

    def __init__(
        self,
        fps: float,
        skeleton: SimpleNamespace,
        name: str,
        stats: Optional[Stats] = None,
    ):
        super().__init__()
        self.stats = stats
        self.fps = fps
        self.skeleton = skeleton
        self.name = name
        self.num_joints = self.nbjoints = skeleton.nbjoints

    def normalize(self, motion: torch.Tensor, index=None) -> torch.Tensor:
        """Normalize motion features with the layout's statistics."""
        return self.stats.normalize(motion, index=index)

    def unnormalize(self, motion: torch.Tensor, index=None) -> torch.Tensor:
        """Undo normalize()."""
        return self.stats.unnormalize(motion, index=index)


class SeparatedRootLocalBody(MotionRepBase):
    """Motion features split into a root part and a body part."""

    def __init__(
        self,
        fps: float,
        skeleton: SimpleNamespace,
        name: str,
        stats: Optional[Stats] = None,
    ):
        super().__init__(fps, skeleton, name, stats)
        self.name = name
        [*self.root_keys], [*self.root_ps] = zip(*self.root_keys_dim.items())
        [*self.body_keys], [*self.body_ps] = zip(*self.body_keys_dim.items())
        self.keys_dim = self.root_keys_dim | self.body_keys_dim
        self.keys = self.root_keys + self.body_keys
        self.ps = self.root_ps + self.body_ps
        self.motion_rep_dim = sum((sum(x) if x else 1 for x in self.ps))
        self.motion_root_dim = sum((sum(x) if x else 1 for x in self.root_ps))
        self.motion_body_dim = sum((sum(x) if x else 1 for x in self.body_ps))
        if stats is not None and stats.is_loaded():
            assert self.motion_rep_dim == stats.get_dim()
        self.motion_rep_subset_dim = self.motion_rep_dim
        self.indices = {}
        idx = 0
        for key, x in zip(self.keys, self.ps):
            dim = sum(x) if x else 1
            self.indices[key] = np.arange(idx, idx + dim)
            idx += dim
        self.indices["all"] = np.arange(0, self.motion_rep_dim)
        self.indices["root"] = np.arange(0, self.motion_root_dim)
        self.indices["body"] = np.arange(
            self.motion_root_dim, self.motion_root_dim + self.motion_body_dim
        )
        self.removed_heading = self.compute_kwargs.get("removing_heading", True)
        self.using_smooth_root = self.compute_kwargs.get("using_smooth_root", False)
        self.extra_skel = self.compute_kwargs.get("extra_skel", False)
        if self.extra_skel:
            assert repr(self.skeleton.__class__).split("'")[1].endswith("Extra"), (
                "The skeleton should be an extra skeleton"
            )

    def slice(self, motion, key: str):
        """Return the features stored under key."""
        assert key in self.indices
        assert motion.shape[-1] == self.motion_rep_dim
        return motion[..., self.indices[key]]

    def detect_all_body_or_root(self, motion: torch.Tensor):
        """Return which part motion holds: "all", "root", or "body".

        The width of the last axis decides.
        """
        dim = motion.shape[-1]
        if dim == self.motion_rep_dim:
            return "all"
        elif dim == self.motion_root_dim:
            return "root"
        elif dim == self.motion_body_dim:
            return "body"
        else:
            raise ValueError(f"Unrecognized input dim: {dim}.")

    def normalize(self, motion: torch.Tensor, index=None) -> torch.Tensor:
        """Normalize motion features.

        Without index, the part they cover decides which statistics apply.
        """
        if index is None:
            index = self.indices[self.detect_all_body_or_root(motion)]
        return self.stats.normalize(motion, index=index)

    def unnormalize(self, motion: torch.Tensor, index=None) -> torch.Tensor:
        """Undo normalize()."""
        if index is None:
            index = self.indices[self.detect_all_body_or_root(motion)]
        return self.stats.unnormalize(motion, index=index)

    def extract_root(self, motion: torch.Tensor) -> torch.Tensor:
        """Return the root features of motion."""
        type = self.detect_all_body_or_root(motion)
        if type == "root":
            return motion
        elif type == "all":
            return self.slice(motion, "root")
        else:
            raise ValueError("Cannot compute root info without root data.")

    def extract_body(self, motion: torch.Tensor) -> torch.Tensor:
        """Return the body features of motion."""
        type = self.detect_all_body_or_root(motion)
        if type == "body":
            return motion
        elif type == "all":
            return self.slice(motion, "body")
        else:
            raise ValueError("Cannot compute body info without body data.")

    def concat_root_body(self, root_data: torch.Tensor, body_data: torch.Tensor):
        """Join root and body features into full motion features."""
        assert root_data.shape[-1] == self.motion_root_dim
        assert body_data.shape[-1] == self.motion_body_dim
        motion_data = torch.cat([root_data, body_data], axis=-1)
        assert motion_data.shape[-1] == self.motion_rep_dim
        return motion_data


class GlobalRootLocalBody(SeparatedRootLocalBody):
    """Motion features with the root position and heading given in model space."""

    dual_class = None

    def __init__(
        self,
        fps: float,
        skeleton: SimpleNamespace,
        name: str,
        stats: Optional[Stats] = None,
    ):
        self.root_keys_dim = {"global_root_pos": [3], "global_root_heading": [2]}
        self.body_keys_dim = self.get_body_keys_dim(skeleton.nbjoints)
        if self.dual_class is not None and self.dual_class._name_ in name:
            super().__init__(fps, skeleton, name, stats=None)
            self.dual_rep = self.dual_class(fps, skeleton, name, stats=stats)
            self.dual_rep_mode = "global"
            self.stats = self.dual_rep.global_motion_rep.stats
        else:
            self.dual_rep = None
            super().__init__(fps, skeleton, name, stats)
        self.indices["global_root_pos_2d"] = self.indices["global_root_pos"][[0, 2]]
        self.root_mode = "global"

    def compute_root_pos_and_rot(
        self, motion: torch.Tensor, return_quat: bool = True, return_angle: bool = False
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """Return the root positions and rotations, as quaternions or angles."""
        root_motion = self.extract_root(motion)
        r_pos = root_motion[..., :3]
        rot_cos = root_motion[..., 3]
        rot_sin = root_motion[..., 4]
        r_rot_ang = torch.atan2(rot_sin, rot_cos)
        r_rot_quat = torch.stack(
            [
                torch.cos(r_rot_ang / 2),
                torch.zeros_like(rot_cos),
                torch.sin(r_rot_ang / 2),
                torch.zeros_like(rot_cos),
            ],
            dim=-1,
        )
        if return_quat and (not return_angle):
            return (r_pos, r_rot_quat)
        if not return_quat and return_angle:
            return (r_pos, r_rot_ang)
        if return_quat and return_angle:
            return (r_pos, r_rot_quat, r_rot_ang)
        if not return_quat and (not return_angle):
            return r_pos

    def compute_root_rep_from_root_pos_and_rot(
        self,
        r_pos: torch.Tensor,
        r_rot_quat: torch.Tensor,
        lengths: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        """Build root features from root positions and rotations."""
        root_rot_angles = torch.arctan2(r_rot_quat[..., 2], r_rot_quat[..., 0]) * 2
        root_motion = torch.cat(
            [
                r_pos,
                torch.cos(root_rot_angles)[..., None],
                torch.sin(root_rot_angles)[..., None],
            ],
            axis=-1,
        )
        return root_motion


class LocalRootLocalBody(SeparatedRootLocalBody):
    """Motion features with the root given as velocities and height."""

    dual_class = None

    def __init__(
        self,
        fps: float,
        skeleton: SimpleNamespace,
        name: str,
        stats: Optional[Stats] = None,
    ):
        self.nfeats_vel = 3 if self.compute_kwargs["local_root_vel_with_y"] else 2
        self.root_keys_dim = {
            "local_root_rot_vel": [],
            "local_root_vel": [self.nfeats_vel],
            "global_root_y": [],
        }
        self.body_keys_dim = self.get_body_keys_dim(skeleton.nbjoints)
        if self.dual_class is not None and self.dual_class._name_ in name:
            super().__init__(fps, skeleton, name, stats=None)
            self.dual_rep = self.dual_class(fps, skeleton, name, stats=stats)
            self.dual_rep_mode = "local"
            self.stats = self.dual_rep.local_motion_rep.stats
        else:
            self.dual_rep = None
            super().__init__(fps, skeleton, name, stats)
        self.root_mode = "local"

    def compute_root_pos_and_rot(
        self, motion: torch.Tensor
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """Integrate root velocities into root positions and rotations."""
        root_motion = self.extract_root(motion)
        device = root_motion.device
        batch, nbframes = root_motion.shape[:2]
        root_rot_vel_angles = root_motion[..., 0]
        root_local_vel = root_motion[..., 1 : 1 + self.nfeats_vel]
        root_y_pos = root_motion[..., 1 + self.nfeats_vel]
        root_rot_vel_angles = root_rot_vel_angles / self.fps
        root_local_vel = root_local_vel / self.fps
        r_rot_ang = torch.zeros_like(root_rot_vel_angles, device=device)
        r_rot_ang[..., 1:] = root_rot_vel_angles[..., :-1]
        r_rot_ang = torch.cumsum(r_rot_ang, dim=-1)
        r_rot_quat = torch.zeros((batch, nbframes, 4), device=device)
        r_rot_quat[..., 0] = torch.cos(r_rot_ang / 2)
        r_rot_quat[..., 2] = torch.sin(r_rot_ang / 2)
        r_pos = torch.zeros((batch, nbframes, 3), device=device)
        r_pos[..., 1:, [0, 2]] = root_local_vel[..., :-1, :]
        removed_heading = self.compute_kwargs.get("removing_heading", True)
        if removed_heading:
            r_pos[..., 1:, :] = quat_apply(r_rot_quat[..., :-1, :], r_pos[..., 1:, :])
        r_pos = torch.cumsum(r_pos, dim=-2)
        r_pos[..., 1] = root_y_pos
        return (r_pos, r_rot_quat)


class DualRootLocalBody(SeparatedRootLocalBody):
    """Pair of the global and local feature layouts, with conversion between them."""

    def __init__(
        self,
        fps: float,
        skeleton: SimpleNamespace,
        name: str,
        *,
        global_class: GlobalRootLocalBody,
        local_class: LocalRootLocalBody,
        stats: Optional[Stats] = None,
    ):
        global_motion_rep = global_class(fps=fps, skeleton=skeleton, name="global")
        local_motion_rep = local_class(fps=fps, skeleton=skeleton, name="local")
        assert (
            global_motion_rep.default_joint_positions_from
            == local_motion_rep.default_joint_positions_from
        )
        self.default_joint_positions_from = (
            global_motion_rep.default_joint_positions_from
        )
        self.root_keys_dim = (
            global_motion_rep.root_keys_dim | local_motion_rep.root_keys_dim
        )
        assert len(self.root_keys_dim) == len(global_motion_rep.root_keys_dim) + len(
            local_motion_rep.root_keys_dim
        )
        assert global_motion_rep.body_keys_dim == local_motion_rep.body_keys_dim
        self.body_keys_dim = global_motion_rep.body_keys_dim
        super().__init__(fps, skeleton, name, stats)
        self.global_motion_rep = global_motion_rep
        self.local_motion_rep = local_motion_rep
        self.indices["global_root"] = np.arange(0, global_motion_rep.motion_root_dim)
        self.indices["local_root"] = np.arange(
            global_motion_rep.motion_root_dim, self.motion_root_dim
        )
        self.indices["global_rep"] = np.concatenate(
            (self.indices["global_root"], self.indices["body"])
        )
        self.indices["local_rep"] = np.concatenate(
            (self.indices["local_root"], self.indices["body"])
        )
        assert global_motion_rep.compute_kwargs == local_motion_rep.compute_kwargs
        self.compute_kwargs = global_motion_rep.compute_kwargs
        self.motion_rep_subset_dim = self.global_motion_rep.motion_rep_dim
        self.dual_rep = None
        if stats is not None and stats.is_loaded():
            global_mean = stats.mean[self.indices["global_rep"]]
            global_std = stats.std[self.indices["global_rep"]]
            self.global_motion_rep.stats = Stats(eps=stats.eps, legacy=stats.legacy)
            self.global_motion_rep.stats.register_from_tensors(global_mean, global_std)
            local_mean = stats.mean[self.indices["local_rep"]]
            local_std = stats.std[self.indices["local_rep"]]
            self.local_motion_rep.stats = Stats(eps=stats.eps, legacy=stats.legacy)
            self.local_motion_rep.stats.register_from_tensors(local_mean, local_std)

    def _one_subset_to_the_other(
        self,
        features: torch.Tensor,
        is_normalized: bool,
        to_normalize: bool,
        mode: str,
        lengths: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        if mode == "local_to_global":
            motion_rep_from = self.local_motion_rep
            motion_rep_to = self.global_motion_rep
        elif mode == "global_to_local":
            motion_rep_from = self.global_motion_rep
            motion_rep_to = self.local_motion_rep
        else:
            raise ValueError("Unrecognized mode.")
        from_rep, to_rep = mode.split("_to_")
        data_type = motion_rep_from.detect_all_body_or_root(features)
        if data_type == "all":
            body_motion = motion_rep_from.extract_body(features)
        else:
            assert data_type == "root"
        root_motion = motion_rep_from.extract_root(features)
        if is_normalized:
            root_motion = self.unnormalize(
                root_motion, index=self.indices[f"{from_rep}_root"]
            )
            if not to_normalize and data_type == "all":
                body_motion = self.unnormalize(body_motion, index=self.indices["body"])
        r_pos, r_rot_quat = motion_rep_from.compute_root_pos_and_rot(root_motion)
        new_root_motion = motion_rep_to.compute_root_rep_from_root_pos_and_rot(
            r_pos, r_rot_quat, lengths
        )
        if to_normalize:
            new_root_motion = self.normalize(
                new_root_motion, index=self.indices[f"{to_rep}_root"]
            )
            if not is_normalized and data_type == "all":
                body_motion = self.normalize(body_motion, index=self.indices["body"])
        if data_type == "root":
            assert new_root_motion.shape[-1] == motion_rep_to.motion_root_dim
            return new_root_motion
        new_features = motion_rep_to.concat_root_body(new_root_motion, body_motion)
        assert new_features.shape[-1] == motion_rep_to.motion_rep_dim
        return new_features

    def global_to_local(
        self,
        features: torch.Tensor,
        is_normalized: bool,
        to_normalize: bool,
        lengths: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        """Convert features from the global root layout to the local one."""
        return self._one_subset_to_the_other(
            features,
            is_normalized=is_normalized,
            to_normalize=to_normalize,
            mode="global_to_local",
            lengths=lengths,
        )

    def local_to_global(
        self,
        features: torch.Tensor,
        is_normalized: bool,
        to_normalize: bool,
        lengths: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        """Convert features from the local root layout to the global one."""
        return self._one_subset_to_the_other(
            features,
            is_normalized=is_normalized,
            to_normalize=to_normalize,
            mode="local_to_global",
            lengths=lengths,
        )


COMPUTE_KWARGS = {
    "local_vel_without_root": False,
    "local_root_vel_with_y": False,
    "compute_heading_method": "hips_pos",
    "removing_heading": False,
}
DEFAULT_JOINT_POSITIONS_FROM = "global_rot_data"


class DualRootGlobalJoints(DualRootLocalBody):
    """Dual layout whose body features hold joint positions and rotations in model space."""

    _name_ = "dual_root_global_joints"
    compute_kwargs = COMPUTE_KWARGS
    default_joint_positions_from = DEFAULT_JOINT_POSITIONS_FROM

    def __init__(self, *args, **kwargs):
        super().__init__(
            *args,
            global_class=GlobalRootGlobalJoints,
            local_class=LocalRootGlobalJoints,
            **kwargs,
        )


def get_body_keys_dim(self, nbjoints: int):
    """Return the size of each body feature.

    The features are joint positions, 6D rotations, velocities, and foot contacts.
    """
    return {
        "ric_data": [(nbjoints - 1) * 3],
        "global_rot_data": [nbjoints * 6],
        "local_vel": [nbjoints * 3],
        "foot_contacts": [4],
    }


class GlobalRootGlobalJoints(GlobalRootLocalBody):
    """Global root layout with joint positions and rotations in model space."""

    dual_class = DualRootGlobalJoints
    get_body_keys_dim = get_body_keys_dim
    compute_kwargs = COMPUTE_KWARGS
    default_joint_positions_from = DEFAULT_JOINT_POSITIONS_FROM

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)


class LocalRootGlobalJoints(LocalRootLocalBody):
    """Local root layout with joint positions and rotations in model space."""

    dual_class = DualRootGlobalJoints
    get_body_keys_dim = get_body_keys_dim
    compute_kwargs = COMPUTE_KWARGS
    default_joint_positions_from = DEFAULT_JOINT_POSITIONS_FROM

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)


def global_root_to_local_onnx_compatible(
    normalized_global_root: torch.Tensor, dual_motion_rep, lengths: torch.Tensor
) -> torch.Tensor:
    """Convert normalized global root features to local ones, with ONNX operations only.

    Velocities at each request's last frame repeat the frame before it.
    """
    global_motion_rep = dual_motion_rep.global_motion_rep
    local_motion_rep = dual_motion_rep.local_motion_rep
    if local_motion_rep.compute_kwargs.get("local_root_vel_with_y", False):
        raise NotImplementedError(
            "Dynamic ONNX root conversion does not support local Y velocity."
        )
    global_root = dual_motion_rep.unnormalize(
        normalized_global_root, index=dual_motion_rep.indices["global_root"]
    )
    root_positions, root_rotations = global_motion_rep.compute_root_pos_and_rot(
        global_root
    )
    root_angles = torch.atan2(root_rotations[..., 2], root_rotations[..., 0]) * 2
    angular_velocity = diff_angles(root_angles, local_motion_rep.fps)
    angular_velocity = torch.cat(
        [angular_velocity, torch.zeros_like(root_angles[:, :1])], dim=1
    )
    root_velocity = local_motion_rep.fps * (
        root_positions[:, 1:] - root_positions[:, :-1]
    )
    root_velocity = torch.cat(
        [root_velocity, torch.zeros_like(root_positions[:, :1])], dim=1
    )
    last_indices = lengths.reshape([-1, 1]).long() - 1
    angular_velocity = angular_velocity.scatter(
        1, last_indices, angular_velocity.gather(1, last_indices - 1)
    )
    root_last_indices = last_indices[..., None].expand_as(root_velocity[:, :1])
    root_velocity = root_velocity.scatter(
        1, root_last_indices, root_velocity.gather(1, root_last_indices - 1)
    )
    if local_motion_rep.compute_kwargs.get("removing_heading", True):
        root_velocity = quat_apply(quat_conjugate(root_rotations), root_velocity)
    local_root_velocity = root_velocity[..., [0, 2]]
    local_root = torch.cat(
        [angular_velocity[..., None], local_root_velocity, root_positions[..., [1]]],
        dim=-1,
    )
    return dual_motion_rep.normalize(
        local_root, index=dual_motion_rep.indices["local_root"]
    )