Official implementation of MotionBricks: Scalable Real-Time Motions with Modular Latent Generative Model and Smart Primitives (SIGGRAPH 2026)
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"Combine the SOMA predictors and decoder for batched inference."
from __future__ import annotations
import logging
from typing import Dict
import numpy as np
import torch as t
from .features import numeric_mask_union
from ..features import extract_feature_from_motion_rep
from ..representation import global_root_to_local_onnx_compatible
logger = logging.getLogger(__name__)
def target_frame_indices(
pred_num_tokens: t.Tensor, num_frames_per_token: int
) -> t.Tensor:
"""Return the frame indices of each request's last motion token."""
return (
pred_num_tokens.long().reshape([-1, 1]) - 1
) * num_frames_per_token + t.arange(
num_frames_per_token, device=pred_num_tokens.device
).reshape([1, -1])
class mmm_inference_class(t.nn.Module):
"""Run the SOMA root predictor, pose predictor, and motion decoder for inference.
predict() runs them in order on one batch.
"""
EXTERNAL_POSE_FEATURE_MODE = "joint_positions_and_rotations"
EXTERNAL_ROOT_FEATURE_MODE = "root"
INTERNAL_POSE_FEATURE_MODE = "joint_positions_and_rotations_and_hip_height"
EPS = 1e-05
def __init__(self, models: list, args: Dict, device: str = "cuda"):
super(mmm_inference_class, self).__init__()
self._mmm_pose_model: t.nn.Module = models["pose"].eval().to(device)
self._mmm_root_model: t.nn.Module = models["root"].eval().to(device)
self._vqvae_pose_model: t.nn.Module = (
self._mmm_pose_model.supporting_nets["pose_net"].eval().to(device)
)
self.global_motion_rep = self._mmm_pose_model.global_motion_rep
self.local_motion_rep = self._mmm_pose_model.local_motion_rep
self.motion_rep = self._mmm_pose_model.motion_rep
self._args = args
self._device = device
self._IS_ROOT_MODEL_TOKENIZED = self._mmm_root_model.mmm_net.IS_MODEL_TOKENIZED
assert (
self._mmm_pose_model.mmm_net.initted
and self._mmm_pose_model.vqvae_model_loaded
and self._mmm_root_model.vqvae_model_loaded
), "The model should be initialized before inference."
assert not self._IS_ROOT_MODEL_TOKENIZED, (
"The root model is not tokenized."
)
def predict(
self,
global_root_values: t.Tensor,
has_global_root_values: t.Tensor,
local_root_values: t.Tensor,
has_local_root_values: t.Tensor,
local_poses: t.Tensor,
has_local_poses: t.Tensor,
num_tokens: t.Tensor,
text_embeddings: t.Tensor = None,
has_text_embeddings: t.Tensor = None,
allowed_pred_num_tokens: t.Tensor = None,
config: dict = {},
info: dict = {},
):
"""Predict the motion features and token count for a batch of conditions.
It predicts the root trajectory, then the pose tokens, then decodes the motion.
"""
batch_size, device, dtype = (
has_global_root_values.shape[0],
has_global_root_values.device,
local_poses.dtype,
)
num_frames_per_token = self._mmm_pose_model.mmm_net.get_num_frames_per_token()
if not (
t.all(has_global_root_values[:, :num_frames_per_token])
and t.all(has_local_root_values[:, :num_frames_per_token])
and t.all(has_local_poses[:, :num_frames_per_token])
):
if not getattr(self, "_warned_missing_frames", False):
logger.warning("Results are best when all of the first four frames are provided.")
self._warned_missing_frames = True
if type(num_tokens) == int:
num_tokens = t.full([batch_size, 1], num_tokens, dtype=t.int).to(device)
elif num_tokens is None:
num_tokens = self._mmm_root_model.mmm_net.MASKED_NUM_TOKENS * t.ones(
[batch_size, 1], dtype=t.int
).to(device)
batch = {}
(
batch["reference_start_root_global_offsets"],
batch["reference_start_root_global_heading"],
recentered_global_root_values,
) = self._extract_initial_root_info(global_root_values)
batch["global_root_values"] = self.global_motion_rep.normalize(
recentered_global_root_values
)
batch["local_root_values"] = self.local_motion_rep.normalize(local_root_values)
local_pose_feat_idx = extract_feature_from_motion_rep(
t.zeros([1, 1, len(self.local_motion_rep.indices["all"])]),
self.local_motion_rep,
self.INTERNAL_POSE_FEATURE_MODE,
fetch_feat_idx=True,
)
global_height_values = recentered_global_root_values[
:, :, self.global_motion_rep.indices["global_root_pos"][[1]]
]
mean = self.local_motion_rep.stats.mean[None, None, local_pose_feat_idx].to(
device=device, dtype=dtype
)
std = self.local_motion_rep.stats.std[None, None, local_pose_feat_idx].to(
device=device, dtype=dtype
)
batch["local_poses"] = (
t.concat([global_height_values, local_poses], dim=-1) - mean
) / t.sqrt(std**2 + self.EPS)
batch["has_global_root_values"] = has_global_root_values
batch["has_local_root_values"] = has_local_root_values
batch["has_local_poses"] = has_local_poses
batch["text_embeddings"] = text_embeddings
batch["has_text_embeddings"] = has_text_embeddings
batch["num_tokens"] = num_tokens
batch["allowed_pred_num_tokens"] = allowed_pred_num_tokens
(
batch["pred_num_tokens"],
batch["pred_global_root_values"],
batch["pred_local_root_values"],
) = self._predict_root_trajectories(batch, config)
(
batch["pred_pose_tokens"],
batch["pred_pose_cond"],
batch["pred_has_pose_cond"],
) = self._predict_pose_tokens(batch, config, info)
batch["pred_local_poses"], batch["pred_global_poses"] = (
self._decode_motions_from_predicted_root_and_pose_tokens(
batch, config, info
)
)
batch["pred_global_poses"] = self._reapply_initial_root_info(batch)
return (batch["pred_global_poses"], batch["pred_num_tokens"])
def _sample_tokens_with_highest_prob(
self,
pose_tokens: t.Tensor,
pose_tokens_prob: t.Tensor,
pred_num_tokens: t.Tensor,
step: int,
num_pose_inference_steps: int,
):
batch_size, device = (pose_tokens.shape[0], pose_tokens.device)
rand_mask_prob = np.cos(float(step) / num_pose_inference_steps * np.pi * 0.5)
num_pose_heads = self._mmm_pose_model.mmm_net.get_num_heads()[0]
num_tokens = (
self._args["max_tokens"] * self._mmm_pose_model.mmm_net.get_num_heads()[0]
)
num_pose_token_masked = t.clip(
(
rand_mask_prob
* pred_num_tokens
* self._mmm_pose_model.mmm_net.get_num_heads()[0]
).int(),
min=1,
)
pose_tokens_prob = t.where(
t.arange(self._args["max_tokens"])
.to(device)
.view([1, -1, 1])
.expand([batch_size, -1, -1])
< pred_num_tokens.view([batch_size, 1, 1]).repeat([1, 1, num_pose_heads]),
pose_tokens_prob,
t.full_like(pose_tokens_prob, t.inf),
)
indices = pose_tokens_prob.view([batch_size, -1]).sort(descending=False)[1]
tokens_to_be_masked = t.arange(num_tokens).to(device)[None, :].repeat(
[batch_size, 1]
) < num_pose_token_masked.view([batch_size, 1])
indices = indices * tokens_to_be_masked + indices[:, :1] * ~tokens_to_be_masked
pose_tokens = pose_tokens.view([batch_size, -1]).clone()
pose_tokens.scatter_(
dim=-1, index=indices, value=self._mmm_pose_model.mmm_net.POSE_MASK_ID
)
return pose_tokens.view([batch_size, self._args["max_tokens"], -1])
@property
def device(self):
"""Device the networks run on."""
return self._device
def _extract_initial_root_info(self, global_root_values: t.Tensor):
reference_start_root_global_offsets = global_root_values[
:, :1, self.global_motion_rep.indices["global_root_pos_2d"]
].clone()
_, reference_start_root_global_heading = (
self.global_motion_rep.compute_root_pos_and_rot(
global_root_values[:, :1, :], return_angle=True, return_quat=False
)
)
recentered_global_root_values = global_root_values.clone()
recentered_global_root_values[
:, :, self.global_motion_rep.indices["global_root_pos_2d"]
] -= reference_start_root_global_offsets
return (
reference_start_root_global_offsets,
reference_start_root_global_heading,
recentered_global_root_values,
)
def _reapply_initial_root_info(self, batch: dict):
_, root_rot_angle = self.global_motion_rep.compute_root_pos_and_rot(
batch["pred_global_poses"], return_angle=True, return_quat=False
)
corrective_angle = batch["reference_start_root_global_heading"].reshape(
root_rot_angle[..., 0].shape
)
new_angles = root_rot_angle + corrective_angle[..., None]
new_heading = t.stack([t.cos(new_angles), t.sin(new_angles)], dim=-1)
batch["pred_global_poses"][
:, :, self.global_motion_rep.indices["global_root_pos_2d"]
] += batch["reference_start_root_global_offsets"]
batch["pred_global_poses"][
:, :, self.global_motion_rep.indices["global_root_heading"]
] = new_heading
return batch["pred_global_poses"]
def _predict_root_trajectories(self, batch: dict = {}, config: dict = {}):
assert (
self._mmm_root_model.args["local_root_feature"]
== self.EXTERNAL_ROOT_FEATURE_MODE
), (
self._mmm_root_model.args["global_root_feature"]
== self.EXTERNAL_ROOT_FEATURE_MODE
)
assert (
self._mmm_root_model.args["local_pose_feature"]
== self.INTERNAL_POSE_FEATURE_MODE
), (
f"All submodules should only use {self.INTERNAL_POSE_FEATURE_MODE} as the local pose feature."
)
assert self.EXTERNAL_ROOT_FEATURE_MODE == "root", (
"The root model supports only full root features."
)
num_frames_per_token = self._mmm_pose_model.mmm_net.get_num_frames_per_token()
assert (
num_frames_per_token
== self._mmm_root_model.mmm_net.get_num_frames_per_token()
)
root_mmm_outputs = self._mmm_root_model.mmm_net(
batch["global_root_values"],
batch["has_global_root_values"],
batch["local_root_values"],
batch["has_local_root_values"],
batch["local_poses"],
batch["has_local_poses"],
batch["num_tokens"],
text_embeddings=batch["text_embeddings"],
has_text_embeddings=batch["has_text_embeddings"],
allowed_pred_num_tokens=batch["allowed_pred_num_tokens"],
config=config,
)
pred_num_tokens = root_mmm_outputs["pred_num_tokens"]
pred_global_root_values = root_mmm_outputs["pred_global_root_values"]
if config.get("debug_ground_truth_root_trajectories", None) is not None:
pred_num_tokens[:] = (
config["debug_ground_truth_root_trajectories"].shape[1]
// num_frames_per_token
)
pred_global_root_values[
:, : config["debug_ground_truth_root_trajectories"].shape[1]
] = config["debug_ground_truth_root_trajectories"]
num_pred_frames = pred_num_tokens * num_frames_per_token
if config.get("use_onnx_dynamic_batch_root_conversion", False):
pred_local_root_values = global_root_to_local_onnx_compatible(
pred_global_root_values, self.motion_rep.dual_rep, num_pred_frames
)
else:
pred_local_root_values = self.motion_rep.dual_rep.global_to_local(
pred_global_root_values,
is_normalized=True,
to_normalize=True,
lengths=num_pred_frames,
)
local_root_feat_dim = len(self.local_motion_rep.indices["root"])
final_root_indices = (
pred_num_tokens.long()
.view([-1, 1])
.repeat([1, local_root_feat_dim])[:, None, :]
* num_frames_per_token
- 1
)
estimated_final_local_root_motion = t.gather(
pred_local_root_values, 1, final_root_indices
)
final_local_root_motion = (
batch["has_local_root_values"][:, -1:, None]
.expand([-1, -1, local_root_feat_dim])
.float()
* batch["local_root_values"][:, -1:]
+ (
1
- batch["has_local_root_values"][:, -1:, None]
.expand([-1, -1, local_root_feat_dim])
.float()
)
* estimated_final_local_root_motion
)
pred_local_root_values = pred_local_root_values.scatter(
1, final_root_indices, final_local_root_motion
)
return (pred_num_tokens, pred_global_root_values, pred_local_root_values)
def _predict_pose_tokens(
self, batch: dict = {}, config: dict = {}, info: dict = {}
):
assert (
self._mmm_pose_model.args["local_pose_feature"]
== self.INTERNAL_POSE_FEATURE_MODE
)
assert (
self.INTERNAL_POSE_FEATURE_MODE
== "joint_positions_and_rotations_and_hip_height"
)
assert (
self._mmm_pose_model.args["cond_root_feature_is_from_motion_rep"] == "local"
and self._mmm_pose_model.args["cond_root_feature"]
== "root_without_hip_height_without_heading"
or (
self._mmm_pose_model.args["cond_root_feature_is_from_motion_rep"]
== "global"
and self._mmm_pose_model.args["cond_root_feature"]
== "root_without_hip_height"
)
), "Only these two root condition feature combinations are supported."
num_pose_heads, num_frames_per_token = (
self._mmm_pose_model.mmm_net.get_num_heads()[0],
self._mmm_pose_model.mmm_net.get_num_frames_per_token(),
)
batch_size, device = (
batch["local_poses"].shape[0],
batch["local_poses"].device,
)
num_pose_inference_steps = config.get("num_inference_step", 1)
pose_tokens = t.full(
[batch_size, self._args["max_tokens"], num_pose_heads],
self._mmm_pose_model.mmm_net.POSE_MASK_ID,
).to(device)
chosen_pose_tokens_prob = t.full(
[batch_size, self._args["max_tokens"], num_pose_heads], 1.0 / num_pose_heads
).to(device)
pose_cond = t.concat(
[
batch["local_poses"][:, :num_frames_per_token],
t.zeros(
[
batch_size,
(self._args["max_tokens"] - 1) * num_frames_per_token,
batch["local_poses"].shape[-1],
]
).to(device),
],
dim=1,
)
has_pose_cond = t.concat(
[
batch["has_local_poses"][:, :num_frames_per_token],
t.zeros(
[batch_size, (self._args["max_tokens"] - 1) * num_frames_per_token],
dtype=batch["has_local_poses"].dtype,
device=device,
),
],
dim=1,
)
for i in range(num_frames_per_token):
onehot_idx = t.nn.functional.one_hot(
batch["pred_num_tokens"].view([-1]).long() * num_frames_per_token
- num_frames_per_token
+ i,
num_classes=self._args["max_tokens"] * num_frames_per_token,
)
pose_cond = pose_cond + onehot_idx.float().view(
[batch_size, -1, 1]
) * batch["local_poses"][:, -num_frames_per_token + i].view(
[batch_size, 1, -1]
)
target_pose_mask = onehot_idx.to(has_pose_cond.dtype).view(
[batch_size, -1]
) * batch["has_local_poses"][:, -num_frames_per_token + i].view(
[batch_size, 1]
)
has_pose_cond = numeric_mask_union(has_pose_cond, target_pose_mask)
has_pose_cond = has_pose_cond.bool()
for step in range(num_pose_inference_steps):
pose_tokens = self._sample_tokens_with_highest_prob(
pose_tokens,
chosen_pose_tokens_prob,
batch["pred_num_tokens"],
step,
num_pose_inference_steps,
)
assert self._args["cond_root_feature_is_from_motion_rep"] in [
"local",
"global",
]
if self._args["cond_root_feature_is_from_motion_rep"] == "local":
pose_mmm_root_cond = extract_feature_from_motion_rep(
batch["pred_local_root_values"],
self.local_motion_rep,
self._mmm_pose_model.args["cond_root_feature"],
)
else:
assert self._args["cond_root_feature_is_from_motion_rep"] == "global"
pose_mmm_root_cond = extract_feature_from_motion_rep(
batch["pred_global_root_values"],
self.global_motion_rep,
self._mmm_pose_model.args["cond_root_feature"],
)
pose_mmm_output = self._mmm_pose_model.mmm_net(
pose_tokens,
pose_mmm_root_cond,
pose_cond,
has_pose_cond,
batch["pred_num_tokens"],
batch["text_embeddings"],
batch["has_text_embeddings"],
)
if config.get("pose_token_sampling_use_argmax", False):
pose_tokens = pose_mmm_output["pose_logits"].argmax(dim=-1)
else:
raise ValueError("The release exporter uses argmax token selection.")
pose_tokens_prob = pose_mmm_output["pose_logits"].softmax(dim=-1)
chosen_pose_tokens_prob = pose_tokens_prob.gather(
dim=-1, index=pose_tokens.unsqueeze(-1)
).squeeze(-1)
if config.get("debug_ground_truth_pose_tokens", None) is not None:
pose_tokens[:, : config["debug_ground_truth_pose_tokens"].shape[1]] = (
config["debug_ground_truth_pose_tokens"][:, :]
)
if config.get("perturb_token_percentage", 0.0) > 0.0:
perturb_token_percentage = config.get("perturb_token_percentage", 0.0)
random_tokens = t.randint_like(
pose_tokens,
high=self._mmm_pose_model.mmm_net.get_num_codes(False)[0],
low=0,
)
pose_tokens = t.where(
t.rand_like(pose_tokens.float()) < perturb_token_percentage,
random_tokens,
pose_tokens,
)
return (pose_tokens, pose_cond, has_pose_cond)
def _decode_motions_from_predicted_root_and_pose_tokens(
self, batch: dict = {}, config: dict = {}, info: dict = {}
):
if self._mmm_pose_model.args["pose_vqvae_motion_rep"] == "local":
assert self._vqvae_pose_model.motion_rep.name == "local"
assert (
self._vqvae_pose_model.decoder_external_cond_feature_mode
== "root_without_hip_height_without_heading"
)
else:
assert self._vqvae_pose_model.motion_rep.name == "global"
assert (
self._vqvae_pose_model.decoder_external_cond_feature_mode
== "root_without_hip_height_without_heading_with_mask"
)
assert (
self._vqvae_pose_model.decoder_target_cond_feature_mode
== self.INTERNAL_POSE_FEATURE_MODE
)
pose_external_root_cond = extract_feature_from_motion_rep(
batch["pred_local_root_values"]
if self._vqvae_pose_model.motion_rep.name == "local"
else batch["pred_global_root_values"],
self._vqvae_pose_model.motion_rep,
self._vqvae_pose_model.decoder_external_cond_feature_mode,
)
if self._vqvae_pose_model.motion_rep.name == "global":
raise NotImplementedError("The global root feature is not supported.")
batch_size, device = (
batch["local_poses"].shape[0],
batch["local_poses"].device,
)
pose_token_mask = t.arange(self._args["max_tokens"]).to(device)[None, :].repeat(
[batch_size, 1]
) < batch["pred_num_tokens"].view([batch_size, 1])
if type(config.get("use_target_cond_in_decoder", True)) == t.Tensor:
global_target_cond = (
config["use_target_cond_in_decoder"]
.bool()
.expand_as(batch["pred_has_pose_cond"])
)
has_target_cond = t.logical_and(
batch["pred_has_pose_cond"], global_target_cond
)
else:
assert type(config.get("use_target_cond_in_decoder", True)) == bool
has_target_cond = (
batch["pred_has_pose_cond"]
if config.get("use_target_cond_in_decoder", True)
else t.zeros_like(batch["pred_has_pose_cond"]).bool()
)
if not config.get("use_constraints_at_decoder", True):
has_target_cond = t.zeros_like(has_target_cond).bool()
if config.get("skip_ending_target_cond", False):
num_frames_per_token = (
self._mmm_pose_model.mmm_net.get_num_frames_per_token()
)
has_target_cond[:, num_frames_per_token:] = False
if abs(config.get("root_interpolation_ratio", 1.0) - 1.0) > 1e-06:
pose_external_root_cond = self.local_motion_rep.unnormalize(
pose_external_root_cond, self.local_motion_rep.indices["local_root_vel"]
)
pose_external_root_cond *= config.get("root_interpolation_ratio", 1.0)
pose_external_root_cond = self.local_motion_rep.normalize(
pose_external_root_cond, self.local_motion_rep.indices["local_root_vel"]
)
pred_poses = self._vqvae_pose_model.forward_decoder(
batch["pred_pose_tokens"],
target_cond=batch["pred_pose_cond"],
has_target_cond=has_target_cond,
external_cond=pose_external_root_cond,
use_overall_indices=False,
token_mask=pose_token_mask,
)["recon_state"]
num_pred_frames = (
batch["pred_num_tokens"]
* self._mmm_pose_model.mmm_net.get_num_frames_per_token()
)
if self._vqvae_pose_model.motion_rep.name == "local":
pred_global_poses = self.motion_rep.dual_rep.local_to_global(
pred_poses,
is_normalized=True,
to_normalize=False,
lengths=num_pred_frames,
)
pred_local_poses = self.local_motion_rep.unnormalize(pred_poses)
else:
pred_local_poses = self.motion_rep.dual_rep.global_to_local(
pred_poses,
is_normalized=True,
to_normalize=False,
lengths=num_pred_frames,
)
pred_global_poses = self.global_motion_rep.unnormalize(pred_poses)
if config.get("final_root_pred_mode", "from_pose_module") == "from_pose_module":
pass
elif config["final_root_pred_mode"] == "from_root_module":
local_root_index, global_root_index = (
self.local_motion_rep.indices["root"],
self.global_motion_rep.indices["root"],
)
pred_global_poses[:, :, global_root_index] = (
self.global_motion_rep.unnormalize(batch["pred_global_root_values"])
)
pred_local_poses[:, :, local_root_index] = (
self.local_motion_rep.unnormalize(batch["pred_local_root_values"])
)
else:
assert config["final_root_pred_mode"] == "from_height_constraint_violation"
num_frames_per_token = (
self._mmm_pose_model.mmm_net.get_num_frames_per_token()
)
target_frame_inds = target_frame_indices(
batch["pred_num_tokens"], num_frames_per_token
)
root_height_index = self.global_motion_rep.indices["global_root_pos"][1]
pred_global_height_from_root_module = t.gather(
self.global_motion_rep.unnormalize(batch["pred_global_root_values"])[
:, :, root_height_index
],
1,
target_frame_inds,
)
pred_global_height_from_pose_module = t.gather(
pred_global_poses[:, :, root_height_index], 1, target_frame_inds
)
target_global_height = self.global_motion_rep.unnormalize(
batch["global_root_values"]
)[:, -num_frames_per_token:, root_height_index]
avg_height_error_from_pose = (
(pred_global_height_from_pose_module - target_global_height).abs()
* batch["has_global_root_values"][:, -num_frames_per_token:].float()
).sum(dim=1) / batch["has_global_root_values"][
:, -num_frames_per_token:
].float().sum(dim=1)
avg_height_error_from_root = (
(pred_global_height_from_root_module - target_global_height).abs()
* batch["has_global_root_values"][:, -num_frames_per_token:].float()
).sum(dim=1) / batch["has_global_root_values"][
:, -num_frames_per_token:
].float().sum(dim=1)
use_root_module = t.logical_and(
avg_height_error_from_root < avg_height_error_from_pose,
avg_height_error_from_pose > 0.05,
)[:, None, None].float()
local_root_index, global_root_index = (
self.local_motion_rep.indices["root"],
self.global_motion_rep.indices["root"],
)
pred_global_poses[:, :, global_root_index] = (
use_root_module
* self.global_motion_rep.unnormalize(batch["pred_global_root_values"])
+ (1 - use_root_module) * pred_global_poses[:, :, global_root_index]
)
pred_local_poses[:, :, local_root_index] = (
use_root_module
* self.local_motion_rep.unnormalize(batch["pred_local_root_values"])
+ (1 - use_root_module) * pred_local_poses[:, :, local_root_index]
)
return (pred_local_poses, pred_global_poses)