Official implementation of MotionBricks: Scalable Real-Time Motions with Modular Latent Generative Model and Smart Primitives (SIGGRAPH 2026)
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"Combine the G1 predictors and decoder into a batched inference model."
from __future__ import annotations
import logging
from typing import Dict
import numpy as np
import torch as t
from ..features import extract_feature_from_motion_rep
from ..representation import global_root_to_local_onnx_compatible
logger = logging.getLogger(__name__)
class motion_inference(t.nn.Module):
"""Run the G1 root predictor, pose predictor, and motion decoder for inference.
predict() runs them in order on one batch.
"""
BATCH_SIZE = 1
EXTERNAL_POSE_FEATURE_MODE = "joint_positions_and_rotations"
EXTERNAL_ROOT_FEATURE_MODE = "root"
INTERNAL_POSE_FEATURE_MODE = "joint_positions_and_rotations_and_hip_height"
EPS = 1e-05
def __init__(self, models: list, args: Dict, device: str = "cuda"):
super(motion_inference, self).__init__()
self._pose_model: t.nn.Module = models["pose"].eval().to(device)
self._root_model: t.nn.Module = models["root"].eval().to(device)
self._vqvae_pose_model: t.nn.Module = (
self._pose_model.supporting_nets["pose_net"].eval().to(device)
)
self.global_motion_rep = self._pose_model.global_motion_rep
self.local_motion_rep = self._pose_model.local_motion_rep
self.motion_rep = self._pose_model.motion_rep
self._args = args
self._device = device
self._IS_ROOT_MODEL_TOKENIZED = self._root_model.backbone_net.IS_MODEL_TOKENIZED
assert (
self._pose_model.backbone_net.initted
and self._pose_model.vqvae_model_loaded
and self._root_model.vqvae_model_loaded
), "The model should be initialized before inference."
assert not self._IS_ROOT_MODEL_TOKENIZED, "The root model is not tokenized."
def predict(
self,
global_root_values: t.Tensor,
has_global_root_values: t.Tensor,
local_root_values: t.Tensor,
has_local_root_values: t.Tensor,
local_poses: t.Tensor,
has_local_poses: t.Tensor,
num_tokens: t.Tensor,
text_embeddings: t.Tensor = None,
has_text_embeddings: t.Tensor = None,
allowed_pred_num_tokens: t.Tensor = None,
config: dict = {},
info: dict = {},
):
"""Predict the motion features and token count for a batch of conditions.
It predicts the root trajectory, then the pose tokens, then decodes the motion.
"""
batch_size, device, dtype = (
has_global_root_values.shape[0],
has_global_root_values.device,
local_poses.dtype,
)
num_frames_per_token = self._pose_model.backbone_net.get_num_frames_per_token()
if not (
t.all(has_global_root_values[:, :num_frames_per_token])
and t.all(has_local_root_values[:, :num_frames_per_token])
and t.all(has_local_poses[:, :num_frames_per_token])
):
if not getattr(self, "_warned_missing_frames", False):
logger.warning("Results are best when all of the first four frames are provided.")
self._warned_missing_frames = True
if type(num_tokens) == int:
num_tokens = t.full([batch_size, 1], num_tokens, dtype=t.int).to(device)
elif num_tokens is None:
num_tokens = self._root_model.backbone_net.MASKED_NUM_TOKENS * t.ones(
[batch_size, 1], dtype=t.int
).to(device)
batch = {}
(
batch["reference_start_root_global_offsets"],
batch["reference_start_root_global_heading"],
recentered_global_root_values,
) = self._extract_initial_root_info(global_root_values)
batch["global_root_values"] = self.global_motion_rep.normalize(
recentered_global_root_values
)
batch["local_root_values"] = self.local_motion_rep.normalize(local_root_values)
local_pose_feat_idx = extract_feature_from_motion_rep(
t.zeros([1, 1, len(self.local_motion_rep.indices["all"])]),
self.local_motion_rep,
self.INTERNAL_POSE_FEATURE_MODE,
fetch_feat_idx=True,
)
global_height_values = recentered_global_root_values[
:, :, self.global_motion_rep.indices["global_root_pos"][[1]]
]
mean = self.local_motion_rep.stats.mean[None, None, local_pose_feat_idx].to(
device=device, dtype=dtype
)
std = self.local_motion_rep.stats.std[None, None, local_pose_feat_idx].to(
device=device, dtype=dtype
)
batch["local_poses"] = (
t.concat([global_height_values, local_poses], dim=-1) - mean
) / t.sqrt(std**2 + self.EPS)
batch["has_global_root_values"] = has_global_root_values
batch["has_local_root_values"] = has_local_root_values
batch["has_local_poses"] = has_local_poses
batch["text_embeddings"] = text_embeddings
batch["has_text_embeddings"] = has_text_embeddings
batch["num_tokens"] = num_tokens
batch["allowed_pred_num_tokens"] = allowed_pred_num_tokens
(
batch["pred_num_tokens"],
batch["pred_global_root_values"],
batch["pred_local_root_values"],
) = self._predict_root_trajectories(batch, config)
(
batch["pred_pose_tokens"],
batch["pred_pose_cond"],
batch["pred_has_pose_cond"],
) = self._predict_pose_tokens(batch, config, info)
batch["pred_local_poses"], batch["pred_global_poses"] = (
self._decode_motions_from_predicted_root_and_pose_tokens(
batch, config, info
)
)
batch["pred_global_poses"] = self._reapply_initial_root_info(batch)
return (batch["pred_global_poses"], batch["pred_num_tokens"])
def _sample_tokens_with_highest_prob(
self,
pose_tokens: t.Tensor,
pose_tokens_prob: t.Tensor,
pred_num_tokens: t.Tensor,
step: int,
num_pose_inference_steps: int,
):
batch_size, device = (pose_tokens.shape[0], pose_tokens.device)
rand_mask_prob = np.cos(float(step) / num_pose_inference_steps * np.pi * 0.5)
num_pose_heads = self._pose_model.backbone_net.get_num_heads()[0]
num_tokens = (
self._args["max_tokens"] * self._pose_model.backbone_net.get_num_heads()[0]
)
num_pose_token_masked = t.clip(
(
rand_mask_prob
* pred_num_tokens
* self._pose_model.backbone_net.get_num_heads()[0]
).int(),
min=1,
)
pose_tokens_prob = t.where(
t.arange(self._args["max_tokens"]).to(device).view([1, -1, 1])
< pred_num_tokens.view([batch_size, 1, 1]).repeat([1, 1, num_pose_heads]),
pose_tokens_prob,
t.full_like(pose_tokens_prob, t.inf),
)
indices = pose_tokens_prob.view([batch_size, -1]).sort(descending=False)[1]
tokens_to_be_masked = t.arange(num_tokens).to(device)[None, :].expand(
batch_size, -1
) < num_pose_token_masked.reshape(-1, 1)
indices = indices * tokens_to_be_masked + indices[:, :1] * ~tokens_to_be_masked
pose_tokens = pose_tokens.view([batch_size, -1]).clone()
pose_tokens.scatter_(
dim=-1, index=indices, value=self._pose_model.backbone_net.POSE_MASK_ID
)
return pose_tokens.view([batch_size, self._args["max_tokens"], -1])
@property
def device(self):
"""Device the networks run on."""
return self._device
def _extract_initial_root_info(self, global_root_values: t.Tensor):
reference_start_root_global_offsets = global_root_values[
:, :1, self.global_motion_rep.indices["global_root_pos_2d"]
].clone()
_, reference_start_root_global_heading = (
self.global_motion_rep.compute_root_pos_and_rot(
global_root_values[:, :1, :], return_angle=True, return_quat=False
)
)
recentered_global_root_values = global_root_values.clone()
recentered_global_root_values[
:, :, self.global_motion_rep.indices["global_root_pos_2d"]
] -= reference_start_root_global_offsets
return (
reference_start_root_global_offsets,
reference_start_root_global_heading,
recentered_global_root_values,
)
def _reapply_initial_root_info(self, batch: dict):
_, root_rot_angle = self.global_motion_rep.compute_root_pos_and_rot(
batch["pred_global_poses"], return_angle=True, return_quat=False
)
corrective_angle = batch["reference_start_root_global_heading"].reshape(
root_rot_angle[..., 0].shape
)
new_angles = root_rot_angle + corrective_angle[..., None]
new_heading = t.stack([t.cos(new_angles), t.sin(new_angles)], dim=-1)
batch["pred_global_poses"][
:, :, self.global_motion_rep.indices["global_root_pos_2d"]
] += batch["reference_start_root_global_offsets"]
batch["pred_global_poses"][
:, :, self.global_motion_rep.indices["global_root_heading"]
] = new_heading
return batch["pred_global_poses"]
def _predict_root_trajectories(self, batch: dict = {}, config: dict = {}):
assert (
self._root_model.args["local_root_feature"]
== self.EXTERNAL_ROOT_FEATURE_MODE
), (
self._root_model.args["global_root_feature"]
== self.EXTERNAL_ROOT_FEATURE_MODE
)
assert (
self._root_model.args["local_pose_feature"]
== self.INTERNAL_POSE_FEATURE_MODE
), (
f"All submodules should only use {self.INTERNAL_POSE_FEATURE_MODE} as the local pose feature."
)
assert self.EXTERNAL_ROOT_FEATURE_MODE == "root", (
"The root model supports only full root features."
)
num_frames_per_token = self._pose_model.backbone_net.get_num_frames_per_token()
assert (
num_frames_per_token
== self._root_model.backbone_net.get_num_frames_per_token()
)
root_model_outputs = self._root_model.backbone_net(
batch["global_root_values"],
batch["has_global_root_values"],
batch["local_root_values"],
batch["has_local_root_values"],
batch["local_poses"],
batch["has_local_poses"],
batch["num_tokens"],
text_embeddings=batch["text_embeddings"],
has_text_embeddings=batch["has_text_embeddings"],
allowed_pred_num_tokens=batch["allowed_pred_num_tokens"],
config=config,
)
pred_num_tokens = root_model_outputs["pred_num_tokens"]
pred_global_root_values = root_model_outputs["pred_global_root_values"]
if config.get("debug_ground_truth_root_trajectories", None) is not None:
pred_num_tokens[:] = (
config["debug_ground_truth_root_trajectories"].shape[1]
// num_frames_per_token
)
pred_global_root_values[
:, : config["debug_ground_truth_root_trajectories"].shape[1]
] = config["debug_ground_truth_root_trajectories"]
pred_local_root_values = global_root_to_local_onnx_compatible(
pred_global_root_values,
self.motion_rep.dual_rep,
pred_num_tokens * num_frames_per_token,
)
estimated_final_local_root_motion = t.gather(
pred_local_root_values,
1,
pred_num_tokens.long().reshape(-1, 1).repeat([1, 4])[:, None, :]
* num_frames_per_token
- 1,
)
local_root_feat_dim = len(self.local_motion_rep.indices["root"])
final_local_root_motion = (
batch["has_local_root_values"][:, -1:, None]
.expand([-1, -1, local_root_feat_dim])
.float()
* batch["local_root_values"][:, -1:]
+ (
1
- batch["has_local_root_values"][:, -1:, None]
.expand([-1, -1, local_root_feat_dim])
.float()
)
* estimated_final_local_root_motion
)
pred_local_root_values = pred_local_root_values.scatter(
1,
pred_num_tokens.long().reshape(-1, 1).repeat([1, 4])[:, None, :]
* num_frames_per_token
- 1,
final_local_root_motion,
)
return (pred_num_tokens, pred_global_root_values, pred_local_root_values)
def _predict_pose_tokens(
self, batch: dict = {}, config: dict = {}, info: dict = {}
):
assert (
self._pose_model.args["local_pose_feature"]
== self.INTERNAL_POSE_FEATURE_MODE
)
assert (
self.INTERNAL_POSE_FEATURE_MODE
== "joint_positions_and_rotations_and_hip_height"
)
assert (
self._pose_model.args["cond_root_feature_is_from_motion_rep"] == "local"
and self._pose_model.args["cond_root_feature"]
== "root_without_hip_height_without_heading"
or (
self._pose_model.args["cond_root_feature_is_from_motion_rep"]
== "global"
and self._pose_model.args["cond_root_feature"]
== "root_without_hip_height"
)
), "Only these two root condition feature combinations are supported."
num_pose_heads, num_frames_per_token = (
self._pose_model.backbone_net.get_num_heads()[0],
self._pose_model.backbone_net.get_num_frames_per_token(),
)
batch_size, device = (
batch["local_poses"].shape[0],
batch["local_poses"].device,
)
num_pose_inference_steps = config.get("num_inference_step", 1)
pose_tokens = t.full(
[batch_size, self._args["max_tokens"], num_pose_heads],
self._pose_model.backbone_net.POSE_MASK_ID,
).to(device)
chosen_pose_tokens_prob = t.full(
[batch_size, self._args["max_tokens"], num_pose_heads], 1.0 / num_pose_heads
).to(device)
pose_cond = t.concat(
[
batch["local_poses"][:, :num_frames_per_token],
t.zeros(
[
batch_size,
(self._args["max_tokens"] - 1) * num_frames_per_token,
batch["local_poses"].shape[-1],
]
).to(device),
],
dim=1,
)
has_pose_cond = t.concat(
[
batch["has_local_poses"][:, :num_frames_per_token],
t.zeros(
[batch_size, (self._args["max_tokens"] - 1) * num_frames_per_token],
dtype=bool,
).to(device),
],
dim=1,
)
for i in range(num_frames_per_token):
onehot_idx = t.nn.functional.one_hot(
batch["pred_num_tokens"].view([-1]).long() * num_frames_per_token
- num_frames_per_token
+ i,
num_classes=self._args["max_tokens"] * num_frames_per_token,
)
pose_cond = pose_cond + onehot_idx.float().view(
[batch_size, -1, 1]
) * batch["local_poses"][:, -num_frames_per_token + i].view(
[batch_size, 1, -1]
)
has_pose_cond = t.logical_or(
has_pose_cond,
(
onehot_idx.float().view([batch_size, -1])
* batch["has_local_poses"][:, -num_frames_per_token + i].view(
[batch_size, 1]
)
).bool(),
)
for step in range(num_pose_inference_steps):
pose_tokens = self._sample_tokens_with_highest_prob(
pose_tokens,
chosen_pose_tokens_prob,
batch["pred_num_tokens"],
step,
num_pose_inference_steps,
)
assert self._args["cond_root_feature_is_from_motion_rep"] in [
"local",
"global",
]
if self._args["cond_root_feature_is_from_motion_rep"] == "local":
pose_root_cond = extract_feature_from_motion_rep(
batch["pred_local_root_values"],
self.local_motion_rep,
self._pose_model.args["cond_root_feature"],
)
else:
assert self._args["cond_root_feature_is_from_motion_rep"] == "global"
pose_root_cond = extract_feature_from_motion_rep(
batch["pred_global_root_values"],
self.global_motion_rep,
self._pose_model.args["cond_root_feature"],
)
pose_model_output = self._pose_model.backbone_net(
pose_tokens,
pose_root_cond,
pose_cond,
has_pose_cond,
batch["pred_num_tokens"],
batch["text_embeddings"],
batch["has_text_embeddings"],
)
if config.get("pose_token_sampling_use_argmax", False):
pose_tokens = pose_model_output["pose_logits"].argmax(dim=-1)
else:
raise ValueError("The release exporter uses argmax token selection.")
pose_tokens_prob = pose_model_output["pose_logits"].softmax(dim=-1)
chosen_pose_tokens_prob = pose_tokens_prob.gather(
dim=-1, index=pose_tokens.unsqueeze(-1)
).squeeze(-1)
if config.get("debug_ground_truth_pose_tokens", None) is not None:
pose_tokens[:, : config["debug_ground_truth_pose_tokens"].shape[1]] = (
config["debug_ground_truth_pose_tokens"][:, :]
)
return (pose_tokens, pose_cond, has_pose_cond)
def _decode_motions_from_predicted_root_and_pose_tokens(
self, batch: dict = {}, config: dict = {}, info: dict = {}
):
if self._pose_model.args["pose_vqvae_motion_rep"] == "local":
assert self._vqvae_pose_model.motion_rep.name == "local"
assert (
self._vqvae_pose_model.decoder_external_cond_feature_mode
== "root_without_hip_height_without_heading"
)
else:
assert self._vqvae_pose_model.motion_rep.name == "global"
assert (
self._vqvae_pose_model.decoder_external_cond_feature_mode
== "root_without_hip_height_without_heading_with_mask"
)
assert (
self._vqvae_pose_model.decoder_target_cond_feature_mode
== self.INTERNAL_POSE_FEATURE_MODE
)
pose_external_root_cond = extract_feature_from_motion_rep(
batch["pred_local_root_values"]
if self._vqvae_pose_model.motion_rep.name == "local"
else batch["pred_global_root_values"],
self._vqvae_pose_model.motion_rep,
self._vqvae_pose_model.decoder_external_cond_feature_mode,
)
if self._vqvae_pose_model.motion_rep.name == "global":
raise NotImplementedError("The global root feature is not supported.")
batch_size, device = (
batch["local_poses"].shape[0],
batch["local_poses"].device,
)
pose_token_mask = t.arange(self._args["max_tokens"]).to(device)[None, :].repeat(
[batch_size, 1]
) < batch["pred_num_tokens"].view([batch_size, 1])
if type(config.get("use_target_cond_in_decoder", True)) == t.Tensor:
global_target_cond = (
config["use_target_cond_in_decoder"]
.bool()
.expand_as(batch["pred_has_pose_cond"])
)
has_target_cond = t.logical_and(
batch["pred_has_pose_cond"], global_target_cond
)
else:
assert type(config.get("use_target_cond_in_decoder", True)) == bool
has_target_cond = (
batch["pred_has_pose_cond"]
if config.get("use_target_cond_in_decoder", True)
else t.zeros_like(batch["pred_has_pose_cond"]).bool()
)
if not config.get("use_constraints_at_decoder", True):
has_target_cond = t.zeros_like(has_target_cond).bool()
if config.get("skip_ending_target_cond", False):
num_frames_per_token = (
self._pose_model.backbone_net.get_num_frames_per_token()
)
has_target_cond[:, num_frames_per_token:] = False
pred_poses = self._vqvae_pose_model.forward_decoder(
batch["pred_pose_tokens"],
target_cond=batch["pred_pose_cond"],
has_target_cond=has_target_cond,
external_cond=pose_external_root_cond,
use_overall_indices=False,
token_mask=pose_token_mask,
)["recon_state"]
num_pred_frames = (
batch["pred_num_tokens"]
* self._pose_model.backbone_net.get_num_frames_per_token()
)
if self._vqvae_pose_model.motion_rep.name == "local":
pred_global_poses = self.motion_rep.dual_rep.local_to_global(
pred_poses,
is_normalized=True,
to_normalize=False,
lengths=num_pred_frames,
)
pred_local_poses = self.local_motion_rep.unnormalize(pred_poses)
else:
pred_local_poses = self.motion_rep.dual_rep.global_to_local(
pred_poses,
is_normalized=True,
to_normalize=False,
lengths=num_pred_frames,
)
pred_global_poses = self.global_motion_rep.unnormalize(pred_poses)
if config.get("final_root_pred_mode", "from_pose_module") == "from_pose_module":
pass
elif config["final_root_pred_mode"] == "from_root_module":
local_root_index, global_root_index = (
self.local_motion_rep.indices["root"],
self.global_motion_rep.indices["root"],
)
pred_global_poses[:, :, global_root_index] = (
self.global_motion_rep.unnormalize(batch["pred_global_root_values"])
)
pred_local_poses[:, :, local_root_index] = (
self.local_motion_rep.unnormalize(batch["pred_local_root_values"])
)
else:
raise NotImplementedError(
f"Unsupported final_root_pred_mode: {config['final_root_pred_mode']}."
)
return (pred_local_poses, pred_global_poses)