Official implementation of MotionBricks: Scalable Real-Time Motions with Modular Latent Generative Model and Smart Primitives (SIGGRAPH 2026)
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Shared Smart Primitive documents, clip loading, and runtime asset preparation."""
from __future__ import annotations
import colorsys
import math
import random
from collections.abc import Iterator
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
import numpy as np
import warp as wp
from motionbricks.model.contract import (
MODEL_MAX_TOKEN_COUNT,
MODEL_MIN_TOKEN_COUNT,
MODEL_POSES_PER_TOKEN,
MODEL_SAMPLE_RATE,
ModelRig,
)
from motionbricks.runtime.pivot import Axis, ConeSide, ConstraintType, OperationOrder, Pivot
from motionbricks.runtime.smart_primitive import (
SmartPrimitive,
SmartPrimitiveAsset,
SmartPrimitiveData,
SmartPrimitiveSequence,
)
from motionbricks.runtime.target_pose import (
IKPin,
MAX_TRANSITION_EARLY_BY_FRAMES,
TargetPoseInferenceParams,
)
from . import document_io
from ..clip_registry import ClipAsset, ClipLoadError, ClipRegistry
from .document_io import DocumentError, DocumentObject, DocumentType
_VELOCITY_EPSILON = 1.0e-6
MIN_TRAJECTORY_SCALE = 1.0
MAX_TRAJECTORY_SCALE = 4.0
_DEFAULT_TRAJECTORY_SCALE = 2.0
_DEFAULT_MIN_TOKENS = 6
_DEFAULT_MAX_TOKENS = 14
_TRAJECTORY_HISTORY_FRAMES = 10
_TRAJECTORY_FUTURE_FRAMES = 10
TRAJECTORY_POINT_COUNT = _TRAJECTORY_HISTORY_FRAMES + 1 + _TRAJECTORY_FUTURE_FRAMES
SMART_PRIMITIVE_DOCUMENT = DocumentType(
kind="Smart Primitive",
format="motionbricks.smart_primitive.authoring",
schema_version=1,
suffix=".smart_primitive.json",
selected_field="selected_node_id",
)
_TARGET_POSE_COMPONENT_TYPE = "target_pose"
_INTERACTION_BOX_COMPONENT_TYPE = "interaction_box"
_PIVOT_COMPONENT_TYPE = "pivot"
_CONTENT_FIELDS = ("sequences", "root")
_NODE_FIELDS = (
"id",
"name",
"local_transform",
"euler_degrees",
"visible",
"translation_locks",
"rotation_locks",
"components",
"children",
)
_TARGET_POSE_FIELDS = (
"type",
"id",
"clip",
"sample_time_seconds",
"frame_index",
"playback_mode",
"hips_constrained",
"num_poses_to_use",
"transition_early_by_frames",
"min_tokens",
"max_tokens",
"ik_pin_mask",
"show_mesh",
"show_skeletons",
"show_velocity",
"show_trajectory",
"trajectory_scale",
"color",
)
_INTERACTION_BOX_FIELDS = (
"type",
"id",
"width",
"depth",
"height",
"color",
)
_PIVOT_FIELDS = (
"type", "id", "motion_direction", "operation_order", "translation_axis", "translation_constraint",
"line_min_offset", "line_max_offset",
"rotation_axis", "rotation_constraint", "cone_half_angle_degrees", "cone_side",
"show_motion_direction", "show_translation_constraint", "show_rotation_constraint",
)
_SEQUENCE_FIELDS = ("id", "name", "node_ids", "pivot_node_id")
def _random_color() -> wp.vec3:
return wp.vec3(*colorsys.hsv_to_rgb(random.random(), 0.65, 0.95))
class PlaybackMode(Enum):
"""How the editor shows a target pose's sample position: as a time or as a frame."""
TIME = "time"
FRAMES = "frames"
class InteractionBoxComponent:
"""An interaction box's size and color. Its node sets its placement."""
def __init__(self, component_id: int) -> None:
"""Create a 1 m interaction box with a random color."""
self.id = component_id
self.width = 1.0
self.depth = 1.0
self.height = 1.0
self.color = _random_color()
def set_dimensions(self, width: float, depth: float, height: float) -> None:
"""Set the box size in meters. Values must be finite and non-negative."""
dimensions = (float(width), float(depth), float(height))
if any(not math.isfinite(value) or value < 0.0 for value in dimensions):
raise ValueError(
"Interaction Box dimensions must be non-negative and finite."
)
self.width, self.depth, self.height = dimensions
class TargetPoseComponent:
"""A target pose's settings and the clip samples used to build it.
Poses use the layout of reference_pose, either local joint transforms or
joint coordinates, with the root first. up_index names the clip axis that
holds height.
"""
def __init__(self, component_id: int, reference_pose: wp.array, up_index: int) -> None:
"""Create a target pose without a clip, using all four poses and 6 to 14 tokens."""
self.id = component_id
self.clip: ClipAsset | None = None
self.num_poses_to_use = MODEL_POSES_PER_TOKEN
self.transition_early_by_frames = 0
self.min_tokens = _DEFAULT_MIN_TOKENS
self.max_tokens = _DEFAULT_MAX_TOKENS
self.ik_pin_mask = IKPin.NONE
self.show_mesh = True
self.show_velocity = False
self.show_trajectory = False
self.show_skeletons = False
self.trajectory_scale = _DEFAULT_TRAJECTORY_SCALE
self.sample_time = 0.0
self.playback_mode = PlaybackMode.TIME
self.hips_constrained = False
self.color = _random_color()
self.set_reference_pose(reference_pose, up_index)
def set_reference_pose(self, reference_pose: wp.array, up_index: int) -> None:
"""Allocate pose buffers that match reference_pose, then resample."""
self.reference_pose = reference_pose
self._up_index = up_index
self.sampled_poses = wp.empty(
(MODEL_POSES_PER_TOKEN, *reference_pose.shape),
dtype=reference_pose.dtype,
device=reference_pose.device,
)
# The displayed pose is the first sample, taken at sample_time.
self.pose = self.sampled_poses[0]
self._trajectory_poses = wp.empty(
(TRAJECTORY_POINT_COUNT, *reference_pose.shape),
dtype=reference_pose.dtype,
device=reference_pose.device,
)
self._trajectory_transforms = wp.empty(
TRAJECTORY_POINT_COUNT,
dtype=wp.transform,
device=reference_pose.device,
)
self._sample()
@property
def frame(self) -> int:
"""Clip frame nearest the sample time, or 0 without a clip."""
if self.clip is None:
return 0
return min(
round(self.sample_time * self.clip.animation.sample_rate),
self.clip.last_frame,
)
@property
def duration(self) -> float:
"""Clip duration in seconds, or 0 without a clip."""
if self.clip is None:
return 0.0
return self.clip.duration
@property
def velocity(self) -> wp.vec3:
"""Root velocity between the first two sampled poses, in m/s.
It is zero when only one pose is used.
"""
if self.num_poses_to_use == 1:
return wp.vec3(0.0)
return self._sampled_velocity
@property
def velocity_magnitude(self) -> float:
"""Length of velocity, in m/s."""
return float(wp.length(self.velocity))
@property
def has_velocity(self) -> bool:
"""Whether velocity is above a small threshold."""
return self.velocity_magnitude > _VELOCITY_EPSILON
@property
def has_trajectory(self) -> bool:
"""Whether a root trajectory is available, which needs a clip."""
return self.clip is not None
@property
def trajectory_transforms(self) -> wp.array:
"""Sample the preview path on demand, reusing it until the target changes."""
if self._trajectory_dirty:
self._update_trajectory_samples()
self._trajectory_dirty = False
return self._trajectory_transforms
@property
def skeleton_count(self) -> int:
"""Number of preview skeletons: the poses to use, or 1 without a clip."""
if self.clip is None:
return 1
return self.num_poses_to_use
def set_clip(self, clip: ClipAsset) -> None:
"""Use a clip and sample it from its start."""
self.clip = clip
self.sample_time = 0.0
self._sample()
def restore_sample_state(
self,
clip: ClipAsset | None,
playback_mode: PlaybackMode,
sample_time: float,
frame_index: int,
hips_constrained: bool,
) -> None:
"""Restore saved clip settings, then resample the target pose once."""
self.clip = clip
self.playback_mode = playback_mode
self.hips_constrained = hips_constrained
self.sample_time = (
frame_index / clip.animation.sample_rate
if clip is not None and playback_mode is PlaybackMode.FRAMES
else sample_time
)
self._sample()
def set_time(self, sample_time: float) -> None:
"""Sample the clip at sample_time (in seconds), clamped to its duration."""
self.sample_time = min(max(float(sample_time), 0.0), self.duration)
self._sample()
def constrain_hips(self, enabled: bool) -> None:
"""Turn the root constraint on or off, then resample the clip."""
enabled = bool(enabled)
if enabled == self.hips_constrained:
return
self.hips_constrained = enabled
self._sample()
def set_num_poses_to_use(self, num_poses: int) -> None:
"""Set the poses to use, clamped to 1 to 4."""
self.num_poses_to_use = min(
max(int(num_poses), 1),
MODEL_POSES_PER_TOKEN,
)
def set_token_range(self, min_tokens: int, max_tokens: int) -> None:
"""Set the token counts the model may choose, clamped to 6 to 16.
The maximum never drops below the minimum.
"""
self.min_tokens = min(max(int(min_tokens), MODEL_MIN_TOKEN_COUNT), MODEL_MAX_TOKEN_COUNT)
self.max_tokens = min(max(int(max_tokens), self.min_tokens), MODEL_MAX_TOKEN_COUNT)
def set_ik_pin_enabled(self, pin: IKPin, enabled: bool) -> None:
"""Pin or unpin one hand or foot for this target pose."""
pin = IKPin(pin)
if enabled:
self.ik_pin_mask |= pin
else:
self.ik_pin_mask &= ~pin
def set_trajectory_scale(self, scale: float) -> None:
"""Set the spacing of the trajectory preview points, in frames at the model's sample rate.
It must be MIN_TRAJECTORY_SCALE to MAX_TRAJECTORY_SCALE.
"""
scale = float(scale)
if (
not math.isfinite(scale)
or scale < MIN_TRAJECTORY_SCALE
or scale > MAX_TRAJECTORY_SCALE
):
raise ValueError(
"Trajectory scale must be finite, from "
f"{MIN_TRAJECTORY_SCALE:.1f} to {MAX_TRAJECTORY_SCALE:.1f}."
)
if scale == self.trajectory_scale:
return
self.trajectory_scale = scale
self._trajectory_dirty = True
def _sample(self) -> None:
"""Sample four 30 Hz poses from sample_time."""
self._root_offset = wp.transform_identity()
if self.clip is None:
for frame in range(MODEL_POSES_PER_TOKEN):
wp.copy(self.sampled_poses[frame], self.reference_pose)
else:
animation = self.clip.animation
if self.hips_constrained:
# Remove the root's offset along the ground, keeping its height.
animation.sample(self.sample_time, self.pose)
offset = -self.pose.numpy().reshape(-1)[:3]
offset[self._up_index] = 0.0
self._root_offset = wp.transform(wp.vec3(*offset), wp.quat_identity())
for frame in range(MODEL_POSES_PER_TOKEN):
animation.sample(
self.sample_time + frame / MODEL_SAMPLE_RATE,
self.sampled_poses[frame],
root_offset=self._root_offset,
)
roots = self.sampled_poses.numpy().reshape(MODEL_POSES_PER_TOKEN, -1)[:, :3]
self.hips_position = wp.vec3(*roots[0])
self._sampled_velocity = (wp.vec3(*roots[1]) - wp.vec3(*roots[0])) * MODEL_SAMPLE_RATE
self._trajectory_dirty = True
def _update_trajectory_samples(self) -> None:
if self.clip is None:
return
interval = self.trajectory_scale / MODEL_SAMPLE_RATE
for point in range(TRAJECTORY_POINT_COUNT):
self.clip.animation.sample(
self.sample_time + (point - _TRAJECTORY_HISTORY_FRAMES) * interval,
self._trajectory_poses[point],
root_offset=self._root_offset,
)
roots = self._trajectory_poses.numpy().reshape(TRAJECTORY_POINT_COUNT, -1)[:, :7]
self._trajectory_transforms.assign(np.ascontiguousarray(roots))
class PivotComponent:
"""Pivot rules and preview visibility for a hierarchy node."""
def __init__(self, component_id: int) -> None:
"""Create a pivot with default rules and every preview shown."""
self.id = component_id
self.pivot = Pivot()
self.show_motion_direction = True
self.show_translation_constraint = True
self.show_rotation_constraint = True
class SceneNode:
"""A node's local transform, components, and children."""
def __init__(
self,
node_id: int,
name: str,
parent: SceneNode | None,
local_xform: wp.transform | None = None,
) -> None:
"""Create a visible node without components under parent."""
self.id = node_id
self.name = name
self.parent = parent
self.visible = True
self.translation_locks = [False, False, False]
self.rotation_locks = [False, False, False]
self.euler_degrees: list[float] | None = None
self.children: list[SceneNode] = []
# A node holds at most one component of each kind. An empty slot is None.
self.target_pose: TargetPoseComponent | None = None
self.interaction_box: InteractionBoxComponent | None = None
self.pivot: PivotComponent | None = None
self.local_xform = (
wp.transform_identity()
if local_xform is None
else wp.transform(local_xform)
)
@property
def components(self) -> tuple[TargetPoseComponent | InteractionBoxComponent | PivotComponent, ...]:
"""The filled slots: Target Pose, Interaction Box, then Pivot."""
return tuple(
component for component in (self.target_pose, self.interaction_box, self.pivot) if component is not None
)
@dataclass(slots=True)
class TargetPoseSequence:
"""Targets in generation order, with an optional pivot to align them."""
id: int
name: str
node_ids: list[int]
pivot_node_id: int | None = None
class SmartPrimitiveDocument:
"""The editable hierarchy, components, and target pose sequences."""
def __init__(self, rig: ModelRig, reference_pose: wp.array, model_to_runtime: wp.transform) -> None:
"""Create an empty document that holds only the root node.
Args:
rig: Rig the document is authored for.
reference_pose: Reference pose that sets the pose layout, either local joint
transforms or joint coordinates.
model_to_runtime: Transform that places clip poses in world space.
"""
self._next_id = 1
self._nodes: dict[int, SceneNode] = {}
self.root = SceneNode(self._new_id(), "Root", None)
self._nodes[self.root.id] = self.root
self.sequences: list[TargetPoseSequence] = []
self.set_rig(rig, reference_pose, model_to_runtime)
def set_rig(self, rig: ModelRig, reference_pose: wp.array, model_to_runtime: wp.transform) -> None:
"""Switch the rig and reset every target to its reference pose.
No target may have a clip.
"""
rig = ModelRig(rig)
targets = [component for _, component in self.iter_target_poses()]
if any(target.clip is not None for target in targets):
raise ValueError("Cannot change the rig while a target has a clip.")
self.rig = rig
self.reference_pose = reference_pose
self.model_to_runtime = wp.transform(model_to_runtime)
# Clips store height along Y (BVH) or Z (CSV). Find the clip axis that model_to_runtime turns into Z.
up = wp.quat_rotate_inv(self.model_to_runtime.q, wp.vec3(0.0, 0.0, 1.0))
self.up_index = int(np.argmax(np.abs(np.asarray(up))))
for target in targets:
target.set_reference_pose(reference_pose, self.up_index)
def set_contents(self, root: SceneNode, sequences: list[TargetPoseSequence], next_id: int) -> None:
"""Replace the hierarchy and the sequences, such as with ones loaded from a file.
next_id must exceed every ID in use. New nodes, components, and sequences take
their IDs from it.
"""
self.root = root
self._nodes = {node.id: node for node in self.iter_subtree(root)}
self.sequences = sequences
self._next_id = next_id
def _new_id(self) -> int:
item_id = self._next_id
self._next_id += 1
return item_id
def add_node(self, parent: SceneNode | None = None) -> SceneNode:
"""Add a node under parent, or under the root node, and return it."""
parent = parent or self.root
node_id = self._new_id()
node = SceneNode(node_id, f"Node {node_id}", parent)
parent.children.append(node)
self._nodes[node.id] = node
return node
def remove_node(self, node: SceneNode) -> None:
"""Delete a node with its descendants, and drop them from every sequence.
The root node stays.
"""
if node is self.root:
return
node.parent.children.remove(node)
removed_node_ids = {descendant.id for descendant in self.iter_subtree(node)}
for descendant in self.iter_subtree(node):
self._nodes.pop(descendant.id, None)
for sequence in self.sequences:
if sequence.pivot_node_id in removed_node_ids:
sequence.pivot_node_id = None
sequence.node_ids[:] = [
node_id
for node_id in sequence.node_ids
if node_id not in removed_node_ids
]
def find_node(self, node_id: int | None) -> SceneNode | None:
"""Return the node with node_id, or None."""
return self._nodes.get(node_id)
def iter_nodes(self) -> Iterator[SceneNode]:
"""Visit the root and descendants with every parent before its children."""
return self.iter_subtree(self.root)
@staticmethod
def iter_subtree(root: SceneNode) -> Iterator[SceneNode]:
"""Visit parents before children, preserving the authored sibling order."""
pending = [root]
while pending:
node = pending.pop()
yield node
pending.extend(reversed(node.children))
@staticmethod
def iter_ancestors(node: SceneNode) -> Iterator[SceneNode]:
"""Visit a node, then walk up through its parents."""
current: SceneNode | None = node
while current is not None:
yield current
current = current.parent
def world_xform(self, node: SceneNode) -> wp.transform:
"""Return the node's transform composed through all of its parents."""
world_xform = wp.transform(node.local_xform)
parent = node.parent
while parent is not None:
world_xform = wp.mul(parent.local_xform, world_xform)
parent = parent.parent
return world_xform
def target_pose_world_xform(self, node: SceneNode) -> wp.transform:
"""Return the transform that places the node's clip poses in world space."""
return wp.mul(self.world_xform(node), self.model_to_runtime)
def is_visible(self, node: SceneNode) -> bool:
"""Return whether the node and all of its ancestors are visible."""
return all(ancestor.visible for ancestor in self.iter_ancestors(node))
def can_reparent(self, node: SceneNode, new_parent: SceneNode) -> bool:
"""Return whether node can move under new_parent without forming a cycle."""
if node is self.root or node.parent is new_parent:
return False
return all(ancestor is not node for ancestor in self.iter_ancestors(new_parent))
def reparent(self, node: SceneNode, new_parent: SceneNode) -> bool:
"""Move node under new_parent, keeping its placement.
Return whether the node moved.
"""
if not self.can_reparent(node, new_parent):
return False
world_xform = self.world_xform(node)
new_parent_world = self.world_xform(new_parent)
node.parent.children.remove(node)
node.parent = new_parent
new_parent.children.append(node)
node.local_xform = wp.mul(
wp.transform_inverse(new_parent_world),
world_xform,
)
return True
def add_target_pose(self, node: SceneNode) -> TargetPoseComponent:
"""Return the node's Target Pose component, adding one when it has none."""
if node.target_pose is None:
node.target_pose = TargetPoseComponent(self._new_id(), self.reference_pose, self.up_index)
return node.target_pose
def add_interaction_box(self, node: SceneNode) -> InteractionBoxComponent:
"""Return the node's Interaction Box component, adding one when it has none."""
if node.interaction_box is None:
node.interaction_box = InteractionBoxComponent(self._new_id())
return node.interaction_box
def add_pivot(self, node: SceneNode) -> PivotComponent:
"""Return the node's Pivot component, adding one when it has none."""
if node.pivot is None:
node.pivot = PivotComponent(self._new_id())
return node.pivot
def remove_component(
self,
node: SceneNode,
component: TargetPoseComponent | InteractionBoxComponent | PivotComponent,
) -> None:
"""Remove a component from a node, and drop every sequence reference to it."""
if component is node.target_pose:
node.target_pose = None
for sequence in self.sequences:
sequence.node_ids[:] = [
node_id for node_id in sequence.node_ids if node_id != node.id
]
elif component is node.interaction_box:
node.interaction_box = None
elif component is node.pivot:
node.pivot = None
for sequence in self.sequences:
if sequence.pivot_node_id == node.id:
sequence.pivot_node_id = None
def add_sequence(self) -> TargetPoseSequence:
"""Add an empty sequence with the next free name, and return it."""
existing_names = {sequence.name for sequence in self.sequences}
sequence_number = 1
while f"Sequence {sequence_number}" in existing_names:
sequence_number += 1
sequence = TargetPoseSequence(
self._new_id(),
f"Sequence {sequence_number}",
[],
)
self.sequences.append(sequence)
return sequence
def find_sequence(self, sequence_id: int | None) -> TargetPoseSequence | None:
"""Return the sequence with sequence_id, or None."""
return next(
(
sequence
for sequence in self.sequences
if sequence.id == sequence_id
),
None,
)
def remove_sequence(self, sequence: TargetPoseSequence) -> None:
"""Delete a sequence."""
self.sequences.remove(sequence)
def add_sequence_node(
self,
sequence: TargetPoseSequence,
node: SceneNode,
) -> None:
"""Append a node that has a Target Pose to a sequence."""
if self.find_sequence(sequence.id) is not sequence:
raise ValueError("The sequence must belong to this Smart Primitive.")
if self.find_node(node.id) is not node:
raise ValueError("The node must belong to this Smart Primitive.")
if node.target_pose is None:
raise ValueError("Sequence nodes must own a Target Pose component.")
sequence.node_ids.append(node.id)
def set_sequence_pivot(
self, sequence: TargetPoseSequence, node: SceneNode | None,
) -> None:
"""Set the node whose pivot aligns the sequence. None removes the pivot."""
if self.find_sequence(sequence.id) is not sequence:
raise ValueError("The sequence must belong to this Smart Primitive.")
if node is not None:
if self.find_node(node.id) is not node or node.pivot is None:
raise ValueError("The pivot node must belong here and own a Pivot component.")
sequence.pivot_node_id = None if node is None else node.id
@staticmethod
def remove_sequence_node(
sequence: TargetPoseSequence,
index: int,
) -> None:
"""Remove the step at index from a sequence."""
if not 0 <= index < len(sequence.node_ids):
raise IndexError(f"Sequence node index {index!r} is out of range.")
sequence.node_ids.pop(index)
@staticmethod
def move_sequence_node(
sequence: TargetPoseSequence,
source_index: int,
destination_index: int,
) -> None:
"""Move a sequence step from one index to another."""
node_count = len(sequence.node_ids)
if not 0 <= source_index < node_count:
raise IndexError(f"Sequence node index {source_index!r} is out of range.")
if not 0 <= destination_index < node_count:
raise IndexError(
f"Sequence destination index {destination_index!r} is out of range."
)
node_id = sequence.node_ids.pop(source_index)
sequence.node_ids.insert(destination_index, node_id)
def sequences_referencing(
self,
node: SceneNode,
) -> Iterator[TargetPoseSequence]:
"""Yield each sequence that uses node as a step or as its pivot."""
for sequence in self.sequences:
if node.id in sequence.node_ids or node.id == sequence.pivot_node_id:
yield sequence
def iter_target_poses(self) -> Iterator[tuple[SceneNode, TargetPoseComponent]]:
"""Yield each node with its Target Pose component, parents before children."""
return ((node, node.target_pose) for node in self.iter_nodes() if node.target_pose is not None)
def iter_interaction_boxes(self) -> Iterator[tuple[SceneNode, InteractionBoxComponent]]:
"""Yield each node with its Interaction Box component, parents before children."""
return ((node, node.interaction_box) for node in self.iter_nodes() if node.interaction_box is not None)
def iter_pivots(self) -> Iterator[tuple[SceneNode, PivotComponent]]:
"""Yield each node with its Pivot component, parents before children."""
return ((node, node.pivot) for node in self.iter_nodes() if node.pivot is not None)
class SmartPrimitiveDocumentError(DocumentError):
"""Raised when a Smart Primitive breaks a document rule or cannot be prepared for runtime.
A clip that fails to load also raises it. JSON and file problems raise DocumentError, the
base class.
"""
def _serialize_target_pose(
component: TargetPoseComponent,
document_path: Path,
) -> dict:
return {
"type": _TARGET_POSE_COMPONENT_TYPE,
"id": component.id,
"clip": (
None
if component.clip is None
else document_io.serialize_reference(component.clip.path, document_path)
),
"sample_time_seconds": component.sample_time,
"frame_index": component.frame,
"playback_mode": component.playback_mode.value,
"hips_constrained": component.hips_constrained,
"num_poses_to_use": component.num_poses_to_use,
"transition_early_by_frames": component.transition_early_by_frames,
"min_tokens": component.min_tokens,
"max_tokens": component.max_tokens,
"ik_pin_mask": int(component.ik_pin_mask),
"show_mesh": component.show_mesh,
"show_skeletons": component.show_skeletons,
"show_velocity": component.show_velocity,
"show_trajectory": component.show_trajectory,
"trajectory_scale": component.trajectory_scale,
"color": document_io.serialize_color(component.color, f"Target Pose {component.id}.color"),
}
def _serialize_interaction_box(
component: InteractionBoxComponent,
) -> dict:
return {
"type": _INTERACTION_BOX_COMPONENT_TYPE,
"id": component.id,
"width": component.width,
"depth": component.depth,
"height": component.height,
"color": document_io.serialize_color(component.color, f"Interaction Box {component.id}.color"),
}
def _serialize_pivot(component: PivotComponent) -> dict:
pivot = component.pivot
return {
"type": _PIVOT_COMPONENT_TYPE,
"id": component.id,
"motion_direction": list(pivot.motion_direction),
"operation_order": pivot.operation_order.value,
"translation_axis": pivot.translation_axis.value,
"translation_constraint": pivot.translation_constraint.value,
"line_min_offset": pivot.line_min_offset,
"line_max_offset": pivot.line_max_offset,
"rotation_axis": pivot.rotation_axis.value,
"rotation_constraint": pivot.rotation_constraint.value,
"cone_half_angle_degrees": pivot.cone_half_angle_degrees,
"cone_side": pivot.cone_side.value,
"show_motion_direction": component.show_motion_direction,
"show_translation_constraint": component.show_translation_constraint,
"show_rotation_constraint": component.show_rotation_constraint,
}
def _serialize_scene_node(node: SceneNode, document_path: Path) -> dict:
components = []
if node.target_pose is not None:
components.append(_serialize_target_pose(node.target_pose, document_path))
if node.interaction_box is not None:
components.append(_serialize_interaction_box(node.interaction_box))
if node.pivot is not None:
components.append(_serialize_pivot(node.pivot))
return {
"id": node.id,
"name": node.name,
"local_transform": document_io.serialize_transform(node.local_xform, f"Node {node.name!r}.local_transform"),
"euler_degrees": (
None
if node.euler_degrees is None
else [float(value) for value in node.euler_degrees]
),
"visible": node.visible,
"translation_locks": list(node.translation_locks),
"rotation_locks": list(node.rotation_locks),
"components": components,
"children": [
_serialize_scene_node(child, document_path) for child in node.children
],
}
def _serialize_sequence(sequence: TargetPoseSequence) -> dict:
return {
"id": sequence.id,
"name": sequence.name,
"node_ids": list(sequence.node_ids),
"pivot_node_id": sequence.pivot_node_id,
}
def _prepare_runtime_target_poses(
authoring: SmartPrimitiveDocument,
targets: list[tuple[SceneNode, TargetPoseComponent]],
) -> tuple[np.ndarray, tuple[TargetPoseInferenceParams, ...]]:
pose_shape = authoring.reference_pose.numpy().shape
packed_poses = np.empty(
(len(targets) * MODEL_POSES_PER_TOKEN, *pose_shape),
dtype=np.float32,
)
inference_params = []
for target_index, (node, component) in enumerate(targets):
if component.clip is None:
raise SmartPrimitiveDocumentError(
f"Target Pose on node {node.name!r} has no clip."
)
sampled_poses = component.sampled_poses.numpy()
# Checks every joint of local joint transforms, or the root of joint coordinates.
document_io.validate_transforms(sampled_poses[..., :7], f"Target Pose on node {node.name!r}")
if not np.all(np.isfinite(sampled_poses)):
raise SmartPrimitiveDocumentError(
f"Target Pose on node {node.name!r} contains non-finite values."
)
frame_offset = target_index * MODEL_POSES_PER_TOKEN
target_slice = packed_poses[
frame_offset : frame_offset + MODEL_POSES_PER_TOKEN
]
target_slice[:] = sampled_poses
# Only the root moves to world space. Joint transforms and angles stay unchanged.
roots = target_slice.reshape(MODEL_POSES_PER_TOKEN, -1)[:, :7]
for frame in range(MODEL_POSES_PER_TOKEN):
roots[frame] = wp.mul(authoring.model_to_runtime, wp.transform(*roots[frame]))
inference_params.append(TargetPoseInferenceParams(
transition_early_by_frames=component.transition_early_by_frames,
num_poses_to_use=component.num_poses_to_use,
ik_pin_mask=component.ik_pin_mask,
allowed_token_counts=tuple(range(component.min_tokens, component.max_tokens + 1)),
))
return packed_poses, tuple(inference_params)
def _prepare_runtime_sequences(
authoring: SmartPrimitiveDocument,
node_indices: dict[SceneNode, int],
) -> tuple[SmartPrimitiveSequence, ...]:
"""Map saved node IDs to runtime indices while preserving sequence and step order."""
prepared = []
for sequence in authoring.sequences:
if not sequence.node_ids:
raise SmartPrimitiveDocumentError(
f"Sequence {sequence.name!r} must contain at least one Target Pose."
)
nodes = [authoring.find_node(node_id) for node_id in sequence.node_ids]
start_speed = nodes[0].target_pose.velocity_magnitude
if not math.isfinite(start_speed):
raise SmartPrimitiveDocumentError(
f"Sequence {sequence.name!r} has a non-finite start speed."
)
pivot_node_index = -1
if sequence.pivot_node_id is not None:
pivot_node_index = node_indices[authoring.find_node(sequence.pivot_node_id)]
prepared.append(SmartPrimitiveSequence(
name=sequence.name,
node_indices=tuple(node_indices[node] for node in nodes),
start_speed=start_speed,
pivot_node_index=pivot_node_index,
))
return tuple(prepared)
def build_runtime_smart_primitive_data(
authoring: SmartPrimitiveDocument,
) -> SmartPrimitiveData:
"""Prepare the hierarchy, sampled poses, and interaction settings for playback."""
# Runtime transforms require parents before children. IDs reflect creation
# order, so derive indices from the hierarchy, including after reparenting.
nodes = list(authoring.iter_nodes())
node_indices = {node: index for index, node in enumerate(nodes)}
node_local_transforms = np.array([list(node.local_xform) for node in nodes], dtype=np.float32)
node_parent_indices = np.asarray(
[
-1 if node.parent is None else node_indices[node.parent]
for node in nodes
],
dtype=np.int32,
)
node_names = tuple(node.name for node in nodes)
if len(set(node_names)) != len(node_names):
raise SmartPrimitiveDocumentError("Runtime node names must be unique.")
sequences = _prepare_runtime_sequences(authoring, node_indices)
pivot_components = list(authoring.iter_pivots())
pivot_node_indices = np.asarray([node_indices[node] for node, _ in pivot_components], dtype=np.int32)
pivots = tuple(component.pivot for _, component in pivot_components)
targets = list(authoring.iter_target_poses())
target_node_indices = np.asarray(
[node_indices[node] for node, _ in targets],
dtype=np.int32,
)
target_poses, target_inference_params = _prepare_runtime_target_poses(authoring, targets)
boxes = list(authoring.iter_interaction_boxes())
box_node_indices = np.asarray(
[node_indices[node] for node, _ in boxes],
dtype=np.int32,
)
box_half_extents = np.asarray(
[
(0.5 * component.width, 0.5 * component.depth, 0.5 * component.height)
for _, component in boxes
],
dtype=np.float32,
).reshape((len(boxes), 3))
return SmartPrimitiveData(
rig=authoring.rig,
node_local_transforms=node_local_transforms,
node_parent_indices=node_parent_indices,
node_names=node_names,
target_node_indices=target_node_indices,
target_poses=target_poses,
target_inference_params=target_inference_params,
box_node_indices=box_node_indices,
box_half_extents=box_half_extents,
pivot_node_indices=pivot_node_indices,
pivots=pivots,
sequences=sequences,
)
def save_smart_primitive_document(
authoring: SmartPrimitiveDocument,
path: str | Path,
selected_node_id: int,
camera_state: dict,
) -> Path:
"""Save the document and editor settings, then return the file path."""
document_path = SMART_PRIMITIVE_DOCUMENT.normalize_path(path).resolve()
document = SMART_PRIMITIVE_DOCUMENT.header(document_path, authoring.rig, selected_node_id, camera_state)
document["sequences"] = [_serialize_sequence(sequence) for sequence in authoring.sequences]
document["root"] = _serialize_scene_node(authoring.root, document_path)
document_io.save_json(document, document_path)
return document_path
class _SmartPrimitiveDocumentLoader:
def __init__(
self,
clip_registry: ClipRegistry,
reference_pose: wp.array,
model_to_runtime: wp.transform,
document_path: Path,
) -> None:
self.clip_registry = clip_registry
self.reference_pose = reference_pose
self.model_to_runtime = model_to_runtime
self.document_path = document_path
self.used_ids: set[int] = set()
self.maximum_id = 0
self.nodes: dict[int, SceneNode] = {}
def load(self) -> tuple[SmartPrimitiveDocument, int, dict]:
document, rig, selected_node_id, camera_state = SMART_PRIMITIVE_DOCUMENT.read(
self.document_path, _CONTENT_FIELDS,
)
# Check the rig before loading any clips.
if rig != self.clip_registry.rig:
raise SmartPrimitiveDocumentError(
f"{self.document_path.name} was saved for the '{rig}' rig, "
f"but this app uses '{self.clip_registry.rig}'."
)
authoring = SmartPrimitiveDocument(
self.clip_registry.rig,
self.reference_pose,
self.model_to_runtime,
)
root = self._load_node(
document.object("root", _NODE_FIELDS),
None,
authoring,
)
sequences = [
self._load_sequence(data, authoring)
for data in document.object_array("sequences", _SEQUENCE_FIELDS)
]
sequence_names = [sequence.name for sequence in sequences]
if len(set(sequence_names)) != len(sequence_names):
raise SmartPrimitiveDocumentError("Sequence names must be unique.")
authoring.set_contents(root, sequences, self.maximum_id + 1)
if authoring.find_node(selected_node_id) is None:
selected_node_id = authoring.root.id
return authoring, selected_node_id, camera_state
def _load_sequence(
self,
data: DocumentObject,
authoring: SmartPrimitiveDocument,
) -> TargetPoseSequence:
sequence = TargetPoseSequence(
self._claim_id(data),
data.string("name"),
[],
)
for index, value in enumerate(data.array("node_ids")):
location = f"{data.field_location('node_ids')}[{index}]"
node_id = document_io.integer(value, location, minimum=1)
node = self.nodes.get(node_id)
if node is None:
raise SmartPrimitiveDocumentError(
f"{location} does not identify a hierarchy node."
)
if node.target_pose is None:
raise SmartPrimitiveDocumentError(
f"{location} identifies node {node.name!r}, which has no Target Pose."
)
sequence.node_ids.append(node_id)
if data.value("pivot_node_id") is not None:
pivot_id = data.integer("pivot_node_id", minimum=1)
node = self.nodes.get(pivot_id)
if node is None or node.pivot is None:
raise SmartPrimitiveDocumentError(
f"{data.field_location('pivot_node_id')} must identify a node with a Pivot component."
)
sequence.pivot_node_id = pivot_id
return sequence
def _claim_id(self, data: DocumentObject) -> int:
item_id = data.integer("id", minimum=1)
if item_id in self.used_ids:
raise SmartPrimitiveDocumentError(
f"Duplicate ID {item_id} at {data.field_location('id')}."
)
self.used_ids.add(item_id)
self.maximum_id = max(self.maximum_id, item_id)
return item_id
def _load_node(
self,
data: DocumentObject,
parent: SceneNode | None,
authoring: SmartPrimitiveDocument,
) -> SceneNode:
node_id = self._claim_id(data)
node = SceneNode(
node_id,
data.string("name"),
parent,
document_io.load_transform(data.object("local_transform", document_io.TRANSFORM_FIELDS)),
)
node.visible = data.boolean("visible")
node.translation_locks = data.bool_vector("translation_locks", 3)
node.rotation_locks = data.bool_vector("rotation_locks", 3)
euler_degrees = data.value("euler_degrees")
node.euler_degrees = (
None if euler_degrees is None else data.number_vector("euler_degrees", 3)
)
self.nodes[node.id] = node
components_location = data.field_location("components")
for index, value in enumerate(data.array("components")):
location = f"{components_location}[{index}]"
component_type = document_io.object_type(value, location)
if component_type == _TARGET_POSE_COMPONENT_TYPE:
if node.target_pose is not None:
raise SmartPrimitiveDocumentError(f"{data.location} contains more than one Target Pose.")
node.target_pose = self._load_target_pose_component(
DocumentObject(value, location, _TARGET_POSE_FIELDS), authoring,
)
elif component_type == _INTERACTION_BOX_COMPONENT_TYPE:
if node.interaction_box is not None:
raise SmartPrimitiveDocumentError(f"{data.location} contains more than one Interaction Box.")
node.interaction_box = self._load_interaction_box_component(
DocumentObject(value, location, _INTERACTION_BOX_FIELDS),
)
elif component_type == _PIVOT_COMPONENT_TYPE:
if node.pivot is not None:
raise SmartPrimitiveDocumentError(f"{data.location} contains more than one Pivot.")
node.pivot = self._load_pivot_component(DocumentObject(value, location, _PIVOT_FIELDS))
else:
raise SmartPrimitiveDocumentError(f"Unsupported component type {component_type!r} at {location}.")
for child_data in data.object_array("children", _NODE_FIELDS):
child = self._load_node(child_data, node, authoring)
node.children.append(child)
return node
def _load_pivot_component(self, data: DocumentObject) -> PivotComponent:
component = PivotComponent(self._claim_id(data))
try:
component.pivot = Pivot(
motion_direction=tuple(data.number_vector("motion_direction", 3)),
operation_order=OperationOrder(data.string("operation_order")),
translation_axis=Axis(data.string("translation_axis")),
translation_constraint=ConstraintType(data.string("translation_constraint")),
line_min_offset=data.number("line_min_offset"),
line_max_offset=data.number("line_max_offset"),
rotation_axis=Axis(data.string("rotation_axis")),
rotation_constraint=ConstraintType(data.string("rotation_constraint")),
cone_half_angle_degrees=data.number("cone_half_angle_degrees"),
cone_side=ConeSide(data.string("cone_side")),
)
except ValueError as exc:
raise SmartPrimitiveDocumentError(f"Invalid Pivot at {data.location}: {exc}") from exc
component.show_motion_direction = data.boolean("show_motion_direction")
component.show_translation_constraint = data.boolean("show_translation_constraint")
component.show_rotation_constraint = data.boolean("show_rotation_constraint")
return component
def _load_target_pose_component(
self,
data: DocumentObject,
authoring: SmartPrimitiveDocument,
) -> TargetPoseComponent:
component_id = self._claim_id(data)
component = TargetPoseComponent(component_id, authoring.reference_pose, authoring.up_index)
playback_mode = self._load_playback_mode(data)
sample_time = data.number("sample_time_seconds")
if sample_time < 0.0:
raise SmartPrimitiveDocumentError(
f"{data.field_location('sample_time_seconds')} cannot be negative."
)
frame_index = data.integer("frame_index", minimum=0)
hips_constrained = data.boolean("hips_constrained")
self._load_target_pose_properties(component, data)
clip = self._load_clip(data)
self._validate_sample_position(
clip,
sample_time,
frame_index,
data.location,
)
component.restore_sample_state(
clip,
playback_mode,
sample_time,
frame_index,
hips_constrained,
)
return component
def _load_interaction_box_component(
self,
data: DocumentObject,
) -> InteractionBoxComponent:
component = InteractionBoxComponent(self._claim_id(data))
dimension_fields = ("width", "depth", "height")
dimensions = [data.number(field) for field in dimension_fields]
for field, value in zip(dimension_fields, dimensions):
if value < 0.0:
raise SmartPrimitiveDocumentError(
f"{data.field_location(field)} cannot be negative."
)
component.set_dimensions(*dimensions)
component.color = data.color("color")
return component
@staticmethod
def _load_playback_mode(data: DocumentObject) -> PlaybackMode:
value = data.string("playback_mode")
try:
return PlaybackMode(value)
except ValueError as exc:
raise SmartPrimitiveDocumentError(
f"{data.field_location('playback_mode')} must be 'time' or 'frames'."
) from exc
def _load_target_pose_properties(
self,
component: TargetPoseComponent,
data: DocumentObject,
) -> None:
component.num_poses_to_use = data.integer(
"num_poses_to_use",
minimum=1,
maximum=MODEL_POSES_PER_TOKEN,
)
component.transition_early_by_frames = data.integer(
"transition_early_by_frames",
minimum=0,
maximum=MAX_TRANSITION_EARLY_BY_FRAMES,
)
component.min_tokens = data.integer("min_tokens", minimum=MODEL_MIN_TOKEN_COUNT, maximum=MODEL_MAX_TOKEN_COUNT)
component.max_tokens = data.integer("max_tokens", minimum=component.min_tokens, maximum=MODEL_MAX_TOKEN_COUNT)
ik_pin_mask = data.integer("ik_pin_mask", minimum=0)
if ik_pin_mask & ~int(IKPin.ALL):
raise SmartPrimitiveDocumentError(
f"{data.field_location('ik_pin_mask')} contains unsupported bits."
)
component.ik_pin_mask = IKPin(ik_pin_mask)
component.show_mesh = data.boolean("show_mesh")
component.show_skeletons = data.boolean("show_skeletons")
component.show_velocity = data.boolean("show_velocity")
component.show_trajectory = data.boolean("show_trajectory")
trajectory_scale = data.number("trajectory_scale")
try:
component.set_trajectory_scale(trajectory_scale)
except ValueError as exc:
raise SmartPrimitiveDocumentError(
f"{data.field_location('trajectory_scale')} must be "
f"{MIN_TRAJECTORY_SCALE:.1f} to {MAX_TRAJECTORY_SCALE:.1f}."
) from exc
component.color = data.color("color")
@staticmethod
def _validate_sample_position(
clip: ClipAsset | None,
sample_time: float,
frame_index: int,
location: str,
) -> None:
if clip is None:
if sample_time != 0.0 or frame_index != 0:
raise SmartPrimitiveDocumentError(
f"{location} cannot select a frame without a clip."
)
return
if frame_index > clip.last_frame:
raise SmartPrimitiveDocumentError(
f"{location}.frame_index exceeds the clip's last frame."
)
if sample_time > clip.duration + 1.0e-6:
raise SmartPrimitiveDocumentError(
f"{location}.sample_time_seconds exceeds the clip duration."
)
def _load_clip(self, data: DocumentObject) -> ClipAsset | None:
resolved_path = document_io.resolve_reference(data, "clip", self.document_path, kind="clip")
if resolved_path is None:
return None
try:
return self.clip_registry.load(resolved_path)
except ClipLoadError as exc:
raise SmartPrimitiveDocumentError(str(exc)) from exc
def load_smart_primitive_rig(path: str | Path) -> ModelRig:
"""Return the rig a document was saved for, reading only its header."""
_, rig, _, _ = SMART_PRIMITIVE_DOCUMENT.read(Path(path).expanduser().resolve(), _CONTENT_FIELDS)
return rig
def load_smart_primitive_document(
clip_registry: ClipRegistry,
reference_pose: wp.array,
model_to_runtime: wp.transform,
path: str | Path,
) -> tuple[SmartPrimitiveDocument, int, dict]:
"""Load a document for the registry's rig, checking it before loading clips."""
return _SmartPrimitiveDocumentLoader(
clip_registry,
reference_pose,
model_to_runtime,
Path(path).expanduser().resolve(),
).load()
class SmartPrimitiveAssetPool:
"""Load documents and clips once and share prepared data between runtime instances."""
def __init__(
self,
clip_registry: ClipRegistry,
reference_pose: wp.array,
model_to_runtime: wp.transform,
) -> None:
"""Create an empty pool for one rig's clip registry."""
self.clip_registry = clip_registry
self.reference_pose = reference_pose
self.device = clip_registry.device
self.model_to_runtime = wp.transform(model_to_runtime)
self._assets: dict[Path, SmartPrimitiveAsset] = {}
def load(self, path: str | Path) -> SmartPrimitiveAsset:
"""Return the Smart Primitive asset for path, preparing it on first use."""
resolved_path = Path(path).expanduser().resolve()
asset = self._assets.get(resolved_path)
if asset is None:
with wp.ScopedDevice(self.device):
authoring, _, _ = load_smart_primitive_document(
self.clip_registry,
self.reference_pose,
self.model_to_runtime,
resolved_path,
)
asset = SmartPrimitiveAsset(
build_runtime_smart_primitive_data(authoring), device=self.device,
)
self._assets[resolved_path] = asset
return asset
def instantiate(
self,
path: str | Path,
smart_primitive_type: type[SmartPrimitive] = SmartPrimitive,
) -> SmartPrimitive:
"""Return a new Smart Primitive instance of smart_primitive_type that shares the asset at path."""
return smart_primitive_type(self.load(path))