Skip to content

Official implementation of MotionBricks: Scalable Real-Time Motions with Modular Latent Generative Model and Smart Primitives (SIGGRAPH 2026)

Python
26.4 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""G1 root and pose predictors."""

from __future__ import annotations

from functools import cached_property
from typing import Dict, Optional

import numpy as np
import torch
import torch as t
from torch import nn

from ..features import (
    convert_sparse_cond_to_dense_cond_if_needed,
    extract_feature_from_motion_rep,
)
from ..layers import FCBlock as mlp, PositionEmbedding, DoubleCondDecoder


class root_backbone_network(nn.Module):
    """Transformer that predicts the root trajectory and the token count for G1."""

    def __init__(self, args: Dict, motion_rep: nn.Module):
        super().__init__()
        self._args = args
        self.motion_rep = motion_rep
        self.global_motion_rep = motion_rep.dual_rep.global_motion_rep
        self.local_motion_rep = motion_rep.dual_rep.local_motion_rep
        self.IS_MODEL_TOKENIZED = False
        self._build_backbone()
        self._build_input_embeddings()

    def _build_backbone(self):
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=self._args["n_embd"], nhead=self._args["n_head"], batch_first=True
        )
        self._shared_transformer_model = nn.TransformerEncoder(
            encoder_layer, num_layers=self._args["n_layers_shared"]
        )
        if self._args["n_layers_root_token"] > 0:
            self._root_token_transformer_model = nn.TransformerEncoder(
                encoder_layer, num_layers=self._args["n_layers_root_token"]
            )
        else:
            self._root_token_transformer_model = None
        self._args["local_root_dim"] = extract_feature_from_motion_rep(
            t.zeros([1, 1, 1000]),
            self.local_motion_rep,
            self._args["local_root_feature"],
        ).shape[2]
        self._args["global_root_dim"] = extract_feature_from_motion_rep(
            t.zeros([1, 1, 1000]),
            self.global_motion_rep,
            self._args["global_root_feature"],
        ).shape[2]
        self._args["local_pose_dim"] = extract_feature_from_motion_rep(
            t.zeros([1, 1, 1000]),
            self.local_motion_rep,
            self._args["local_pose_feature"],
        ).shape[2]
        self._conv_output = DoubleCondDecoder(
            self._args["global_root_dim"],
            self._args["n_embd"],
            down_t=self._args["down_t"],
            width=self._args["width"],
            depth=self._args["depth"],
            dilation_growth_rate=self._args["dilation_growth_rate"],
            activation=self._args["activation"],
            norm=self._args["norm"],
            target_cond_dim=self._args["global_root_dim"],
            external_cond_dim=self._args["n_embd"],
        )

    def _build_input_embeddings(self):
        num_frames_per_token = self.get_num_frames_per_token()
        self._proj_local_pose = nn.Linear(
            self._args["local_pose_dim"], self._args["pose_feat_dim"]
        )
        self._proj_local_root_value = nn.Linear(
            self.args["local_root_dim"], self._args["local_root_feat_dim"]
        )
        self._proj_global_root_value = nn.Linear(
            self.args["global_root_dim"], self._args["global_root_feat_dim"]
        )
        self._no_local_pose_emb = nn.Parameter(
            torch.randn([self._args["pose_feat_dim"]])
        )
        self._no_local_root_emb = nn.Parameter(
            torch.randn([self._args["local_root_feat_dim"]])
        )
        self._no_global_root_emb = nn.Parameter(
            torch.randn([self._args["global_root_feat_dim"]])
        )
        self._conv_no_frame_emb = nn.Parameter(torch.randn([self._args["n_embd"]]))
        proj_input_dim = (
            self._args["pose_feat_dim"]
            + self._args["local_root_feat_dim"]
            + self._args["global_root_feat_dim"]
        )
        self._proj_start_input = mlp(
            num_layers=self._args["input_feat_mlp_num_layers"],
            layer_width=self._args["n_embd"],
            size_in=proj_input_dim,
            size_out=self._args["n_embd"],
        )
        self._proj_end_input = mlp(
            num_layers=self._args["input_feat_mlp_num_layers"],
            layer_width=self._args["n_embd"],
            size_in=proj_input_dim,
            size_out=self._args["n_embd"],
        )
        self._input_position_emb = t.nn.Embedding(
            num_frames_per_token * 2, self._args["n_embd"]
        )
        self._proj_input_num_tokens = t.nn.Embedding(
            self._args["max_tokens"] - self._args["min_tokens"] + 1 + 1 + 1,
            self._args["n_embd"],
        )
        self._position_emb = PositionEmbedding(
            seq_length=self._args["max_tokens"], dim=self._args["n_embd"]
        )
        self._proj_num_token_output_logit = t.nn.Linear(
            self._args["n_embd"],
            self._args["max_tokens"] - self._args["min_tokens"] + 1 + 1,
        )
        if self.ACCEPT_TEXT_EMB_INPUT:
            self._proj_text_embeddings = mlp(
                num_layers=self._args["input_feat_mlp_num_layers"],
                layer_width=self._args["n_embd"],
                size_in=self._args["text_emb_dim"],
                size_out=self._args["n_embd"],
            )
        if (
            self.USE_HARD_NUM_TOKEN_EMB_FOR_ROOT_PREDICTION
            and self._root_token_transformer_model is not None
        ):
            self._middle_token_emb = t.nn.Embedding(
                self._args["max_tokens"] - self._args["min_tokens"] + 1 + 1,
                self._args["n_embd"],
            )

    def forward(
        self,
        global_root_values: t.Tensor,
        has_global_root_values: t.Tensor,
        local_root_values: t.Tensor,
        has_local_root_values: t.Tensor,
        poses: t.Tensor,
        has_poses: t.Tensor,
        num_tokens: t.Tensor,
        text_embeddings: Optional[t.Tensor] = None,
        has_text_embeddings: Optional[t.Tensor] = None,
        groundtruth_num_tokens: t.Tensor = None,
        allowed_pred_num_tokens: t.Tensor = None,
        config: dict = {},
    ):
        batch_size, device = (poses.shape[0], poses.device)
        num_frames_per_token = self.get_num_frames_per_token()
        local_pose_emb = self._proj_local_pose(poses)
        local_root_emb = self._proj_local_root_value(local_root_values)
        global_root_emb = self._proj_global_root_value(global_root_values)
        local_pose_emb = local_pose_emb * has_poses[
            :, :, None
        ].float() + self._no_local_pose_emb[None, None, :] * (
            1 - has_poses[:, :, None].float()
        )
        local_root_emb = local_root_emb * has_local_root_values[
            :, :, None
        ].float() + self._no_local_root_emb[None, None, :] * (
            1 - has_local_root_values[:, :, None].float()
        )
        global_root_emb = global_root_emb * has_global_root_values[
            :, :, None
        ].float() + self._no_global_root_emb[None, None, :] * (
            1 - has_global_root_values[:, :, None].float()
        )
        start_frame_emb = torch.cat(
            [
                local_pose_emb[:, :num_frames_per_token, :],
                local_root_emb[:, :num_frames_per_token, :],
                global_root_emb[:, :num_frames_per_token, :],
            ],
            dim=-1,
        )
        start_frame_emb = self._proj_start_input(start_frame_emb)
        end_frame_emb = torch.cat(
            [
                local_pose_emb[:, -num_frames_per_token:, :],
                local_root_emb[:, -num_frames_per_token:, :],
                global_root_emb[:, -num_frames_per_token:, :],
            ],
            dim=-1,
        )
        end_frame_emb = self._proj_end_input(end_frame_emb)
        frame_emb = torch.cat([start_frame_emb, end_frame_emb], dim=1)
        positioned_frame_emb = frame_emb + self._input_position_emb.weight[None, :, :]
        first_stage_time_emb = self._proj_input_num_tokens(
            num_tokens - self._args["min_tokens"]
        )
        first_stage_time_emb = first_stage_time_emb.view(
            [batch_size, 1, self._args["n_embd"]]
        )
        first_stage_input_emb = t.concat(
            [first_stage_time_emb, positioned_frame_emb], dim=1
        )
        if self.ACCEPT_TEXT_EMB_INPUT and text_embeddings is not None:
            text_embeddings = self._proj_text_embeddings(text_embeddings)[:, None, :]
            first_stage_input_emb = t.concat(
                [text_embeddings, first_stage_input_emb], dim=1
            )
            first_stage_mask = t.zeros(
                [batch_size, 1 + 1 + num_frames_per_token * 2], device=device
            ).bool()
            first_stage_mask[:, :1] = ~has_text_embeddings.bool()
        else:
            first_stage_mask = None
        first_stage_output_emb = self._shared_transformer_model(
            first_stage_input_emb, src_key_padding_mask=first_stage_mask
        )
        num_token_logits = self._proj_num_token_output_logit(
            first_stage_output_emb[:, self.TRANSFORMER_TIME_LOGIT_ID, :]
        )
        assert self.USE_HARD_NUM_TOKEN_EMB_FOR_ROOT_PREDICTION, (
            "Only support hard num token emb."
        )
        if groundtruth_num_tokens is not None:
            chosen_token = (
                groundtruth_num_tokens.view([batch_size]) - self._args["min_tokens"]
            )
        else:
            assert not self.training, "groundtruth_num_tokens is required in training."
            if not config.get("allow_pred_out_of_reach_num_tokens", True):
                num_token_logits = num_token_logits.clone()
                num_token_logits[
                    :, self.OUT_OF_REACH_NUM_TOKENS - self._args["min_tokens"]
                ] = -t.inf
            if allowed_pred_num_tokens is not None:
                num_time_tokens = (
                    self._args["max_tokens"] - self._args["min_tokens"] + 1
                )
                modified_num_token_logits = t.where(
                    allowed_pred_num_tokens == 1,
                    num_token_logits[:, :num_time_tokens],
                    t.full([batch_size, num_time_tokens], -t.inf).to(device),
                )
                num_token_logits = t.cat(
                    [modified_num_token_logits, num_token_logits[:, num_time_tokens:]],
                    dim=-1,
                )
            chosen_token = torch.argmax(num_token_logits, dim=-1).int()
            chosen_token = t.where(
                num_tokens.view([batch_size]) == self.MASKED_NUM_TOKENS,
                chosen_token,
                num_tokens.view([batch_size]) - self._args["min_tokens"],
            )
        position_ids = t.arange(self._args["max_tokens"]).to(device=device)
        position_emb = self._position_emb.embed[position_ids].view(
            [1, self._args["max_tokens"], self._args["n_embd"]]
        )
        position_emb = position_emb.expand([batch_size, -1, -1])
        token_mask = (
            t.arange(self._args["max_tokens"]).view([1, -1]).to(device)
            < chosen_token.view([-1, 1]) + self._args["min_tokens"]
        )
        if self._root_token_transformer_model is None:
            second_stage_output_emb = position_emb
        else:
            second_stage_token_emb = self._middle_token_emb(chosen_token)
            second_stage_input_emb = t.concat(
                [
                    second_stage_token_emb[:, None, :],
                    positioned_frame_emb,
                    position_emb,
                ],
                dim=1,
            )
            second_stage_mask = t.cat(
                [
                    t.zeros(
                        [batch_size, 1 + num_frames_per_token * 2], device=device
                    ).bool(),
                    ~token_mask,
                ],
                dim=1,
            )
            second_stage_output_emb = self._root_token_transformer_model(
                second_stage_input_emb, src_key_padding_mask=second_stage_mask
            )
            second_stage_output_emb = second_stage_output_emb[
                :, -self._args["max_tokens"] :
            ]
        pred_num_tokens = chosen_token + self._args["min_tokens"]
        batch = {}
        keys = ["frame", "global_root"]
        num_total_frames = self._args["max_tokens"] * num_frames_per_token
        batch["has_frame"] = t.logical_or(
            t.logical_or(has_poses, has_global_root_values), has_local_root_values
        )
        batch["frame"] = t.where(
            batch["has_frame"][:, :, None],
            frame_emb,
            self._conv_no_frame_emb[None, None, :],
        )
        batch["dense_frame"] = t.cat(
            [
                batch["frame"][:, :num_frames_per_token],
                self._conv_no_frame_emb[None, None, :].repeat(
                    [batch_size, num_total_frames - num_frames_per_token, 1]
                ),
            ],
            dim=1,
        )
        batch["global_root"], batch["has_global_root"] = (
            global_root_values,
            has_global_root_values,
        )
        batch["dense_global_root"] = t.cat(
            [
                batch["global_root"][:, :num_frames_per_token],
                t.zeros(
                    [
                        batch_size,
                        num_total_frames - num_frames_per_token,
                        self._args["global_root_dim"],
                    ],
                    device=device,
                ),
            ],
            dim=1,
        )
        for key in keys:
            batch["dense_has_" + key] = t.cat(
                [
                    batch["has_" + key][:, :num_frames_per_token],
                    t.zeros(
                        [batch_size, num_total_frames - num_frames_per_token],
                        device=device,
                    ).bool(),
                ],
                dim=1,
            )
            for i in range(num_frames_per_token):
                offsets = (
                    pred_num_tokens[:, None] * num_frames_per_token
                    - num_frames_per_token
                    + i
                ).long()
                batch["dense_" + key] = batch["dense_" + key].scatter(
                    1,
                    offsets[:, :, None].repeat([1, 1, batch[key].shape[-1]]),
                    batch[key][:, -num_frames_per_token + i][:, None, :],
                )
                batch["dense_has_" + key] = batch["dense_has_" + key].scatter(
                    1,
                    offsets[:, :],
                    batch["has_" + key][:, -num_frames_per_token + i][:, None],
                )
        pred_global_root_values = self._conv_output(
            second_stage_output_emb.transpose(1, 2),
            external_cond=batch["dense_frame"],
            target_cond=batch["dense_global_root"],
            has_target_cond=batch["dense_has_global_root"],
            token_mask=token_mask,
        ).transpose(1, 2)
        return {
            "num_token_logits": num_token_logits,
            "pred_num_tokens": pred_num_tokens,
            "pred_global_root_values": pred_global_root_values,
        }

    def get_num_frames_per_token(self):
        """Return the frames per motion token, 2 ** down_t."""
        return 2 ** self._args["down_t"]

    @property
    def args(self):
        """Network settings from the checkpoint."""
        return self._args

    @property
    def OUT_OF_REACH_NUM_TOKENS(self):
        """Token count class for a target that is out of reach."""
        return self._args["max_tokens"] + 1

    @property
    def MASKED_NUM_TOKENS(self):
        """Token count class for a masked token count."""
        return self._args["max_tokens"] + 2

    @cached_property
    def ACCEPT_TEXT_EMB_INPUT(self):
        """Whether the network takes text embeddings."""
        return True if self._args.get("text_embeddings", None) is not None else False

    @property
    def TRANSFORMER_TIME_LOGIT_ID(self):
        """Index of the token count output in the transformer sequence."""
        return 1 if self.ACCEPT_TEXT_EMB_INPUT else 0

    @cached_property
    def USE_HARD_NUM_TOKEN_EMB_FOR_ROOT_PREDICTION(self):
        """Whether root prediction embeds the exact token count."""
        return self._args.get("use_hard_num_token_emb_for_root_prediction", False)


class pose_backbone_network(nn.Module):
    """Transformer that predicts the pose tokens for G1."""

    def __init__(self, motion_rep: nn.Module, args: Dict = None):
        super().__init__()
        self.motion_rep = motion_rep
        self.global_motion_rep = motion_rep.dual_rep.global_motion_rep
        self.local_motion_rep = motion_rep.dual_rep.local_motion_rep
        self._args = args
        self._build_transformer_backbone()
        self._build_input_embeddings_projections()
        self.register_buffer("initted", torch.Tensor([False]).to(dtype=t.bool))

    def _build_transformer_backbone(self):
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=self._args["n_embd"], nhead=self._args["n_head"], batch_first=True
        )
        self._transformer_model = nn.TransformerEncoder(
            encoder_layer, num_layers=self._args["n_layers"]
        )

    def _build_input_embeddings_projections(self):
        num_pose_heads, _ = self.get_num_heads()
        num_pose_tokens, _ = self.get_num_codes(include_aug_tokens=False)
        if self._args["pose_vqvae"].get("has_codebook", True):
            pose_token_dim = self._args["pose_vqvae"]["code_dim"] // num_pose_heads
        else:
            pose_token_dim = self._args["n_embd"] // num_pose_heads
        self._pose_token_emb = t.nn.Embedding(
            num_pose_heads * self.NUM_WITH_AUG_POSE_TOKENS, pose_token_dim
        )
        self._proj_pose_token_emb = mlp(
            num_layers=self._args["pose_token_mlp_num_layers"],
            layer_width=self._args["pose_feat_width"],
            size_in=num_pose_heads * pose_token_dim,
            size_out=self._args["pose_feat_width"],
        )
        num_frames_per_token = self.get_num_frames_per_token()
        if self._args["cond_root_feature_is_from_motion_rep"] == "local":
            self._args["local_root_dim"] = extract_feature_from_motion_rep(
                t.zeros([1, 1, 1000]),
                self.local_motion_rep,
                self._args["cond_root_feature"],
            ).shape[2]
        else:
            self._args["local_root_dim"] = extract_feature_from_motion_rep(
                t.zeros([1, 1, 1000]),
                self.global_motion_rep,
                self._args["cond_root_feature"],
            ).shape[2]
        self._proj_local_root_values = t.nn.Linear(
            self._args["local_root_dim"] * num_frames_per_token,
            self._args["root_feat_width"],
        )
        self._args["local_pose_dim"] = extract_feature_from_motion_rep(
            t.zeros([1, 1, 1000]),
            self.local_motion_rep,
            self._args["local_pose_feature"],
        ).shape[2]
        self._proj_local_pose = t.nn.Linear(
            self._args["local_pose_dim"],
            self._args["pose_feat_width"] // num_frames_per_token,
        )
        assert self._args["pose_feat_width"] % self.get_num_frames_per_token() == 0, (
            "pose_feat_width needs to be divisible by num_frames_per_token"
        )
        self._proj_num_valid_positions = t.nn.Embedding(
            self._args["max_tokens"] - self._args["min_tokens"] + 1,
            self._args["token_length_feat_width"],
        )
        self._proj_input = t.nn.Sequential(
            t.nn.Linear(
                self._args["pose_feat_width"]
                + self._args["root_feat_width"]
                + self._args["token_length_feat_width"],
                self._args["n_embd"],
            ),
            t.nn.ReLU(),
        )
        self._position_emb = PositionEmbedding(
            seq_length=self._args["max_tokens"], dim=self._args["n_embd"]
        )
        self._proj_pose_output_logit = t.nn.Linear(
            self._args["n_embd"], num_pose_heads * num_pose_tokens
        )
        if self.ACCEPT_TEXT_EMB_INPUT:
            self._proj_text_embeddings = mlp(
                num_layers=self._args["pose_token_mlp_num_layers"],
                layer_width=self._args["n_embd"],
                size_in=self._args["text_emb_dim"],
                size_out=self._args["n_embd"],
            )

    def forward(
        self,
        pose_tokens: t.Tensor,
        local_root_values: t.Tensor,
        pose_cond: t.Tensor,
        has_pose_cond: t.Tensor,
        num_tokens: t.Tensor,
        text_embeddings: t.Tensor = None,
        has_text_embeddings: t.Tensor = None,
    ):
        batch_size, num_positions, num_pose_heads = pose_tokens.shape
        device = pose_tokens.device
        num_frames_per_token = self.get_num_frames_per_token()
        local_root_values = local_root_values.reshape(
            [
                batch_size,
                num_positions,
                num_frames_per_token * self._args["local_root_dim"],
            ]
        )
        root_embedding = self._proj_local_root_values(local_root_values)
        dense_pose_cond, dense_has_pose_cond = (
            convert_sparse_cond_to_dense_cond_if_needed(
                pose_cond, has_pose_cond, num_positions * num_frames_per_token
            )
        )
        pose_cond_embedding = self._proj_local_pose(dense_pose_cond)
        pose_cond_embedding = pose_cond_embedding.view(
            [
                batch_size,
                num_positions * num_frames_per_token,
                self._args["pose_feat_width"] // num_frames_per_token,
            ]
        )
        pose_token_id_offsets = (
            torch.arange(num_pose_heads).view([1, 1, num_pose_heads]).to(device=device)
            * self.NUM_WITH_AUG_POSE_TOKENS
        )
        pose_tokens = pose_tokens + pose_token_id_offsets
        pose_token_embedding = self._pose_token_emb(pose_tokens)
        pose_token_embedding = self._proj_pose_token_emb(
            pose_token_embedding.view([batch_size, num_positions, -1])
        )
        pose_token_embedding = pose_token_embedding.view(
            [
                batch_size,
                num_positions * num_frames_per_token,
                self._args["pose_feat_width"] // num_frames_per_token,
            ]
        )
        dense_has_pose_cond = dense_has_pose_cond[:, :, None].float()
        pose_embedding = (
            pose_cond_embedding * dense_has_pose_cond
            + pose_token_embedding * (1 - dense_has_pose_cond)
        )
        pose_embedding = pose_embedding.view(
            [batch_size, num_positions, self._args["pose_feat_width"]]
        )
        num_token_embedding = self._proj_num_valid_positions(
            num_tokens.reshape([batch_size, 1]) - self._args["min_tokens"]
        ).expand([-1, num_positions, -1])
        position_ids = t.arange(num_positions).to(device=device)
        position_emb = self._position_emb.embed[position_ids].view(
            [1, num_positions, self._args["n_embd"]]
        )
        input_embeddings = (
            self._proj_input(
                t.cat([pose_embedding, root_embedding, num_token_embedding], dim=-1)
            )
            + position_emb
        )
        token_mask = t.arange(num_positions).to(device) < num_tokens.view(
            [batch_size, -1]
        )
        if self.ACCEPT_TEXT_EMB_INPUT and text_embeddings is not None:
            text_embeddings = self._proj_text_embeddings(text_embeddings)[:, None, :]
            input_embeddings = t.concat([text_embeddings, input_embeddings], dim=1)
            mask = t.zeros([batch_size, 1 + num_positions], device=device).bool()
            mask[:, :1] = ~has_text_embeddings.bool()
            mask[:, 1:] = ~token_mask.bool()
        else:
            mask = ~token_mask.bool()
        output = self._transformer_model(input_embeddings, src_key_padding_mask=mask)[
            :, -num_positions:
        ]
        pred_logits = self._proj_pose_output_logit(output).reshape(
            [batch_size, num_positions, num_pose_heads, -1]
        )
        return {"pose_logits": pred_logits}

    def get_num_codes(self, include_aug_tokens: bool):
        """Return the code counts of the pose and root codebooks.

        With include_aug_tokens, the counts include the extra tokens, such as the mask
        token.
        """
        num_codes_per_head = {}
        for vqvae in ["pose_vqvae", "root_vqvae"]:
            nb_code = self._args[vqvae]["nb_code"]
            len_nb_code = len(np.array(self._args[vqvae]["nb_code"]).reshape([-1]))
            if len_nb_code > 1:
                assert np.all(np.array(nb_code) == nb_code[0]), (
                    "fsq config should have the same number of codes for each head"
                )
                num_codes_per_head[vqvae] = nb_code[0]
            else:
                num_heads = self._args[vqvae]["num_heads"]
                code_dim = self._args[vqvae]["code_dim"]
                if self._args[vqvae].get("has_codebook", True):
                    assert code_dim % num_heads == 0, (
                        "code dim cannot be divided by the number of heads."
                    )
                    num_codes_per_head[vqvae] = int(
                        round(2 ** (np.log2(nb_code) / num_heads))
                    )
                    assert num_codes_per_head[vqvae] ** num_heads == nb_code, (
                        "the specified number of code is not compatible with the number of heads."
                    )
                else:
                    num_codes_per_head[vqvae] = nb_code
        if include_aug_tokens:
            return (
                num_codes_per_head["pose_vqvae"] + 1,
                num_codes_per_head["root_vqvae"] + 1,
            )
        else:
            return (num_codes_per_head["pose_vqvae"], num_codes_per_head["root_vqvae"])

    def get_num_heads(self):
        """Return the head counts of the pose and root codebooks."""
        return (
            self._args["pose_vqvae"]["num_heads"],
            self._args["root_vqvae"]["num_heads"],
        )

    def get_num_frames_per_token(self):
        """Return the frames per motion token, 2 ** down_t."""
        return 2 ** self._args["down_t"]

    @property
    def NUM_WITH_AUG_POSE_TOKENS(self):
        """Number of pose token values, including the extra tokens."""
        return self.get_num_codes(include_aug_tokens=True)[0]

    @property
    def POSE_MASK_ID(self):
        """Token value that marks a masked pose token."""
        return self.get_num_codes(include_aug_tokens=True)[0] - 1

    @cached_property
    def ACCEPT_TEXT_EMB_INPUT(self):
        """Whether the network takes text embeddings."""
        return True if self._args.get("text_embeddings", None) is not None else False