Skip to content

[SIGGRAPH Asia 2026] UniMate: One Unified Model to Animate Diverse Skeletons

Python
12.5 KB
Raw Blame
"""Build conditioning batches for sampling / visualization / inference.

``create_sample_condition`` selects reference clips from the loaded
``MotionDataset`` (see the mode priority in its docstring), reruns the same
crop → condition-extraction → normalization → padding pipeline used for
training samples, and collates the result with ``mixture_batch_collate`` so
the generative model sees exactly the training-time cond format.
"""

import random
from typing import Dict, List, Optional, Tuple

import numpy as np
import torch

from Quaternions import Quaternions

from unimate.configs.schema import MainConfig
from unimate.dataset.transforms import (
    apply_cropping, extract_conditions, apply_normalization,
    apply_padding, build_parent_features,
)
from unimate.dataset.mixture.collate import mixture_batch_collate
from unimate.utils.motion_utils import realign_unimate_clip
from unimate.utils.topology_utils import compute_edge_indexs


def _resolve_dataset(data):
    """Accept either a Dataset or DataLoader, return the underlying Dataset."""
    if isinstance(data, torch.utils.data.DataLoader):
        return data.dataset
    if isinstance(data, torch.utils.data.Dataset):
        return data
    raise ValueError("data must be either a Dataset or DataLoader instance.")


def create_sample_condition(
    config: MainConfig,
    data,
    num_samples: int = 3,
    object_types: Optional[List[str]] = None,
    test_case_captions: Optional[List[Tuple]] = None,
    gt_start_frame: Optional[int] = None,
):
    """Build a conditioning batch for sampling / visualization / inference.

    Modes (in priority order):
      1. ``test_case_captions`` — list of
         ``(object_type, caption, caption_enc[, clip_name])``, where
         ``caption_enc`` is ``{'caption_emb': (D,)[, 'caption_tokens':
         (T, D)]}``. One reference
         clip per entry provides the target skeleton; the clip's own caption
         is replaced by the user-supplied text + embedding. If a 4th element
         ``clip_name`` is provided, that exact clip is used (resolved against
         train then eval motion dicts); otherwise a random clip is picked.
         The explicit-clip variant is required for motion in-betweening so
         the generated batch lines up with the GT being clamped at sample
         time. Used by ``unimate/inference/sample.py``.
      2. ``object_types`` — pick one train clip and/or one eval clip per
         listed object_type (each emitted only when its map has clips).
      3. default — auto-sample ``num_samples`` types per dataset, routed by
         split mode so visualization always covers both train and eval:
           * ``'object_type'`` (truebones, objaverse): pick ``num_samples``
             train types + ``num_samples`` eval types, one random clip each.
           * ``'clip'`` (mixamo): pick ``num_samples`` types and emit one
             train + one eval clip per picked type (eval skipped when
             ``eval_map`` is empty for that type).

    Args:
        gt_start_frame: when set, the GT clip is cropped starting at this
            absolute frame index instead of the default random ('tpos') /
            zero ('first_frame') start. If fewer than
            ``max_motion_length`` frames remain, the crop is trimmed and the
            tail is zero-padded. ``cond["motion_length"]`` does NOT report
            that: it is deliberately pinned to ``max_motion_length`` so the
            model generates (and attends over) the whole padded window. The
            caller tracks the real length separately — see
            ``sample._gt_valid_lengths`` — and trims outputs at save time.
    """
    dataset = _resolve_dataset(data)
    md = dataset.motion_dataset

    topology_cond_type = config.dataset.topology_condition_type

    train_map = md.train_object_motions_map
    eval_map = md.eval_object_motions_map
    split_mode = md.split_mode_by_dataset

    def _pick(pool, n):
        if not pool:
            return []
        return random.sample(pool, n) if len(pool) >= n else random.choices(pool, k=n)

    # selection: (object_type, clip_name, split_tag) tuples consumed by the
    # batch-building loop below. caption overrides keyed by *selection index*
    # so multiple test cases can share the same reference clip without
    # overwriting each other's caption.
    selection = []
    caption_override_by_idx: Dict[int, Tuple[str, np.ndarray]] = {}

    if test_case_captions is not None:
        # Reference clip provides topology only; the user override supplies
        # the caption. Prefer a train clip; fall back to eval for held-out
        # types. When a 4th tuple element is given, use that exact clip
        # (in-betweening / motion editing pin a specific GT motion).
        for entry in test_case_captions:
            if len(entry) == 4:
                obj_type, caption, caption_enc, explicit_clip = entry
            else:
                obj_type, caption, caption_enc = entry
                explicit_clip = None

            train_clips = train_map.get(obj_type, [])
            eval_clips = eval_map.get(obj_type, [])

            if explicit_clip is not None:
                if explicit_clip in train_clips:
                    clip_name, tag = explicit_clip, 'train'
                elif explicit_clip in eval_clips:
                    clip_name, tag = explicit_clip, 'eval'
                else:
                    raise ValueError(
                        f"clip_name={explicit_clip!r} not found for "
                        f"object_type={obj_type!r} in train or eval maps."
                    )
            else:
                primary = train_clips or eval_clips
                tag = 'train' if train_clips else 'eval'
                if not primary:
                    continue
                clip_name = random.choice(primary)

            caption_override_by_idx[len(selection)] = (caption, caption_enc)
            selection.append((obj_type, clip_name, tag))
    elif object_types is not None:
        # One train + one eval clip per requested type when both pools have
        # entries. With ``test_split_ratio == 0`` or a fully held-out type,
        # emit just one — no synthetic duplicate.
        for object_type in object_types:
            train_clips = train_map.get(object_type, [])
            eval_clips = eval_map.get(object_type, [])
            if not (train_clips or eval_clips):
                continue
            if train_clips:
                selection.append((object_type, random.choice(train_clips), 'train'))
            if eval_clips:
                selection.append((object_type, random.choice(eval_clips), 'eval'))
    else:
        for dataset_type, obj_counts in md.dataset_object_count.items():
            mode = split_mode.get(dataset_type, 'clip')

            if mode == 'object_type':
                train_types = [t for t in obj_counts if train_map.get(t)]
                eval_types = [t for t in obj_counts if eval_map.get(t)]
                for t in _pick(train_types, num_samples):
                    selection.append((t, random.choice(train_map[t]), 'train'))
                for t in _pick(eval_types, num_samples):
                    selection.append((t, random.choice(eval_map[t]), 'eval'))
            else:
                # 'clip' mode: emit train (+eval when present) per picked type.
                # With test_split_ratio==0 (eval_map empty) the train pool is
                # not sampled twice just to hit a sample count.
                all_types = list(obj_counts.keys())
                for t in _pick(all_types, num_samples):
                    train_clips = train_map.get(t, [])
                    eval_clips = eval_map.get(t, [])
                    if not (train_clips or eval_clips):
                        continue
                    if train_clips:
                        selection.append((t, random.choice(train_clips), 'train'))
                    if eval_clips:
                        selection.append((t, random.choice(eval_clips), 'eval'))

    batches = []
    for sel_idx, (object_type, selected_motion_name, _split_tag) in enumerate(selection):
        condition = md.cond_dict[object_type].copy()
        source = md.train_motion_dict if _split_tag == 'train' else md.eval_motion_dict
        motion_data = source[selected_motion_name].copy()
        assert motion_data['object_type'] == object_type

        raw_motion = motion_data['motion']  # (F, J, D)
        parents = motion_data['parents']    # (J,)
        tpos = np.array(condition['tpos_first_frame'])
        # Grounded like the training T-pose (Mixture._precompute_object_type_meta).
        if config.dataset.ground_motion_height:
            tpos[..., 1] -= tpos[..., 1].min()

        max_motion_length = config.dataset.max_motion_length
        raw_motion, start_idx = apply_cropping(
            raw_motion, topology_cond_type, max_motion_length,
            start_idx=gt_start_frame,
        )

        # Re-align so frame-0 facing is identity (matches training-time
        # invariant assumed by the feature pipeline).
        if config.dataset.realign_feature and start_idx > 0:
            raw_motion = realign_unimate_clip(raw_motion, parents)

        # Pad tpos (J, 3) to motion feature dim (J, 12): identity 6D
        # rotation + zero velocity, matching the layout of motion features.
        feature_len = config.dataset.feature_len
        if tpos.shape[-1] < feature_len:
            J = tpos.shape[0]
            identity_6d = Quaternions.id(1).rotation_matrix(cont6d=True)[0]
            pad = np.zeros((J, feature_len - 3))
            pad[:, :6] = identity_6d
            tpos = np.concatenate([tpos, pad], axis=-1)

        conds = extract_conditions(raw_motion, tpos, topology_cond_type)
        motion = conds['motion']
        tpos_first_frame = conds['tpos_first_frame']
        njoints, nfeats = tpos_first_frame.shape

        # Dataset-level normalization stats (root vs. local split).
        dataset_type = motion_data['dataset_type']
        stats = md.dataset_stats[dataset_type]
        mean = np.zeros((njoints, nfeats))
        std = np.zeros((njoints, nfeats))
        mean[0, :], std[0, :] = stats['mean_root'], stats['std_root']
        mean[1:, :], std[1:, :] = stats['mean_local'], stats['std_local']

        motion = apply_normalization(motion, mean, std)
        tpos_first_frame = apply_normalization(tpos_first_frame, mean, std)

        # Pad to fixed length so the generator always produces same-shape outputs.
        motion, _ = apply_padding(motion, max_motion_length)

        parent_feats = build_parent_features(tpos_first_frame, parents)
        # Reuse the clip's stored eigenvectors (loaded via ``create_condition``)
        # rather than recomputing: eigendecomposition signs are arbitrary, and
        # the stored copy is the sign-pinned version the model saw at training
        # time (matters for the non-sign-invariant WIRE spectral backend).
        spectral_feats = motion_data['spectral_feats']

        batch = {
            'motion': motion,
            'max_motion_length': config.dataset.max_motion_length,
            # Always the full window, even for a short crop: the model is
            # asked to generate every frame. See the gt_start_frame note above.
            'motion_length': max_motion_length,
            'max_joints': config.dataset.max_joints,
            'parents': parents,
            'edge_indexs': compute_edge_indexs(parents),
            'tpos_first_frame': tpos_first_frame,
            'tpos_first_frame_parents': parent_feats['tpos_first_frame_parents'],
            'offsets': motion_data['offsets'],
            'joint_graph_dist': motion_data['joint_graph_dist'],
            'joint_relations': motion_data['joint_relations'],
            'joint_depths': motion_data['joint_depths'],
            'spectral_feats': spectral_feats,
            'joint_names_emb': motion_data['joint_names_emb'],
            'object_type': object_type,
            'start_idx': start_idx,
            'mean': mean,
            'std': std,
            'split_tag': _split_tag,
        }

        if sel_idx in caption_override_by_idx:
            caption, caption_enc = caption_override_by_idx[sel_idx]
            batch['caption'] = caption
            # caption_enc is {'caption_emb': (D,)[, 'caption_tokens': (T, D)]};
            # a caller that only produced the pooled vector still works.
            batch.update(caption_enc)
        else:
            for key in ('caption', 'caption_emb', 'caption_tokens'):
                if key in motion_data:
                    batch[key] = motion_data[key]

        batches.append(batch)

    motion, cond = mixture_batch_collate(batches)
    return motion, cond