Skip to content

[SIGGRAPH Asia 2026] UniMate: One Unified Model to Animate Diverse Skeletons

Python
20.3 KB
Raw Blame
"""Drive a rigged character with a canonicalized motion clip and save GLB + FBX.

This is the inference-time path: it takes a motion in the *feature* format
written by the feature-extraction stage (``global_positions`` /
``local_rotations`` / ``root_facing_quat``) — or generated by the model in
that same format — together with the dataset's ``cond.npy`` (skeleton
topology, joint names, scale) and drives a rigged asset of that object type
with it (resolution below).

Pipeline:
  1. Load the cond entry for the requested ``--dataset_type`` and the motion
     file (NPZ, or the ``.npy`` motion features written by the model's sampler).
  2. Reconcile the character armature against the animation's bone list
     (see :func:`data_process.utils.blender_rig.sync_armature_bones`).
  3. Compute per-bone keyframes, re-expressed in the rig's bone axes when they
     differ from the motion's
     (:func:`data_process.utils.skeleton.motion_axis_alignment`), and bind
     them as a Blender Action.
  4. Export the animated character as ``.glb`` and ``.fbx``.

Usage (Blender headless):
    blender -b -P data_process/mesh_animation/animate_motion.py -- \\
        --dataset_type objaverse \\
        --anim_path dataset/features/objaverse/motions/<clip>.npz \\
        --output_dir outputs/animated

``--char_path`` is auto-resolved by ``--asset``:

- ``canonical`` (default): the canonical asset of the object type
  (``dataset/canonical_assets/<ds>/<type>.glb``, feature_extraction/
  canonical_assets.py; mixamo: of ``--character``, default ``Michelle``). An
  object type without one (e.g. a skeleton the bake refused) falls back to
  the ``export`` resolution with a warning; ``--canonical`` makes it an error.
- ``export``: the processed GLB of the object type
  (``dataset/export/<ds>/rigs/<type>.glb``, written by the export stage's
  ``--save_glb`` / ``--glb_only``: the asset in its rest pose on the exported
  skeleton) when it exists, else the raw asset (truebones / objaverse /
  general); for mixamo the processed character ``--character`` (default
  ``Michelle``, else the raw Y Bot FBX).

A canonical character is detected from its ``canonical_joint_order``
property, and the motion then stays in the canonical frame and scale (any
other character gets it divided by the cond's ``scale_factor``, back to the
export's units).
"""

import argparse
import os
import sys

import numpy as np
from loguru import logger

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))

from Animation import Animation, Quaternions  # noqa: E402

from data_process.mesh_animation.common import (  # noqa: E402
    DEFAULT_MIXAMO_CHARACTER,
    check_canonical_order,
    drive_and_export,
    find_processed_asset,
    is_canonical_asset,
    load_character,
    npz_scalar,
    parse_blender_argv,
    processed_asset_path,
    resolve_processed_asset,
)
from data_process.utils.blender_export import asset_name, list_asset_files  # noqa: E402
from data_process.utils.blender_rig import EXTRA_BONES_STRATEGIES  # noqa: E402
from data_process.utils.skeleton import scale_anim  # noqa: E402


MIXAMO_OBJECT_TYPE = 'mixamo'

# Default raw-asset locations used for --char_path auto-resolution.
TRUEBONES_RAW_DIR = 'dataset/raw/truebones/Truebone_Z-OO'
OBJAVERSE_RAW_DIR = 'dataset/raw/objaverse/glb'
GENERAL_RAW_DIR = 'dataset/raw/general/animation'
# Mixamo's clips carry no mesh; the character they were made on.
MIXAMO_RAW_DEFAULT = 'dataset/raw/mixamo/character_refined/Y_Bot.fbx'

# Default cond.npy location per dataset (written by the feature-extraction stage).
COND_PATH_TEMPLATE = 'dataset/features/{dataset_type}/cond.npy'


# ---------------------------------------------------------------------------
# Animation loading
# ---------------------------------------------------------------------------

def build_anim_from_npz(anim_path, cond_data, scale_factor=None):
    """Build ``(anim, rest_anim, tpos_global_rot, raw_anim_data)`` from a
    feature-format NPZ (``global_positions`` + ``local_rotations``).

    Positions are divided by *scale_factor* (default: the cond's, back to the
    export's units; 1.0 keeps the canonical scale, for a canonical asset)."""
    parents = cond_data['parents']
    if scale_factor is None:
        scale_factor = npz_scalar(cond_data, 'scale_factor', 1.0)

    offsets_scaled = cond_data['offsets'] / scale_factor
    tpos_local_rot = cond_data['tpos_local_rotations']
    tpos_positions = cond_data['tpos_first_frame'] / scale_factor
    tpos_positions[1:] = offsets_scaled[1:]

    anim_data = np.load(anim_path, allow_pickle=True)
    motion_local_rot = anim_data['local_rotations']  # (nframes, nbones, 4)
    nbones = motion_local_rot.shape[1]

    anim_positions = anim_data['global_positions'] / scale_factor
    anim_positions[:, 1:] = offsets_scaled[1:][None]

    anim = Animation(
        rotations=Quaternions(motion_local_rot),
        positions=anim_positions,
        orients=Quaternions.id(nbones),
        offsets=offsets_scaled,
        parents=parents,
    )
    rest_anim = Animation(
        rotations=Quaternions(tpos_local_rot[None]),
        positions=tpos_positions[None],
        orients=Quaternions.id(nbones),
        offsets=offsets_scaled,
        parents=parents,
    )
    return anim, rest_anim, None, anim_data


def build_anim_from_npy(anim_path, cond_data, anim_mode, scale_factor=None):
    """Build ``(anim, rest_anim, tpos_global_rot, raw_anim_data)`` from a
    ``.npy`` of model motion features (the format ``unimate.inference.sample`` saves).
    *scale_factor* as in :func:`build_anim_from_npz`.

    Decoding model features back into an ``Animation`` requires the
    training/inference package (``unimate.utils.motion_utils``), which is
    imported lazily here so the NPZ path stays usable without it.
    """
    try:
        from unimate.utils.motion_utils import (
            recover_unimate_anim_from_rot,
            recover_unimate_joint_pos_from_ric,
        )
    except ImportError as exc:  # pragma: no cover — depends on the model package
        raise RuntimeError(
            ".npy model motion features need the training/inference package "
            "(unimate.utils.motion_utils); pass a feature-format .npz instead."
        ) from exc

    parents = cond_data['parents']
    if scale_factor is None:
        scale_factor = npz_scalar(cond_data, 'scale_factor', 1.0)
    tpos_global_rot = cond_data['tpos_global_rotations']

    offsets = cond_data['tpos_offsets']
    offsets_scaled = offsets / scale_factor

    anim_data = np.load(anim_path)
    nbones = anim_data.shape[1]

    if anim_mode == 'fk':
        logger.info("Recovering animation using FK mode")
        anim = recover_unimate_anim_from_rot(anim_data, parents, offsets)
        anim = scale_anim(anim, 1. / scale_factor)
    elif anim_mode == 'ik':
        logger.info("Recovering animation using IK mode")
        from InverseKinematics import animation_from_positions
        recover_pos_ric = recover_unimate_joint_pos_from_ric(anim_data)
        ik_anim, ik_order, _ = animation_from_positions(
            recover_pos_ric, parents, offsets, iterations=150,
        )
        inverse_order = np.argsort(ik_order)
        ik_anim_reordered = Animation(
            rotations=ik_anim.rotations[:, inverse_order],
            positions=ik_anim.positions[:, inverse_order],
            orients=ik_anim.orients[:, inverse_order],
            offsets=offsets,
            parents=parents,
        )
        anim = scale_anim(ik_anim_reordered, 1. / scale_factor)
    else:
        raise RuntimeError(f"Unsupported anim_mode: {anim_mode}")

    rest_anim = Animation(
        rotations=Quaternions.id((1, nbones)),
        positions=offsets_scaled[None],
        orients=Quaternions.id(nbones),
        offsets=offsets_scaled,
        parents=parents,
    )
    return anim, rest_anim, tpos_global_rot, anim_data


def load_animation(anim_path, cond_data, anim_mode, scale_factor=None):
    """Dispatch to the NPZ or NPY loader based on file extension."""
    if anim_path.endswith('.npz'):
        return build_anim_from_npz(anim_path, cond_data, scale_factor)
    if anim_path.endswith('.npy'):
        return build_anim_from_npy(anim_path, cond_data, anim_mode, scale_factor)
    raise RuntimeError(f"Unsupported animation file format: {anim_path}")


# ---------------------------------------------------------------------------
# Path resolution
# ---------------------------------------------------------------------------

def resolve_object_type(anim_path, dataset_type):
    """Map an animation filename to its cond-dict key.

    - truebones / objaverse / general: filename is ``{object_type}-{motion}[-{clip}].npz``,
      so the object type is the prefix before the first '-'.
    - mixamo: motion filenames carry no ``{object_type}-`` prefix; the cond
      dict always uses ``'mixamo'`` as its single key.
    """
    if dataset_type == 'mixamo':
        return MIXAMO_OBJECT_TYPE
    return os.path.basename(anim_path).split("-")[0]


# Characters --asset can auto-resolve (see resolve_char_path).
ASSET_KINDS = ('canonical', 'export')


def resolve_char_path(anim_path, dataset_type, character=None, asset='canonical',
                      strict=False):
    """Auto-resolve a character mesh path from ``anim_path`` + ``dataset_type``.

    *asset* 'canonical' (default): the canonical asset
    (``dataset/canonical_assets/<ds>/``; Mixamo: of *character*, default
    Michelle). When the object type has none, an error with *strict*, else the
    'export' resolution with a warning.

    *asset* 'export': the processed export GLB of the object type
    (``dataset/export/<ds>/rigs/``; Mixamo: of *character*, default Michelle)
    wins over the raw asset: its armature already is the exported skeleton,
    so no bones are merged away at animate time. The raw assets:

    - truebones: ``TRUEBONES_RAW_DIR/{object_type}/*.fbx`` (prefers an
      ``Idle`` / T-pose variant when available).
    - objaverse: ``OBJAVERSE_RAW_DIR/{object_type}.glb``.
    - general: the file in ``GENERAL_RAW_DIR`` whose asset name (stem with
      ``-`` and whitespace replaced by ``_``, as the exporter names it) is
      the object type.
    - mixamo: the raw Y Bot (``MIXAMO_RAW_DEFAULT``, the rig the clips were
      made on) when the default character is asked for; any other raw
      character needs ``--char_path``.
    """
    if asset not in ASSET_KINDS:
        raise ValueError(f"asset must be one of {ASSET_KINDS}, got {asset!r}")
    if asset == 'canonical':
        processed = find_processed_asset(anim_path, dataset_type, 'canonical', character)
        if processed is not None:
            logger.info(f"Using the canonical asset: {processed}")
            return processed
        if strict:
            return resolve_processed_asset(anim_path, dataset_type, 'canonical',
                                           character)  # raises
        logger.warning(
            f"No canonical asset "
            f"{processed_asset_path(anim_path, dataset_type, 'canonical', character)}; "
            f"using the export / raw asset instead (motion at the export's scale)")

    processed = find_processed_asset(anim_path, dataset_type, 'export', character)
    if processed is not None:
        logger.info(f"Using the processed export asset: {processed}")
        return processed
    if (dataset_type == 'mixamo'
            and (character or DEFAULT_MIXAMO_CHARACTER) == DEFAULT_MIXAMO_CHARACTER
            and os.path.isfile(MIXAMO_RAW_DEFAULT)):
        logger.info(f"No processed Mixamo characters; using the raw {MIXAMO_RAW_DEFAULT}")
        return MIXAMO_RAW_DEFAULT
    if dataset_type == 'mixamo':
        return resolve_processed_asset(anim_path, dataset_type, 'export', character)  # raises

    object_type = resolve_object_type(anim_path, dataset_type)

    if dataset_type == 'truebones':
        char_dir = os.path.join(TRUEBONES_RAW_DIR, object_type)
        if not os.path.isdir(char_dir):
            raise FileNotFoundError(
                f"No truebones character directory for object_type='{object_type}' "
                f"(resolved from '{anim_path}') at {char_dir}")
        fbx_files = sorted(f for f in os.listdir(char_dir) if f.lower().endswith('.fbx'))
        if not fbx_files:
            raise FileNotFoundError(f"No FBX file found in {char_dir}")
        idle_files = [f for f in fbx_files if 'idle' in f.lower()]
        return os.path.join(char_dir, idle_files[0] if idle_files else fbx_files[0])

    if dataset_type == 'objaverse':
        char_path = os.path.join(OBJAVERSE_RAW_DIR, f'{object_type}.glb')
        if not os.path.isfile(char_path):
            raise FileNotFoundError(
                f"No objaverse character file for object_type='{object_type}' "
                f"(resolved from '{anim_path}') at {char_path}")
        return char_path

    if dataset_type == 'general':
        matches = [p for p in (list_asset_files(GENERAL_RAW_DIR)
                               if os.path.isdir(GENERAL_RAW_DIR) else [])
                   if asset_name(p) == object_type]
        if not matches:
            raise FileNotFoundError(
                f"No general asset file for object_type='{object_type}' "
                f"(resolved from '{anim_path}') in {GENERAL_RAW_DIR}")
        return str(matches[0])

    raise ValueError(f"Unknown dataset_type: {dataset_type}")


def load_cond_data(cond_path, anim_path, dataset_type):
    """Pick the clip's cond entry out of ``cond_path``.

    The key is the object type resolved from the filename (or ``'mixamo'``).
    A cond file holding exactly one entry (``rig_preprocess`` output, or
    a custom single-asset export) is accepted whatever the clip is called,
    since there is nothing to disambiguate.
    """
    object_type = resolve_object_type(anim_path, dataset_type)
    cond_dict = np.load(cond_path, allow_pickle=True).item()
    if object_type not in cond_dict:
        if len(cond_dict) == 1:
            (only_key,) = cond_dict.keys()
            logger.warning(
                f"object_type '{object_type}' not in {cond_path}; using its single "
                f"entry '{only_key}'")
            return cond_dict[only_key]
        raise KeyError(
            f"object_type '{object_type}' (resolved from '{anim_path}' under "
            f"dataset_type='{dataset_type}') not found in cond file {cond_path}; "
            f"available keys: {sorted(cond_dict.keys())}")
    return cond_dict[object_type]


# ---------------------------------------------------------------------------
# Main pipeline
# ---------------------------------------------------------------------------

def animate_character(char_path, anim_path, cond_path, output_dir,
                      dataset_type, anim_mode='fk',
                      extra_bones_strategy='merge'):
    """Drive a character mesh with a motion clip and export GLB + FBX.

    Args:
        char_path: Character FBX/GLB (provides mesh + armature).
        anim_path: Feature-format motion NPZ, or a ``.npy`` of model motion features.
        cond_path: ``cond.npy`` for the dataset.
        output_dir: Destination directory (created if missing).
        dataset_type: ``'truebones'`` | ``'objaverse'`` | ``'mixamo'`` | ``'general'``;
            controls how the cond-dict key is derived from ``anim_path``.
        anim_mode: Recovery mode for ``.npy`` model-feature input, ``'fk'`` or ``'ik'``.
        extra_bones_strategy: How to handle armature bones not present in
            the animation (see :func:`sync_armature_bones`).
    """
    assert os.path.exists(anim_path), f"Animation file not found: {anim_path}"
    assert os.path.exists(cond_path), f"Condition file not found: {cond_path}"

    os.makedirs(output_dir, exist_ok=True)
    anim_base = os.path.splitext(os.path.basename(anim_path))[0]

    char_armature = load_character(char_path)
    # A canonical asset already is at the cond's canonical scale: keep the
    # motion there instead of dividing it back to the export's units.
    canonical = is_canonical_asset(char_armature)
    if canonical:
        logger.info("Canonical asset: motion kept in the canonical frame and scale")

    cond_data = load_cond_data(cond_path, anim_path, dataset_type)
    if canonical:
        check_canonical_order(char_armature, cond_data['joint_names'], cond_path)
    anim, rest_anim, tpos_global_rot, anim_data = load_animation(
        anim_path, cond_data, anim_mode, scale_factor=1.0 if canonical else None)

    drive_and_export(
        char_armature, anim, rest_anim, cond_data['joint_names'],
        fps=npz_scalar(anim_data, 'fps', 30),
        output_path_no_ext=os.path.join(output_dir, anim_base),
        formats=('glb', 'fbx'),
        extra_bones_strategy=extra_bones_strategy,
        tpos_global_rot=tpos_global_rot,
    )


def parse_args():
    parser = argparse.ArgumentParser(
        description="Drive a rigged character with a canonicalized motion clip.")
    parser.add_argument("--dataset_type", type=str, required=True,
                        choices=['truebones', 'objaverse', 'mixamo', 'general'],
                        help="Dataset that produced the cond/anim files; controls "
                             "how the cond-dict key is resolved.")
    parser.add_argument("--anim_path", type=str, required=True,
                        help="Feature-format motion NPZ, or a .npy of model motion features.")
    parser.add_argument("--char_path", type=str, default=None,
                        help="Character FBX/GLB. If omitted, auto-resolved from "
                             "--dataset_type and --anim_path by --asset.")
    parser.add_argument("--asset", type=str, default='canonical', choices=list(ASSET_KINDS),
                        help="Auto-resolved character: 'canonical' (default) the canonical "
                             "asset in dataset/canonical_assets/<ds>/ (mixamo: of "
                             "--character), falling back to 'export' when the object type "
                             "has none; 'export' the processed GLB in dataset/export/<ds>/rigs/ "
                             "when present, else the raw asset (truebones -> "
                             f"{TRUEBONES_RAW_DIR}/{{object_type}}/*.fbx, objaverse -> "
                             f"{OBJAVERSE_RAW_DIR}/{{object_type}}.glb, general -> the "
                             f"{GENERAL_RAW_DIR} file named like the object type).")
    parser.add_argument("--character", type=str, default=None,
                        help=f"Mixamo character to animate (default {DEFAULT_MIXAMO_CHARACTER}); "
                             "mixamo only (elsewhere the asset is the motion's object type).")
    parser.add_argument("--canonical", action="store_true",
                        help="Require the canonical asset: --asset canonical without the "
                             "export fallback (an error when it is missing).")
    parser.add_argument("--cond_path", type=str, default=None,
                        help=f"cond.npy for the dataset (default: {COND_PATH_TEMPLATE}).")
    parser.add_argument("--output_dir", type=str, default='outputs/animated')
    parser.add_argument("--anim_mode", type=str, default='fk', choices=['fk', 'ik'],
                        help="Recovery mode for .npy model-feature input.")
    parser.add_argument("--extra_bones_strategy", type=str, default='merge',
                        choices=list(EXTRA_BONES_STRATEGIES),
                        help="What to do with armature bones missing from the "
                             "animation. 'merge' (default): transfer vertex weights to "
                             "the nearest kept ancestor, then remove the bones. "
                             "'remove': delete bones AND mesh vertices dominantly "
                             "weighted to them. 'keep': leave in place, no keyframes.")
    return parse_blender_argv(parser)


if __name__ == "__main__":
    args = parse_args()
    if args.canonical and args.asset != 'canonical':
        raise SystemExit("--canonical contradicts --asset export")
    char_path = args.char_path or resolve_char_path(
        args.anim_path, args.dataset_type, character=args.character,
        asset=args.asset, strict=args.canonical)
    cond_path = args.cond_path or COND_PATH_TEMPLATE.format(dataset_type=args.dataset_type)
    logger.info(f"Using character mesh: {char_path}")
    logger.info(f"Using cond file: {cond_path}")
    animate_character(
        char_path=char_path,
        anim_path=args.anim_path,
        cond_path=cond_path,
        output_dir=args.output_dir,
        dataset_type=args.dataset_type,
        anim_mode=args.anim_mode,
        extra_bones_strategy=args.extra_bones_strategy,
    )