[SIGGRAPH Asia 2026] UniMate: One Unified Model to Animate Diverse Skeletons
Python
"""Metadata I/O and clip windowing for the feature-extraction stage.
Loads the per-dataset JSON side files produced by the caption and
joint-annotation stages (``motion_captions.json``, ``category_groups.json``,
``clean_joint_names.json``, ``face_joint_names.json``, ...), provides
case-insensitive per-object lookups, writes the stage outputs (``cond.npy``,
``captions.json``, ``filtered_clips.json``, ``metadata.txt``) and computes
clip windows. Used by :mod:`data_process.feature_extraction.extract_features`.
"""
import json
import os
import tempfile
from contextlib import contextmanager
from os.path import join as pjoin
import numpy as np
# ---------------------------------------------------------------------------
# Atomic output writes
# ---------------------------------------------------------------------------
@contextmanager
def atomic_output_path(path):
# type: (str) -> Iterator[str]
"""Yield a temp path next to *path*, renamed into place on clean exit.
Stage outputs land at the end of runs that can take hours, so writing them
in place risks leaving a truncated ``cond.npy`` / ``captions.json`` behind
if the run is interrupted. Writers fill the yielded temp path instead; the
final ``os.replace`` is atomic, so readers see either the previous file or
the complete new one, and a failed write leaves the previous file intact.
"""
directory = os.path.dirname(os.path.abspath(path))
os.makedirs(directory, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(
dir=directory, prefix=f'.{os.path.basename(path)}.', suffix='.tmp')
os.close(fd)
try:
yield tmp_path
os.replace(tmp_path, path)
except BaseException:
try:
os.remove(tmp_path)
except OSError:
pass
raise
def atomic_np_save(path, data):
# type: (str, object) -> None
"""``np.save`` *data* to *path* atomically (see :func:`atomic_output_path`)."""
with atomic_output_path(path) as tmp_path:
# Passing a file object keeps np.save from appending '.npy' to the
# temp name (it only does that for string paths).
with open(tmp_path, 'wb') as f:
np.save(f, data)
# ---------------------------------------------------------------------------
# JSON loading & metadata helpers
# ---------------------------------------------------------------------------
def load_json(path):
# type: (str) -> Optional[dict]
"""Load a JSON file if it exists, otherwise return None."""
if path and os.path.isfile(path):
with open(path, 'r') as f:
return json.load(f)
return None
def load_motion_captions(source_dir):
# type: (str) -> Optional[Dict[str, str]]
"""Load motion captions from ``motion_captions.json`` in *source_dir*."""
return load_json(pjoin(source_dir, 'motion_captions.json'))
# Extra caption versions beside the normal one (``motion_captions.json`` ->
# ``captions.json``): the short generic caption and the longer detail caption,
# HumanML3D-style. Each comes only from its reviewed patch file through
# ``patch_annotations.py`` (export ``motion_captions_<v>.json``) and is saved by
# stage 4 as ``captions_<v>.json``; training may sample one per step.
EXTRA_CAPTION_VERSIONS = ('generic', 'detail')
def load_extra_captions(source_dir, version):
# type: (str, str) -> Optional[Dict[str, str]]
"""Load one extra caption version from ``motion_captions_<version>.json``
(written by ``patch_annotations.py``; None when the export has none)."""
if version not in EXTRA_CAPTION_VERSIONS:
raise ValueError(f'unknown caption version {version!r}; '
f'expected one of {EXTRA_CAPTION_VERSIONS}')
return load_json(pjoin(source_dir, f'motion_captions_{version}.json'))
def load_generic_captions(source_dir):
# type: (str) -> Optional[Dict[str, str]]
"""``load_extra_captions(source_dir, 'generic')``."""
return load_extra_captions(source_dir, 'generic')
def by_feature_key(feature_captions, export_captions):
# type: (Dict[str, str], Dict[str, str]) -> Dict[str, str]
"""Re-key an export ``{clip: caption}`` map (normal, generic or detail) by
the feature clip keys.
Feature keys are ``<export clip>-<NNN>`` (one per saved segment), the keys
of ``captions.json``; every segment of a clip gets the clip's caption.
Feature clips without an export caption are left out.
"""
out = {}
for key in feature_captions:
stem = key.rsplit('-', 1)[0]
if stem in export_captions:
out[key] = export_captions[stem]
return out
def load_category_groups(source_dir):
# type: (str) -> Optional[Dict[str, List[str]]]
"""Load category-to-object-type groupings from ``category_groups.json``."""
return load_json(pjoin(source_dir, 'category_groups.json'))
def load_clean_joint_names(source_dir):
# type: (str) -> Optional[Dict[str, List[str]]]
"""Load cleaned joint names from ``clean_joint_names.json``."""
return load_json(pjoin(source_dir, 'clean_joint_names.json'))
def load_face_joint_names(source_dir):
# type: (str) -> Optional[Dict[str, dict]]
"""Load face joint info from ``face_joint_names.json``."""
return load_json(pjoin(source_dir, 'face_joint_names.json'))
# The 22-joint humanoid core the mixamo rig is reduced to at extraction time
# (drops the 40 finger bones plus the End bones); toggled by --mixamo_core_joints.
MIXAMO_CORE_JOINTS = [
"mixamorig:Hips",
"mixamorig:Spine", "mixamorig:Spine1", "mixamorig:Spine2",
"mixamorig:Neck", "mixamorig:Head",
"mixamorig:LeftShoulder", "mixamorig:LeftArm",
"mixamorig:LeftForeArm", "mixamorig:LeftHand",
"mixamorig:RightShoulder", "mixamorig:RightArm",
"mixamorig:RightForeArm", "mixamorig:RightHand",
"mixamorig:LeftUpLeg", "mixamorig:LeftLeg",
"mixamorig:LeftFoot", "mixamorig:LeftToeBase",
"mixamorig:RightUpLeg", "mixamorig:RightLeg",
"mixamorig:RightFoot", "mixamorig:RightToeBase",
]
def load_joint_names(source_dir):
# type: (str) -> Optional[Dict[str, List[str]]]
"""Load raw joint names from ``joint_names.json``."""
return load_json(pjoin(source_dir, 'joint_names.json'))
def load_all_metadata(source_dir):
# type: (str) -> dict
"""Load all available JSON metadata files from *source_dir*."""
loaders = {
'motion_captions': load_motion_captions,
'category_groups': load_category_groups,
'clean_joint_names': load_clean_joint_names,
'face_joint_names': load_face_joint_names,
'joint_names': load_joint_names,
}
meta = {}
for key, loader in loaders.items():
data = loader(source_dir)
if data is not None:
meta[key] = data
return meta
def check_metadata_object_types_consistent(
metadata,
keys=('clean_joint_names', 'face_joint_names', 'joint_names'),
):
# type: (dict, tuple) -> None
"""Verify per-object-type metadata dicts cover the same object types.
Compares the case-insensitive key sets of each present dict named by
*keys* in *metadata*; absent or empty dicts are skipped. Raises
``ValueError`` on mismatch so drivers fail loudly on misaligned JSON
sources, and prints a consistency-OK line when all present dicts agree.
"""
present = {n: {k.lower() for k in metadata[n].keys()}
for n in keys if metadata.get(n)}
if len(present) < 2:
return
ref_name, ref_set = next(iter(present.items()))
for other_name, other_set in present.items():
if other_name == ref_name:
continue
only_ref = sorted(ref_set - other_set)
only_other = sorted(other_set - ref_set)
if only_ref or only_other:
raise ValueError(
f"Object type mismatch between '{ref_name}' and '{other_name}': "
f"only in {ref_name}: {only_ref}; "
f"only in {other_name}: {only_other}"
)
print(f'Metadata consistency OK across {list(present.keys())}: '
f'{len(ref_set)} object types')
def _dict_get_ci(d, key):
# type: (dict, str) -> object
"""Case-insensitive dict lookup. Tries exact match first, then lowercase."""
if key in d:
return d[key]
key_lower = key.lower()
for k, v in d.items():
if k.lower() == key_lower:
return v
return None
def get_object_metadata(object_type, metadata_dict):
# type: (str, Optional[dict]) -> object
"""Case-insensitive lookup of *object_type* in a metadata dict, or None."""
if metadata_dict:
return _dict_get_ci(metadata_dict, object_type)
return None
def get_object_face_joints(object_type, face_joint_names=None):
# type: (str, Optional[Dict[str, dict]]) -> Optional[dict]
"""Return face joint info for *object_type*, or None."""
return get_object_metadata(object_type, face_joint_names)
def get_object_captions(object_type, motion_captions=None):
# type: (str, Optional[Dict[str, str]]) -> Dict[str, str]
"""Filter motion captions to entries for *object_type*.
Caption keys follow the ``{object_type}-{motion_name}`` base-name
convention used throughout the pipeline; matching is case-insensitive
on the prefix. Returns an empty dict when no captions are supplied or
none match.
"""
if not motion_captions:
return {}
prefix = f'{object_type.lower()}-'
return {k: v for k, v in motion_captions.items()
if k.lower().startswith(prefix)}
def save_json(path, data):
"""Write *data* as indented JSON to *path*, atomically."""
with atomic_output_path(path) as tmp_path:
with open(tmp_path, 'w') as f:
json.dump(data, f, indent=2)
def collect_captions(cond):
# type: (Dict[str, dict]) -> Dict[str, str]
"""Merge per-object ``captions`` sub-dicts into a single flat dict."""
all_captions = {}
for obj_cond in cond.values():
if 'captions' in obj_cond:
all_captions.update(obj_cond['captions'])
return all_captions
def _remove_stale(save_dir, name, why):
"""Remove ``<save_dir>/<name>`` left by an earlier run, if any."""
path = pjoin(save_dir, name)
if os.path.isfile(path):
os.remove(path)
print(f'Removed stale {path} ({why} this run)')
def save_extra_captions(save_dir, all_captions, version, export_captions):
"""Write ``captions_<version>.json`` (keyed like ``captions.json``) from
the export's ``motion_captions_<version>.json`` map, or remove a stale
one when no saved clip has a caption of that version. Returns the dict
written (empty when none)."""
path = pjoin(save_dir, f'captions_{version}.json')
out = (by_feature_key(all_captions, export_captions)
if all_captions and export_captions else {})
if out:
save_json(path, out)
missing = sorted(set(all_captions) - set(out))
print(f'Saved {len(out)}/{len(all_captions)} {version} clip captions'
+ (f'; {len(missing)} saved clips have none, e.g. {missing[:3]}' if missing else ''))
else:
_remove_stale(save_dir, f'captions_{version}.json', f'no {version} captions')
return out
def save_outputs(save_dir, cond, all_filtered_clips, category_groups=None,
extra_captions=None, keep_category_groups=False):
"""Save ``cond.npy``, filtered clips, captions, and category groupings.
Caption files, keyed alike by saved clip (``<clip>-<NNN>``):
``captions.json`` from ``motion_captions.json`` (the normal caption; a
clip without one is not saved) and, for each version ``v`` of
``EXTRA_CAPTION_VERSIONS``, ``captions_<v>.json`` from
``extra_captions[v]`` (the export's ``motion_captions_<v>.json``). Extra
captions never enter ``cond`` or the ``cond_parts`` cache, so changing them
never re-processes an object; a version without captions has its stale
``captions_<v>.json`` removed.
Filtered clips, captions and category groups are only written when
present; when absent, the file an earlier run wrote is removed (the
training loader takes a missing ``captions.json`` as caption-free data
and selects category subsets from ``category_groups.json``), except a
``category_groups.json`` the run was told to leave alone
(*keep_category_groups*: ``--category_groups ""``, the file is copied in
by hand). Every file is
written atomically, so an interrupted run cannot leave a truncated output
behind for the training loader to trip over. Returns the merged captions
dict so callers can pass it to ``save_metadata_report``.
"""
atomic_np_save(pjoin(save_dir, 'cond.npy'), cond)
if all_filtered_clips:
save_json(pjoin(save_dir, 'filtered_clips.json'), all_filtered_clips)
total = sum(len(v) for v in all_filtered_clips.values())
print(f'Saved {total} filtered clips to filtered_clips.json')
else:
_remove_stale(save_dir, 'filtered_clips.json', 'no filtered clips')
all_captions = collect_captions(cond)
if all_captions:
save_json(pjoin(save_dir, 'captions.json'), all_captions)
print(f'Saved {len(all_captions)} clip captions')
else:
_remove_stale(save_dir, 'captions.json', 'no captions')
for version in EXTRA_CAPTION_VERSIONS:
save_extra_captions(save_dir, all_captions, version,
(extra_captions or {}).get(version))
if category_groups:
save_json(pjoin(save_dir, 'category_groups.json'), category_groups)
elif not keep_category_groups:
_remove_stale(save_dir, 'category_groups.json', 'no category groups')
return all_captions
def build_stats(clips_per_object, joints_per_object, total_frames, max_njoints, fps=30):
"""Aggregate per-object counts into the dataset summary dict.
``duration_min`` is a nominal figure: clips carry their own ``fps`` field
(30 for every source, 30 after the >=60 fps down-sampling in
:func:`~data_process.utils.motion_features._load_npz_anim`), but the
duration here simply divides the frame total by *fps*. The assumption is
reported alongside the value, so ``fps`` is echoed back in the stats.
"""
return {
'total_clips': sum(clips_per_object.values()),
'total_frames': total_frames,
'duration_min': total_frames / fps / 60,
'fps': fps,
'max_njoints': max_njoints,
'clips_per_object': clips_per_object,
'joints_per_object': joints_per_object,
}
def save_metadata_report(save_dir, stats, all_filtered_clips,
all_captions=None, category_groups=None):
"""Write a human-readable metadata summary to ``metadata.txt``.
``all_captions`` and ``category_groups`` are optional; their sections are
omitted when not supplied. The extra-caption counts are read back from
the ``captions_<v>.json`` files ``save_outputs`` wrote. The file is
written atomically.
"""
extra = {v: load_json(pjoin(save_dir, f'captions_{v}.json'))
for v in EXTRA_CAPTION_VERSIONS}
total_filtered = sum(len(v) for v in all_filtered_clips.values())
with atomic_output_path(pjoin(save_dir, 'metadata.txt')) as tmp_path, \
open(tmp_path, 'w') as f:
f.write(f'Total clips: {stats["total_clips"]}\n')
f.write(f'Total frames: {stats["total_frames"]}\n')
f.write(f'Total duration (minutes, assuming {stats.get("fps", 30)} fps): '
f'{stats["duration_min"]:.2f}\n')
f.write(f'Max joints in dataset: {stats["max_njoints"]}\n')
if all_captions is not None:
f.write(f'Total captions: {len(all_captions)}\n')
for version, caps in extra.items():
if caps is not None:
f.write(f'Total {version} captions: {len(caps)}\n')
f.write('Clips per object type:\n')
for obj, count in stats['clips_per_object'].items():
f.write(f' {obj}: {count}\n')
f.write('Joints per object type:\n')
for obj, count in stats['joints_per_object'].items():
f.write(f' {obj}: {count}\n')
f.write(f'Total filtered clips: {total_filtered}\n')
if all_filtered_clips:
f.write('Filtered clips per object type:\n')
for obj, clips in all_filtered_clips.items():
f.write(f' {obj}: {len(clips)}\n')
if category_groups:
f.write('Categories:\n')
for cat, members in category_groups.items():
processed = [m for m in members if m in stats['clips_per_object']]
f.write(f' {cat}: {len(processed)} object types\n')
# ---------------------------------------------------------------------------
# Clip windowing
# ---------------------------------------------------------------------------
def compute_clip_windows(nframes, max_clip_len, clip_stride):
"""Compute (start, end) windows for cropping a sequence into clips.
Windows are ``max_clip_len`` frames long and start every ``clip_stride``
frames. No returned window is ever longer than ``max_clip_len`` (the whole
sequence is returned unsliced only when it already fits).
Leftover frames past the last strided window are handled by tail length,
measured against the window overlap ``max_clip_len - clip_stride`` — the
longest training crop the clips are cut to hold:
- at least one full training crop: emitted as their own (shorter)
trailing window, which is long enough to be croppable on its own;
- shorter than that: covered by a full-length window anchored at the
final frame, replacing the last window when the frames it gives up stay
covered by the one before it (always true once there are two or more
windows);
- shorter than that and there is only one window to replace: dropped,
rather than glued onto the previous window (which would push that clip
past ``max_clip_len``) or duplicated as a near-copy of it.
"""
if not max_clip_len or nframes <= max_clip_len:
return [(0, nframes)]
clips = []
for start in range(0, nframes - max_clip_len + 1, clip_stride):
clips.append((start, start + max_clip_len))
last_end = clips[-1][1]
if last_end < nframes:
tail_len = nframes - last_end
# A trailing window earns its place once it can hold one full
# training crop, i.e. the overlap a normal window is given.
min_tail = max(1, max_clip_len - clip_stride)
if tail_len >= min_tail:
clips.append((last_end, nframes))
else:
# Cover the tail with a full-length window anchored at the end.
slid_start = nframes - max_clip_len
prev_end = clips[-2][1] if len(clips) >= 2 else 0
if slid_start <= prev_end:
# Free: the frames the last window gives up are still covered.
clips[-1] = (slid_start, nframes)
# Otherwise the tail is too small to pay for a window of its own.
return clips
def get_motion_clip_windows(nframes, max_clip_len, clip_stride, apply_clip=True):
"""Return clip windows for a motion sequence.
When ``apply_clip`` is True, crops into overlapping fixed-length clips via
:func:`compute_clip_windows`. When False, returns a single window covering
at most the first ``max_clip_len`` frames (i.e., truncation, not cropping).
"""
if apply_clip:
return compute_clip_windows(nframes, max_clip_len, clip_stride)
clip_end = min(nframes, max_clip_len) if max_clip_len else nframes
return [(0, clip_end)]
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
def print_summary(stats):
print(f'Total clips: {stats["total_clips"]}, Frames: {stats["total_frames"]}, '
f'Duration: {stats["duration_min"]:.1f}m (@{stats.get("fps", 30)} fps), '
f'Max joints: {stats["max_njoints"]}')