Skip to content

[SIGGRAPH Asia 2026] UniMate: One Unified Model to Animate Diverse Skeletons

Python
5.5 KB
Raw Blame
"""Merge per-worker export summary JSONs into the canonical files.

Multi-worker export runs write per-worker summary shards
(``joint_names_worker{i}.json``, ``joint_count_worker{i}.json``,
``clip_frames_worker{i}.json``) so parallel Blender processes never race
on one file. This script merges them — together with any existing canonical
files — into ``joint_names.json`` / ``joint_count.json`` /
``clip_frames.json`` (plus ``summary.json`` with the dataset-level
summary) and removes the shards. Idempotent: rerunning with no shards present
just rewrites the canonical files from themselves.

Plain Python (no bpy), so it runs in the conda env directly:
    python -m data_process.tools.merge_summaries --output_dir <export_dir>
"""

import argparse
import glob
import json
import os
import sys

from loguru import logger

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))

from data_process.utils.export_markers import collect_joint_names_from_markers  # noqa: E402


def _load(path):
    with open(path) as f:
        return json.load(f)


def _save(path, data):
    tmp = f"{path}.tmp.{os.getpid()}"
    with open(tmp, "w") as f:
        json.dump(data, f, indent=2)
    os.replace(tmp, path)


def _scan_missing_frame_counts(motion_dir, known):
    """Frame counts for clips in *motion_dir* that *known* doesn't cover."""
    if not os.path.isdir(motion_dir):
        return {}
    import numpy as np  # local: keeps the module importable without numpy

    found = {}
    for npz_file in sorted(os.listdir(motion_dir)):
        if not npz_file.endswith(".npz"):
            continue
        clip_name = os.path.splitext(npz_file)[0]
        if clip_name in known:
            continue
        try:
            data = np.load(os.path.join(motion_dir, npz_file), allow_pickle=True)
            found[clip_name] = int(data["anim_local_rot"].shape[0])
        except Exception as exc:  # noqa: BLE001 — a bad clip shouldn't stop the merge
            logger.warning(f"Could not read frame count from {npz_file}: {exc}")
    return found


def merge_summaries(output_dir, fps=30, keep_shards=False):
    jn_path = os.path.join(output_dir, "joint_names.json")
    joint_names = _load(jn_path) if os.path.isfile(jn_path) else {}

    # Per-asset completion markers are written as each asset finishes, so they
    # survive a killed worker whose in-memory shard never reached disk. Folding
    # them in here is what makes a truncated joint_names.json self-healing.
    from_markers = collect_joint_names_from_markers(output_dir)
    recovered = len(set(from_markers) - set(joint_names))
    joint_names.update(from_markers)
    if recovered:
        logger.info(f"Recovered {recovered} joint-name entries from completion markers")

    jn_shards = sorted(glob.glob(os.path.join(output_dir, "joint_names_worker*.json")))
    for shard in jn_shards:
        joint_names.update(_load(shard))
    if joint_names:
        _save(jn_path, joint_names)
        _save(os.path.join(output_dir, "joint_count.json"),
              {k: len(v) for k, v in joint_names.items()})
    logger.info(f"joint_names.json: {len(joint_names)} entries "
                f"({len(jn_shards)} shard(s) merged)")

    af_path = os.path.join(output_dir, "clip_frames.json")
    frames = {}
    if os.path.isfile(af_path):
        # A "_summary" block in the file is not a clip.
        frames = {k: v for k, v in _load(af_path).items() if k != "_summary"}
    af_shards = sorted(glob.glob(os.path.join(output_dir, "clip_frames_worker*.json")))
    for shard in af_shards:
        frames.update({k: v for k, v in _load(shard).items() if k != "_summary"})

    # Clips written after the last summary flush (or by a worker that was
    # killed before writing its shard) are on disk but absent here. Scan for
    # those only — the NPZ read is the expensive part, so already-recorded
    # clips are left alone.
    scanned = _scan_missing_frame_counts(os.path.join(output_dir, "motions"), frames)
    if scanned:
        logger.info(f"Recovered {len(scanned)} clip frame counts by scanning motions/")
        frames.update(scanned)

    if frames:
        _save(af_path, frames)
        total_frames = sum(frames.values())
        duration_sec = total_frames / fps if fps else 0.0
        _save(os.path.join(output_dir, "summary.json"), {
            "total_clips": len(frames),
            "total_frames": total_frames,
            "total_duration_sec": round(duration_sec, 2),
            "total_duration_min": round(duration_sec / 60, 2),
            "fps": fps,
        })
    logger.info(f"clip_frames.json: {len(frames)} clips "
                f"({len(af_shards)} shard(s) merged)")

    if not keep_shards:
        jc_shards = sorted(glob.glob(os.path.join(output_dir, "joint_count_worker*.json")))
        for shard in jn_shards + af_shards + jc_shards:
            os.remove(shard)


def main():
    parser = argparse.ArgumentParser(
        description="Merge per-worker export summary JSONs into the canonical files.")
    parser.add_argument("--output_dir", type=str, required=True,
                        help="Export directory holding the *_worker{i}.json shards.")
    parser.add_argument("--fps", type=int, default=30,
                        help="Frame rate for the clip_frames duration summary.")
    parser.add_argument("--keep_shards", action="store_true",
                        help="Keep the per-worker shard files after merging.")
    args = parser.parse_args()
    merge_summaries(args.output_dir, fps=args.fps, keep_shards=args.keep_shards)


if __name__ == "__main__":
    main()