Skip to content

Native Windows video motion capture with ExMR, FlowHMR and GVHMR, contact correction, humanoid retargeting, and FBX/VMD export.

Python
6.6 KB
Raw Blame
"""Pack rendered tasks into WebDataset shards for base training.

For each task directory written by render_pipeline.py:

    <task_id>.mp4, <task_id>_camera.npz, <task_id>_bbox.npz, <stem>.npz (motion)

extract SAM 3D Body features (3072-d per frame) on the tracked range and write one
sample with motion.npz / camera.npz / bbox.npz / feature.pt (format:
docs/resources.md#webdataset-shards-base-training).

Run from the FlowHMR repo root (needs the FlowHMR environment and
ckpts/sam-3d-body-dinov3):

    python data/rendering_data/scripts/make_webdataset.py \\
        --render_root data/rendering_data/examples_output \\
        --output data/webdataset/synthetic/synthetic-%05d.tar
"""

import argparse
import glob
import io
import json
import os
import os.path as osp
import sys

import numpy as np
import torch

REPO_ROOT = osp.normpath(osp.join(osp.dirname(osp.abspath(__file__)), '..', '..', '..'))
FPS = 30
NUM_BETAS = 16


def npz_bytes(**arrays):
    buf = io.BytesIO()
    np.savez(buf, **arrays)
    return buf.getvalue()


def pt_bytes(tensor):
    buf = io.BytesIO()
    torch.save(tensor, buf)
    return buf.getvalue()


def find_tasks(render_root):
    """Task directories that contain a rendered video."""
    tasks = []
    for mp4 in sorted(glob.glob(osp.join(render_root, '**', '*.mp4'), recursive=True)):
        task_dir = osp.dirname(mp4)
        task_id = osp.splitext(osp.basename(mp4))[0]
        if osp.basename(task_dir) != task_id:
            continue  # e.g. debug overlays
        tasks.append((task_dir, task_id))
    return tasks


def load_task(task_dir, task_id):
    camera = np.load(osp.join(task_dir, f'{task_id}_camera.npz'))
    bbox = np.load(osp.join(task_dir, f'{task_id}_bbox.npz'))
    stem = task_id.rsplit('_', 1)[0]
    motion = np.load(osp.join(task_dir, f'{stem}.npz'))

    poses = np.asarray(motion['poses'], dtype=np.float32)
    poses = poses.reshape(poses.shape[0], -1)
    if poses.shape[1] != 156:
        raise ValueError(f'expected SMPL-H poses (F, 156), got {poses.shape}')
    betas = np.asarray(motion['betas'], dtype=np.float32).reshape(1, -1)
    betas = np.pad(betas, ((0, 0), (0, max(0, NUM_BETAS - betas.shape[1]))))[:, :NUM_BETAS]
    fps = float(motion['mocap_framerate']) if 'mocap_framerate' in motion else FPS
    if abs(fps - FPS) > 1e-3:
        raise ValueError(f'expected {FPS} fps motion, got {fps}')

    return {
        'name': task_id,
        'video': osp.join(task_dir, f'{task_id}.mp4'),
        'poses': poses,
        'trans': np.asarray(motion['trans'], dtype=np.float32),
        'betas': betas,
        'RT': np.asarray(camera['RT'], dtype=np.float32),
        'K': np.asarray(camera['K'], dtype=np.float32),
        'bbox': np.asarray(bbox['bbox'], dtype=np.float32)[:, :4],
        'start_end': np.asarray(bbox['start_end'], dtype=np.int64).reshape(2),
    }


def check_lengths(s):
    num_frames = len(s['poses'])
    lengths = {k: len(s[k]) for k in ('trans', 'RT', 'K', 'bbox')}
    if any(n != num_frames for n in lengths.values()):
        raise ValueError(f'frame count mismatch: poses={num_frames}, {lengths}')
    start, end = s['start_end']
    if not (0 <= start <= end < num_frames) or end - start + 1 < 10:
        raise ValueError(f'invalid or too short tracked range {start}..{end} of {num_frames}')


def main():
    parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
    parser.add_argument('--render_root', required=True, help='--root of render_pipeline.py')
    parser.add_argument('--output', required=True, help='shard pattern, e.g. out/synthetic-%%05d.tar')
    parser.add_argument('--maxcount', type=int, default=1000, help='samples per shard')
    parser.add_argument('--save_features', action='store_true',
                        help='also save <task_id>_sam3d_feat.pt next to each task')
    args = parser.parse_args()

    sys.path.insert(0, REPO_ROOT)
    import webdataset as wds
    from flowhmr.utils.runtime_sam_features import build_sam3d_extractor

    tasks = find_tasks(args.render_root)
    print(f'found {len(tasks)} rendered tasks in {args.render_root}')
    if not tasks:
        return

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    extractor = build_sam3d_extractor(device)

    os.makedirs(osp.dirname(osp.abspath(args.output)), exist_ok=True)
    num_written, skipped = 0, []
    with wds.ShardWriter(args.output, maxcount=args.maxcount) as sink:
        for i, (task_dir, task_id) in enumerate(tasks):
            try:
                s = load_task(task_dir, task_id)
                check_lengths(s)
                start, end = (int(x) for x in s['start_end'])
                feature_path = osp.join(task_dir, f'{task_id}_sam3d_feat.pt')
                if osp.exists(feature_path):
                    feature = torch.load(feature_path, weights_only=True)
                else:
                    tokens = extractor.extract_video_tokens(
                        s['video'],
                        bbox_xyxy=torch.from_numpy(s['bbox']),
                        K_all=torch.from_numpy(s['K']),
                        max_frames=end + 1,
                    )
                    feature = tokens[start:end + 1].float().contiguous()
                    if args.save_features:
                        torch.save(feature, feature_path)
                if feature.shape != (end - start + 1, 3072):
                    raise ValueError(f'feature shape {tuple(feature.shape)}, '
                                     f'expected ({end - start + 1}, 3072)')
            except Exception as e:  # keep going on bad tasks
                print(f'[{i + 1}/{len(tasks)}] skip {task_id}: {e}')
                skipped.append(task_id)
                continue

            key = f'{num_written:08d}_{task_id}'.replace('.', '_')
            sink.write({
                '__key__': key,
                'motion.npz': npz_bytes(poses=s['poses'], trans=s['trans'], betas=s['betas'],
                                        mocap_framerate=FPS),
                'camera.npz': npz_bytes(RT=s['RT'], K=s['K']),
                'bbox.npz': npz_bytes(bbox=s['bbox'], start_end=s['start_end']),
                'feature.pt': pt_bytes(feature),
                'metadata.json': json.dumps({'sequence_name': task_id}).encode('utf-8'),
            })
            num_written += 1
            print(f'[{i + 1}/{len(tasks)}] {task_id}: {len(s["poses"])} frames, '
                  f'tracked {start}..{end}')

    print(f'wrote {num_written} samples, skipped {len(skipped)}')


if __name__ == '__main__':
    main()