Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

16.5 KB
Raw Blame
"""A synchronized body, source overlay and complete target-video scene."""

from __future__ import annotations

import hashlib
import json
import shutil
import threading
import uuid
from collections.abc import Callable
from concurrent.futures import CancelledError, ThreadPoolExecutor
from dataclasses import dataclass
from fractions import Fraction
from pathlib import Path

import av
import numpy as np

from fdanyone.config import FRAMING
from fdanyone.errors import FourDAnyoneError
from fdanyone.geometry.cameras import camera_grid, reference_intrinsics
from fdanyone.output_directory import read_output_metadata
from fdanyone.result_videos import read_target_videos
from fdanyone.space import scene

_EXPORT_LOCK = threading.Lock()
RECORDING_VERSION = 17


@dataclass(frozen=True)
class Result:
    directory: Path
    metadata: dict
    cameras: tuple[dict, ...]
    videos: tuple[Path, ...]
    fps: Fraction
    frames: int


def _contained_file(root: Path, relative: str) -> Path:
    path = root / relative
    if not path.is_file() or not path.resolve().is_relative_to(root):
        raise FourDAnyoneError(f"Missing or external result file: {relative}")
    return path


def read_result(directory: str | Path) -> Result:
    root = Path(directory).expanduser().resolve()
    _contained_file(root, "metadata.json")
    metadata = read_output_metadata(root)
    try:
        rig = json.loads(_contained_file(root, "cameras.json").read_text())
        cameras = rig["cameras"]
        if rig.get("camera_model") != "OPENCV" or not isinstance(cameras, list) or not cameras:
            raise ValueError("Expected a nonempty OPENCV camera rig")
        if rig["world_frame"]["name"] != "canonical_human_world" or rig["camera_frame"]["name"] != "opencv_camera":
            raise ValueError("Unsupported camera coordinate system")
        if [camera["camera_id"] for camera in cameras] != list(range(len(cameras))):
            raise ValueError("Cameras must be ordered by camera ID")
        fps = Fraction(metadata["output"]["fps"])
        frames = int(metadata["output"]["frames_per_video"])
        if fps <= 0 or frames != 121 or metadata["output"]["target_views"] != len(cameras):
            raise ValueError("Output does not describe synchronized 121-frame videos")
        videos = read_target_videos(root, cameras)
        for camera in cameras:
            intrinsic = np.asarray(camera["K"], dtype=np.float64)
            transform = np.asarray(camera["camera_to_world"], dtype=np.float64)
            if (
                intrinsic.shape != (3, 3)
                or transform.shape != (4, 4)
                or not np.isfinite(intrinsic).all()
                or not np.isfinite(transform).all()
                or not np.allclose(transform[3], [0, 0, 0, 1])
            ):
                raise ValueError(f"Invalid calibration for camera {camera['camera_id']}")
            if min(camera["image_width"], camera["image_height"]) <= 0:
                raise ValueError("Invalid camera image dimensions")
    except (OSError, ValueError, TypeError, KeyError, ZeroDivisionError) as exc:
        raise FourDAnyoneError(f"Cannot open this 4DAnyone output: {exc}") from exc
    return Result(root, metadata, tuple(cameras), videos, fps, frames)


def _target_video(result: Result, camera_id: int, destination: Path, check_cancelled: Callable) -> tuple[int, int]:
    """Keep the generated H.264 bytes, including their native resolution and quality."""

    camera = result.cameras[camera_id]
    size = camera["image_width"], camera["image_height"]
    if destination.exists():
        return size
    temporary = destination.with_name(f".{destination.stem}-{uuid.uuid4().hex}.mp4")
    count = 0
    try:
        with av.open(str(result.videos[camera_id])) as source:
            source.streams.video[0].codec_context.thread_count = 2
            if source.streams.video[0].average_rate != result.fps:
                raise FourDAnyoneError(f"Camera {camera_id} video FPS does not match its metadata.")
            for count, frame in enumerate(source.decode(video=0), start=1):
                check_cancelled()
                if count > result.frames:
                    raise FourDAnyoneError(f"Camera {camera_id} has more than {result.frames} frames.")
                if (frame.width, frame.height) != (camera["image_width"], camera["image_height"]):
                    raise FourDAnyoneError(f"Camera {camera_id} video dimensions do not match its calibration.")
        if count != result.frames:
            raise FourDAnyoneError(f"Camera {camera_id} has {count} frames; expected {result.frames}.")
        check_cancelled()
        shutil.copyfile(result.videos[camera_id], temporary)
        temporary.replace(destination)
    finally:
        temporary.unlink(missing_ok=True)
    return size


def _prepare_targets(result: Result, media: Path, check_cancelled: Callable) -> dict:
    stopped = threading.Event()

    def check():
        check_cancelled()
        if stopped.is_set():
            raise CancelledError

    def prepare(index):
        check()
        return _target_video(result, index, media / f"{index:02d}.mp4", check)

    with ThreadPoolExecutor(max_workers=min(4, len(result.videos)), thread_name_prefix="space-video") as pool:
        futures = [pool.submit(prepare, index) for index in range(len(result.videos))]
        try:
            return {index: future.result() for index, future in enumerate(futures)}
        finally:
            stopped.set()
            for future in futures:
                future.cancel()


def scene_info(recording: Path) -> dict:
    return json.loads(recording.with_suffix(".json").read_text())


def _identity(path: Path | None):
    if path is None or not path.is_file():
        return None
    stat = path.stat()
    return str(path.resolve()), stat.st_size, stat.st_mtime_ns


def _cache_key(identity) -> str:
    return hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:24]


def _body_identity(motion_dir, model_dir, gvhmr_root):
    from fdanyone.assets import MHR70_REGRESSOR, SMPLX_MODEL

    if motion_dir is None:
        return None
    return [
        [_identity(motion_dir / name) for name in ("motion.json", "motion.safetensors")],
        [_identity(model_dir / name) for name in (MHR70_REGRESSOR, SMPLX_MODEL)] if model_dir else None,
        str(gvhmr_root.resolve()) if gvhmr_root else None,
    ]


def _cached_recording(destination: Path, *, require_body: bool) -> bool:
    try:
        # Failed body preparation must remain retryable, including after a
        # transient failure that did not change any of the input files.
        if not destination.is_file():
            return False
        info = scene_info(destination)
        return not require_body or info["body_frames"] > 0
    except (OSError, ValueError, KeyError):
        return False


def _result_context(result: Result) -> dict:
    framing = result.metadata.get("preprocessing", {}).get("framing", {})
    radius = framing.get("radius", FRAMING.reference_radius)
    height = framing.get("target_height", FRAMING.reference_target_height)
    first = result.cameras[0]
    transform = np.asarray(first["camera_to_world"])
    center = transform[:3, 3] + transform[:3, 2] * radius / np.linalg.norm(transform[[0, 2], 2])
    center[1] = 0
    return {
        "center": center.tolist(),
        "radius": radius,
        "target_height": height,
        "K": first["K"],
        "image_width": first["image_width"],
        "image_height": first["image_height"],
    }


def layout_cameras(context: dict, layout: dict) -> list[dict]:
    cameras = camera_grid(
        center=np.asarray(context["center"]),
        front_direction=np.array([0, 0, -1]),
        K=np.asarray(context["K"]),
        image_height=context["image_height"],
        image_width=context["image_width"],
        radius=context["radius"],
        target_height=context["target_height"],
        **layout,
    )
    return [
        {
            "camera_id": c.camera_id,
            "K": c.K,
            "camera_to_world": c.camera_to_world,
            "image_width": c.image_width,
            "image_height": c.image_height,
            "yaw": c.yaw_degrees,
            "pitch": c.pitch_degrees,
        }
        for c in cameras
    ]


def _save_recording(destination: Path, info: dict, log_scene) -> Path:
    import rerun as rr

    temporary = destination.with_name(f".{uuid.uuid4().hex}.rrd")
    recording = rr.RecordingStream(info["application_id"], recording_id=info["recording_id"])
    try:
        recording.save(str(temporary), default_blueprint=scene.blueprint(info["context"], info["target_count"]))
        recording.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_UP, static=True)
        scene.log_stage(recording, info["context"])
        log_scene(recording)
        recording.flush()
        recording.disconnect()
        temporary.replace(destination)
        destination.with_suffix(".json").write_text(json.dumps(info) + "\n")
    finally:
        recording.disconnect()
        temporary.unlink(missing_ok=True)
    return destination


def _prepare_body(motion_dir, model_dir, gvhmr_root, cache_dir, check_cancelled):
    from fdanyone.space.body import load_body

    if motion_dir is None:
        return None, []
    if model_dir is None or gvhmr_root is None:
        return None, ["Set the model and GVHMR directories to show the recovered body."]
    try:
        return load_body(motion_dir, model_dir, gvhmr_root, cache_dir, check_cancelled), []
    except (FourDAnyoneError, OSError, ValueError) as exc:
        return None, [str(exc)]


def export_recording(
    result: Result,
    cache_dir: Path,
    *,
    source: Path | None = None,
    model_dir: Path | None = None,
    gvhmr_root: Path | None = None,
    check_cancelled: Callable = lambda: None,
) -> Path:
    from fdanyone.space.overlay import export_overlay
    from fdanyone.space.source import prepare_source

    body_identity = _body_identity(result.directory / "gvhmr", model_dir, gvhmr_root)
    identity = [
        RECORDING_VERSION,
        str(result.directory),
        result.metadata,
        result.cameras,
        [_identity(p) for p in result.videos],
        _identity(source),
        body_identity,
    ]
    key = _cache_key(identity)
    recording_id = uuid.uuid4().hex
    destination = cache_dir / f"{key}.rrd"
    with _EXPORT_LOCK:
        check_cancelled()
        if _cached_recording(destination, require_body=bool(model_dir and gvhmr_root)):
            return destination
        media = cache_dir / key
        media.mkdir(parents=True, exist_ok=True)
        sizes = _prepare_targets(result, media, check_cancelled)
        body, notes = _prepare_body(result.directory / "gvhmr", model_dir, gvhmr_root, cache_dir, check_cancelled)
        source_info = None
        if source is not None:
            indices = body["source_indices"] if body is not None else None
            source_info = prepare_source(
                source,
                cache_dir,
                fps=result.fps,
                indices=indices,
                start_time=result.metadata["input"].get("start_time_seconds", 0),
                check_cancelled=check_cancelled,
            )
        else:
            notes.append("Original Source Video Unavailable. Restore the source at its saved path.")
        info = {
            "application_id": f"4danyone_space_{recording_id}",
            "recording_id": recording_id,
            "context": _result_context(result),
            "target_count": len(result.cameras),
            "camera_count": len(result.cameras),
            "notes": notes,
            "source": str(source) if source else None,
            "body_frames": len(body["vertices"]) if body is not None else 0,
            "fps": float(result.fps),
            "frames": result.frames,
            "media": {
                "source": source_info,
                "targets": [
                    {"id": index, "path": str(media / f"{index:02d}.mp4"), "width": size[0], "height": size[1]}
                    for index, size in sizes.items()
                ],
                "overlay": export_overlay(body, cache_dir / "overlays" / _cache_key([1, body_identity]))
                if body is not None
                else None,
            },
        }

        def log_scene(recording):
            scene.log_cameras(
                recording,
                result.cameras,
                videos=[media / f"{index:02d}.mp4" for index in sizes],
                fps=result.fps,
                frames=result.frames,
            )
            if body is not None:
                scene.log_body(recording, body, result.fps, check_cancelled)
            check_cancelled()

        return _save_recording(destination, info, log_scene)


def export_input(
    source: Path | None,
    cache_dir: Path,
    layout: dict,
    *,
    start_time=0.0,
    target_fps="auto",
    motion_dir: Path | None = None,
    model_dir: Path | None = None,
    gvhmr_root: Path | None = None,
    check_cancelled: Callable = lambda: None,
) -> Path:
    from fdanyone.space.overlay import export_overlay
    from fdanyone.space.source import prepare_source
    from fdanyone.video import validate_clip_options

    rate = validate_clip_options(start_time=start_time, fps=None if str(target_fps).lower() == "auto" else target_fps)
    body_identity = _body_identity(motion_dir, model_dir, gvhmr_root)
    key = _cache_key(
        [RECORDING_VERSION, "input", _identity(source), float(start_time), str(rate), layout, body_identity]
    )
    destination = cache_dir / f"input-{key}.rrd"

    with _EXPORT_LOCK:
        check_cancelled()
        if _cached_recording(destination, require_body=bool(motion_dir and model_dir and gvhmr_root)):
            return destination
        cache_dir.mkdir(parents=True, exist_ok=True)
        body, notes = _prepare_body(motion_dir, model_dir, gvhmr_root, cache_dir, check_cancelled)
        source_info = (
            prepare_source(
                source,
                cache_dir,
                start_time=start_time,
                fps=rate,
                check_cancelled=check_cancelled,
                indices=body["source_indices"] if body is not None else None,
            )
            if source
            else None
        )
        width, height = (source_info["width"], source_info["height"]) if source_info else (360, 640)
        context = {
            "center": [0, 0, 0],
            "radius": FRAMING.reference_radius,
            "target_height": FRAMING.reference_target_height,
            "image_width": width,
            "image_height": height,
            "K": reference_intrinsics(height, width).tolist(),
        }
        cameras = layout_cameras(context, layout)
        recording_id = uuid.uuid4().hex
        info = {
            "application_id": f"4danyone_space_{recording_id}",
            "recording_id": recording_id,
            "context": context,
            "target_count": 0,
            "camera_count": len(cameras),
            "notes": notes,
            "body_frames": len(body["vertices"]) if body is not None else 0,
            "source": str(source) if source else None,
            "fps": float(Fraction(source_info["fps"])) if source_info else 25.0,
            "frames": source_info["frames"] if source_info else 0,
            "media": {
                "source": source_info,
                "targets": [],
                "overlay": export_overlay(body, cache_dir / "overlays" / _cache_key([1, body_identity]))
                if body is not None
                else None,
            },
        }

        def log_scene(recording):
            scene.log_cameras(recording, cameras)
            if source_info and body is None:
                scene.log_source_card(recording, context, Path(source_info["path"]), source_info)
            if body is not None:
                scene.log_body(recording, body, info["fps"], check_cancelled)

        return _save_recording(destination, info, log_scene)


def export_layout_update(info: dict, cache_dir: Path, layout: dict) -> tuple[Path, dict]:
    """Append a small camera update to the active store without reloading its videos."""

    import rerun as rr

    if info["target_count"]:
        raise FourDAnyoneError("Saved result cameras are read-only.")
    cameras = layout_cameras(info["context"], layout)
    destination = cache_dir / f"layout-{uuid.uuid4().hex}.rrd"
    cache_dir.mkdir(parents=True, exist_ok=True)
    recording = rr.RecordingStream(info["application_id"], recording_id=info["recording_id"])
    try:
        recording.save(str(destination))
        scene.log_cameras(recording, cameras)
        recording.flush()
    finally:
        recording.disconnect()
    return destination, {**info, "camera_count": len(cameras)}