Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

9.5 KB
Raw Blame
"""Reusable GVHMR result stored as JSON plus safetensors."""

from __future__ import annotations

import json
from dataclasses import dataclass
from fractions import Fraction
from pathlib import Path
from typing import Any

from fdanyone.errors import FourDAnyoneError

SMPL_PARAMETER_NAMES = ("body_pose", "betas", "global_orient", "transl")
SMPL_PARAMETER_WIDTHS = {"body_pose": 63, "betas": 10, "global_orient": 3, "transl": 3}


@dataclass(frozen=True)
class MotionResult:
    gvhmr_revision: str
    fps: Fraction
    frame_timestamps_sec: tuple[float, ...]
    source_frame_indices: tuple[int, ...]
    source_pts: tuple[int | None, ...]
    source_size_bytes: int
    source_mtime_ns: int
    image_height: int
    image_width: int
    smpl_params_global: dict[str, Any]
    smpl_params_incam: dict[str, Any]
    K_fullimg: Any
    observed_keypoints_2d: Any
    motion_world: str = "gvhmr_gravity_aligned_y_up"

    @property
    def num_frames(self) -> int:
        return len(self.frame_timestamps_sec)

    def validate(self, expected_frames: int = 121) -> None:
        try:
            import torch
        except ImportError as exc:
            raise FourDAnyoneError("PyTorch is required to validate motion tensors.") from exc
        if self.motion_world != "gvhmr_gravity_aligned_y_up":
            raise FourDAnyoneError(f"Unknown motion world convention: {self.motion_world!r}.")
        if (
            not isinstance(self.gvhmr_revision, str)
            or len(self.gvhmr_revision) != 40
            or any(character not in "0123456789abcdef" for character in self.gvhmr_revision.lower())
        ):
            raise FourDAnyoneError("MotionResult requires a 40-character GVHMR git revision.")
        if self.fps <= 0:
            raise FourDAnyoneError(f"MotionResult FPS must be positive, got {self.fps}.")
        if self.num_frames != expected_frames:
            raise FourDAnyoneError(f"MotionResult has {self.num_frames} frames, expected {expected_frames}.")
        for name, values in (
            ("source_frame_indices", self.source_frame_indices),
            ("source_pts", self.source_pts),
        ):
            if len(values) != self.num_frames:
                raise FourDAnyoneError(f"MotionResult {name} has {len(values)} values, expected {self.num_frames}.")
        expected_timestamps = tuple(float(Fraction(index, 1) / self.fps) for index in range(self.num_frames))
        if self.frame_timestamps_sec != expected_timestamps:
            raise FourDAnyoneError("MotionResult timestamps are not the exact zero-based CFR timeline.")
        if any(index < 0 for index in self.source_frame_indices) or any(
            right < left for left, right in zip(self.source_frame_indices, self.source_frame_indices[1:], strict=False)
        ):
            raise FourDAnyoneError("MotionResult source-frame indices must be non-negative and monotonic.")
        if self.source_size_bytes <= 0 or self.source_mtime_ns <= 0:
            raise FourDAnyoneError("MotionResult has an invalid source-file identity.")
        if self.image_height <= 0 or self.image_width <= 0:
            raise FourDAnyoneError("MotionResult image dimensions must be positive.")
        for group_name, parameters in (
            ("smpl_params_global", self.smpl_params_global),
            ("smpl_params_incam", self.smpl_params_incam),
        ):
            if set(parameters) != set(SMPL_PARAMETER_NAMES):
                raise FourDAnyoneError(
                    f"MotionResult {group_name} must contain {SMPL_PARAMETER_NAMES}, got {tuple(parameters)}."
                )
            for name, tensor in parameters.items():
                expected_shape = (self.num_frames, SMPL_PARAMETER_WIDTHS[name])
                if not isinstance(tensor, torch.Tensor) or tuple(tensor.shape) != expected_shape:
                    raise FourDAnyoneError(f"{group_name}.{name} must have shape {expected_shape}.")
                if not bool(torch.isfinite(tensor).all()):
                    raise FourDAnyoneError(f"{group_name}.{name} contains non-finite values.")
        if not isinstance(self.K_fullimg, torch.Tensor) or tuple(self.K_fullimg.shape) != (self.num_frames, 3, 3):
            raise FourDAnyoneError(f"K_fullimg must have shape ({self.num_frames}, 3, 3).")
        if not bool(torch.isfinite(self.K_fullimg).all()):
            raise FourDAnyoneError("K_fullimg contains non-finite values.")
        expected_keypoint_shape = (self.num_frames, 17, 3)
        if (
            not isinstance(self.observed_keypoints_2d, torch.Tensor)
            or tuple(self.observed_keypoints_2d.shape) != expected_keypoint_shape
        ):
            raise FourDAnyoneError(f"observed_keypoints_2d must have shape {expected_keypoint_shape}.")
        if not bool(torch.isfinite(self.observed_keypoints_2d).all()):
            raise FourDAnyoneError("observed_keypoints_2d contains non-finite values.")

    def validate_against_clip(self, clip) -> None:
        """Reject a cached result produced from a different video timeline."""

        self.validate(expected_frames=len(clip.frames))
        expected_timestamps = tuple(float(frame.canonical_timestamp) for frame in clip.frames)
        expected_indices = tuple(frame.source_index for frame in clip.frames)
        expected_pts = tuple(frame.source_pts for frame in clip.frames)
        if self.fps != clip.fps:
            raise FourDAnyoneError(f"Motion FPS {self.fps} does not match canonical FPS {clip.fps}.")
        if self.frame_timestamps_sec != expected_timestamps:
            raise FourDAnyoneError("Motion timestamps do not match the canonical clip.")
        if self.source_frame_indices != expected_indices or self.source_pts != expected_pts:
            raise FourDAnyoneError("Motion source-frame identity does not match the canonical clip.")
        if (self.source_size_bytes, self.source_mtime_ns) != (
            clip.source_size_bytes,
            clip.source_mtime_ns,
        ):
            raise FourDAnyoneError("Cached GVHMR motion belongs to a different source file.")

    def save(self, directory: str | Path) -> Path:
        from safetensors.torch import save_file

        self.validate()
        root = Path(directory).expanduser().resolve()
        root.mkdir(parents=True, exist_ok=True)
        tensor_path = root / "motion.safetensors"
        tensors = {
            **{
                f"smpl_params_global.{name}": self.smpl_params_global[name].detach().cpu().contiguous()
                for name in SMPL_PARAMETER_NAMES
            },
            **{
                f"smpl_params_incam.{name}": self.smpl_params_incam[name].detach().cpu().contiguous()
                for name in SMPL_PARAMETER_NAMES
            },
            "K_fullimg": self.K_fullimg.detach().cpu().contiguous(),
            "observed_keypoints_2d": self.observed_keypoints_2d.detach().cpu().contiguous(),
        }
        save_file(tensors, str(tensor_path))
        metadata = {
            "gvhmr_revision": self.gvhmr_revision,
            "num_frames": self.num_frames,
            "fps_num": self.fps.numerator,
            "fps_den": self.fps.denominator,
            "frame_timestamps_sec": self.frame_timestamps_sec,
            "source_frame_indices": self.source_frame_indices,
            "source_pts": self.source_pts,
            "source_size_bytes": self.source_size_bytes,
            "source_mtime_ns": self.source_mtime_ns,
            "image_height": self.image_height,
            "image_width": self.image_width,
            "motion_world": self.motion_world,
            "tensor_file": tensor_path.name,
        }
        (root / "motion.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
        return root

    @classmethod
    def load(cls, directory: str | Path) -> MotionResult:
        from safetensors.torch import load_file

        root = Path(directory).expanduser().resolve()
        metadata = json.loads((root / "motion.json").read_text())
        tensors = load_file(str(root / metadata["tensor_file"]), device="cpu")
        # ``safetensors`` may keep the source file memory-mapped for as long as
        # returned tensors own that storage.  Motion results live until final
        # export, which in turn can leave an in-use ``.efc_*`` tombstone on
        # CPFS/NFS when scratch cleanup unlinks the backing file.  The payload
        # is tiny compared with the generated videos, so take ownership here
        # and close the file mapping at this explicit process boundary.
        owned = {name: tensor.clone() for name, tensor in tensors.items()}
        del tensors
        result = cls(
            gvhmr_revision=str(metadata["gvhmr_revision"]),
            fps=Fraction(int(metadata["fps_num"]), int(metadata["fps_den"])),
            frame_timestamps_sec=tuple(float(value) for value in metadata["frame_timestamps_sec"]),
            source_frame_indices=tuple(int(value) for value in metadata["source_frame_indices"]),
            source_pts=tuple(None if value is None else int(value) for value in metadata["source_pts"]),
            source_size_bytes=int(metadata["source_size_bytes"]),
            source_mtime_ns=int(metadata["source_mtime_ns"]),
            image_height=int(metadata["image_height"]),
            image_width=int(metadata["image_width"]),
            smpl_params_global={name: owned[f"smpl_params_global.{name}"] for name in SMPL_PARAMETER_NAMES},
            smpl_params_incam={name: owned[f"smpl_params_incam.{name}"] for name in SMPL_PARAMETER_NAMES},
            K_fullimg=owned["K_fullimg"],
            observed_keypoints_2d=owned["observed_keypoints_2d"],
            motion_world=str(metadata["motion_world"]),
        )
        result.validate()
        return result