Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

15.1 KB
Raw Blame
"""Frozen, timestep-independent conditioning used by the denoiser."""

from __future__ import annotations

import gc
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from copy import deepcopy
from dataclasses import dataclass
from pathlib import Path
from threading import Event
from typing import TYPE_CHECKING

from fdanyone.config import INFERENCE
from fdanyone.errors import AssetError, FourDAnyoneError
from fdanyone.views import VIEWS_PER_GROUP

if TYPE_CHECKING:
    from torch import Tensor

    from fdanyone.skeleton.pipeline import Conditioning, SkeletonVideo

LOGGER = logging.getLogger("fdanyone")

PROMPT_CONTEXT_FORMAT = "fdanyone.prompt_context"
PROMPT_CONTEXT_VERSION = "2"
PROMPT_CONTEXT_KEY = "context"
PROMPT_CONTEXT_SHAPE = (1, 512, 4096)
PROMPT_CONTEXT_SOURCE_REPO = "Wan-AI/Wan2.1-T2V-1.3B"
PROMPT_CONTEXT_SOURCE_REVISION = "3f40b6dc4ca5c02dd23c9db74d9d2ccb82903b86"
POSE_FEATURE_SHAPE = (3072, 31, 40, 22)
POSE_ENCODER_BATCH_LIMIT = 6


def load_prompt_context(path: str | Path):
    """Load and validate the frozen UMT5 output consumed by the DiT."""

    import torch

    try:
        from safetensors import safe_open
    except ImportError as exc:
        raise AssetError("safetensors is required to load prompt conditioning.") from exc

    resolved = Path(path).expanduser().resolve()
    try:
        with safe_open(str(resolved), framework="pt", device="cpu") as tensors:
            keys = tuple(tensors.keys())
            metadata = tensors.metadata() or {}
            if keys != (PROMPT_CONTEXT_KEY,):
                raise AssetError(f"Prompt conditioning must contain only {PROMPT_CONTEXT_KEY!r}, got {keys}.")
            context = tensors.get_tensor(PROMPT_CONTEXT_KEY)
    except AssetError:
        raise
    except Exception as exc:
        raise AssetError(f"Could not load prompt conditioning from {resolved}: {exc}") from exc

    expected_metadata = {
        "format": PROMPT_CONTEXT_FORMAT,
        "version": PROMPT_CONTEXT_VERSION,
        "prompt": INFERENCE.prompt,
        "source_repo": PROMPT_CONTEXT_SOURCE_REPO,
        "source_revision": PROMPT_CONTEXT_SOURCE_REVISION,
    }
    mismatched = {
        key: (metadata.get(key), expected)
        for key, expected in expected_metadata.items()
        if metadata.get(key) != expected
    }
    if mismatched:
        raise AssetError(f"Prompt conditioning metadata does not match the released method: {mismatched}.")
    for key in ("text_encoder_sha256", "tokenizer_manifest_sha256"):
        value = metadata.get(key, "")
        if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
            raise AssetError(f"Prompt conditioning metadata has no valid {key} provenance hash.")
    if tuple(context.shape) != PROMPT_CONTEXT_SHAPE or context.dtype != torch.bfloat16:
        raise AssetError(
            "Prompt conditioning must be BF16 with shape "
            f"{PROMPT_CONTEXT_SHAPE}, got dtype={context.dtype}, shape={tuple(context.shape)}."
        )
    return context.contiguous()


@dataclass(frozen=True)
class PoseEncodingPlan:
    """Fixed-shape PoseEncoder batches for one ordered camera layout."""

    batch_size: int
    packed_views: int
    feature_groups: tuple[tuple[int, ...], ...]


def plan_pose_encoding(*, num_features: int, group_size: int, packed_views: int) -> PoseEncodingPlan:
    """Partition camera indices while reserving packed-view null inputs.

    Every PoseEncoder call uses the same batch shape.  This preserves the
    released CUDA convolution path while bounding full-resolution inputs to six
    videos.  Camera indices remain the only identity used after decoding.
    """

    if num_features <= 0:
        raise FourDAnyoneError(f"Pose conditioning requires at least one camera, got {num_features}.")
    if group_size <= 0 or num_features % group_size:
        raise FourDAnyoneError(
            f"Pose camera count {num_features} must be divisible by positive group size {group_size}."
        )
    if packed_views <= 0:
        raise FourDAnyoneError(f"Pose conditioning requires packed views, got {packed_views}.")

    batch_size = min(POSE_ENCODER_BATCH_LIMIT, group_size + packed_views)
    feature_capacity = batch_size - packed_views
    if feature_capacity <= 0:
        raise FourDAnyoneError(f"Pose batch limit {POSE_ENCODER_BATCH_LIMIT} cannot hold {packed_views} packed views.")
    groups = tuple(
        tuple(range(start, min(start + feature_capacity, num_features)))
        for start in range(0, num_features, feature_capacity)
    )
    return PoseEncodingPlan(batch_size=batch_size, packed_views=packed_views, feature_groups=groups)


@dataclass(frozen=True)
class PoseFeatureBank:
    """Owned CPU BF16 features in canonical camera-index order."""

    features: Tensor
    null_features: Tensor

    def __post_init__(self) -> None:
        import torch

        feature_shape = tuple(self.features.shape)
        null_shape = tuple(self.null_features.shape)
        if self.features.ndim != 5 or feature_shape[1:] != POSE_FEATURE_SHAPE:
            raise FourDAnyoneError(f"Pose features must have shape [views, {POSE_FEATURE_SHAPE}], got {feature_shape}.")
        if self.null_features.ndim != 5 or null_shape[1:] != POSE_FEATURE_SHAPE or null_shape[0] <= 0:
            raise FourDAnyoneError(
                f"Null pose features must have shape [packed_views, {POSE_FEATURE_SHAPE}], got {null_shape}."
            )
        if self.features.dtype != torch.bfloat16 or self.null_features.dtype != torch.bfloat16:
            raise FourDAnyoneError("Pose and null features must use BF16.")
        if self.features.device.type != "cpu" or self.null_features.device.type != "cpu":
            raise FourDAnyoneError("Pose feature banks must remain CPU-resident between denoising calls.")
        for label, tensor in (("pose", self.features), ("null pose", self.null_features)):
            exact_storage_bytes = tensor.numel() * tensor.element_size()
            if (
                not tensor.is_contiguous()
                or tensor.storage_offset() != 0
                or tensor.untyped_storage().nbytes() != exact_storage_bytes
            ):
                raise FourDAnyoneError(f"The {label} feature tensor must own one exact contiguous storage.")

    @property
    def num_features(self) -> int:
        return int(self.features.shape[0])

    @property
    def num_packed_views(self) -> int:
        return int(self.null_features.shape[0])

    def allocate_group(self, size: int, device: str) -> Tensor:
        """Allocate one reusable group buffer on the requested device."""

        import torch

        if size <= 0:
            raise FourDAnyoneError(f"A pose feature group must be non-empty, got {size}.")
        return torch.empty((size, *POSE_FEATURE_SHAPE), dtype=self.features.dtype, device=device)

    def copy_group(self, indices: tuple[int, ...], destination: Tensor) -> None:
        """Copy canonical camera features into a reusable device buffer."""

        if not indices:
            raise FourDAnyoneError("A pose feature group cannot be empty.")
        expected = (len(indices), *POSE_FEATURE_SHAPE)
        if tuple(destination.shape) != expected or destination.dtype != self.features.dtype:
            raise FourDAnyoneError(
                f"Pose destination must have dtype={self.features.dtype}, shape={expected}; "
                f"got dtype={destination.dtype}, shape={tuple(destination.shape)}."
            )
        invalid = tuple(index for index in indices if index < 0 or index >= self.num_features)
        if invalid:
            raise FourDAnyoneError(
                f"Pose feature indices {invalid} are outside camera range 0..{self.num_features - 1}."
            )
        for output_index, feature_index in enumerate(indices):
            destination[output_index].copy_(self.features[feature_index])


@dataclass(frozen=True)
class PoseFeatureCache:
    """Pose features grouped by the RCP and target denoising layouts."""

    rcp: PoseFeatureBank | None
    target: PoseFeatureBank


@dataclass(frozen=True)
class _PoseEncodingJob:
    """One complete fixed-shape PoseEncoder call."""

    feature_indices: tuple[int, ...]
    skeletons: tuple[SkeletonVideo, ...]
    batch_size: int
    packed_views: int
    builder: _PoseFeatureBuilder


@dataclass
class _PoseFeatureBuilder:
    """Gather independently computed batches into canonical camera order."""

    features: Tensor
    null_features: Tensor

    @classmethod
    def create(cls, num_features: int, packed_views: int) -> _PoseFeatureBuilder:
        import torch

        return cls(
            features=torch.empty((num_features, *POSE_FEATURE_SHAPE), dtype=torch.bfloat16, device="cpu"),
            null_features=torch.empty((packed_views, *POSE_FEATURE_SHAPE), dtype=torch.bfloat16, device="cpu"),
        )

    def store(self, job: _PoseEncodingJob, encoded: Tensor) -> None:
        for batch_index, feature_index in enumerate(job.feature_indices):
            self.features[feature_index].copy_(encoded[batch_index])
        if job.feature_indices[0] == 0:
            null_start = len(job.feature_indices)
            self.null_features.copy_(encoded[null_start : null_start + job.packed_views])

    def build(self) -> PoseFeatureBank:
        return PoseFeatureBank(features=self.features, null_features=self.null_features)


def _encode_pose_batch(pose_encoder, videos):
    """Encode one bounded batch and return its CPU-resident BF16 features."""

    import torch

    with torch.inference_mode(), torch.autocast(device_type="cuda", dtype=torch.bfloat16):
        encoded = pose_encoder(videos)
    expected = (videos.shape[0], *POSE_FEATURE_SHAPE)
    if tuple(encoded.shape) != expected or encoded.dtype != torch.bfloat16:
        raise FourDAnyoneError(
            f"PoseEncoder returned dtype={encoded.dtype}, shape={tuple(encoded.shape)}; expected BF16 {expected}."
        )
    return encoded.detach().to(device="cpu").contiguous()


def _pose_jobs(
    *,
    skeletons: tuple[SkeletonVideo, ...],
    group_size: int,
    packed_views: int,
) -> tuple[tuple[_PoseEncodingJob, ...], _PoseFeatureBuilder]:
    """Plan complete fixed-shape calls and their canonical output owner."""

    plan = plan_pose_encoding(
        num_features=len(skeletons),
        group_size=group_size,
        packed_views=packed_views,
    )
    builder = _PoseFeatureBuilder.create(len(skeletons), packed_views)
    jobs = tuple(
        _PoseEncodingJob(
            feature_indices=indices,
            skeletons=tuple(skeletons[index] for index in indices),
            batch_size=plan.batch_size,
            packed_views=packed_views,
            builder=builder,
        )
        for indices in plan.feature_groups
    )
    return jobs, builder


def _execute_pose_job(
    *,
    job: _PoseEncodingJob,
    pose_encoder,
    conditioning: Conditioning,
    device: str,
) -> None:
    """Decode, encode, and gather one bounded pose batch."""

    import torch

    video_shape = None
    batch = None
    prefix = pose_encoder.temporal_prefix
    for batch_index, skeleton in enumerate(job.skeletons):
        LOGGER.info("Loading skeleton conditioning from %s", skeleton.path.name)
        decoded = conditioning.load_skeleton_tensor([skeleton]).to(dtype=torch.bfloat16, device="cpu").contiguous()
        if decoded.ndim != 5 or decoded.shape[0] != 1:
            raise FourDAnyoneError(f"Expected one 5D skeleton tensor, got shape {tuple(decoded.shape)}.")
        current_shape = tuple(decoded.shape[1:])
        if video_shape is None:
            video_shape = current_shape
            batch = torch.empty(
                (job.batch_size, video_shape[0], video_shape[1] + prefix, *video_shape[2:]),
                dtype=torch.bfloat16,
                device=device,
            ).fill_(-1)
        elif current_shape != video_shape:
            raise FourDAnyoneError(
                f"Skeleton {skeleton.path} has shape {tuple(decoded.shape)}, expected {(1, *video_shape)}."
            )
        # Construct the exact contiguous input previously produced by cat in
        # PoseEncoder. Null slots remain -1, including their temporal prefix.
        batch[batch_index, :, :prefix].copy_(decoded[0, :, :1])
        batch[batch_index, :, prefix:].copy_(decoded[0])
        del decoded

    encoded = _encode_pose_batch(pose_encoder, batch)
    job.builder.store(job, encoded)
    del batch, encoded


def build_pose_feature_cache(
    *,
    conditioning: Conditioning,
    checkpoint_path: str | Path,
    devices: tuple[str, ...],
) -> PoseFeatureCache:
    """Encode all fixed-shape pose jobs on an ordered CUDA device pool.

    One worker owns one PoseEncoder and one full-resolution batch at a time.
    This bounds host/device memory while independent jobs execute concurrently;
    builders restore canonical camera order regardless of completion order.
    """

    import torch

    from fdanyone.model.loader import load_pose_encoder

    view_plan = conditioning.view_plan
    if not conditioning.target_skeletons:
        raise FourDAnyoneError("Pose conditioning requires at least one skeleton video.")
    if not devices:
        raise FourDAnyoneError("Pose conditioning requires at least one CUDA device.")

    rcp_jobs: tuple[_PoseEncodingJob, ...] = ()
    rcp_builder = None
    if view_plan.enable_rcp:
        rcp_jobs, rcp_builder = _pose_jobs(
            skeletons=conditioning.rcp_skeletons,
            group_size=VIEWS_PER_GROUP,
            packed_views=1,
        )
    target_jobs, target_builder = _pose_jobs(
        skeletons=conditioning.target_skeletons,
        group_size=VIEWS_PER_GROUP,
        packed_views=2 if view_plan.enable_rcp else 1,
    )
    jobs = (*rcp_jobs, *target_jobs)
    worker_count = min(len(jobs), len(devices))
    stopped = Event()
    prototype = load_pose_encoder(checkpoint_path, "cpu")
    pose_encoders = [prototype]
    pose_encoders.extend(deepcopy(prototype) for _ in range(worker_count - 1))

    def run(worker_index: int) -> None:
        device = devices[worker_index]
        device_index = int(device.removeprefix("cuda:"))
        torch.cuda.set_device(device_index)
        pose_encoder = pose_encoders[worker_index]
        try:
            pose_encoder.to(device)
            for job in jobs[worker_index::worker_count]:
                if stopped.is_set():
                    return
                _execute_pose_job(
                    job=job,
                    pose_encoder=pose_encoder,
                    conditioning=conditioning,
                    device=device,
                )
        except BaseException:
            stopped.set()
            raise
        finally:
            pose_encoder.to("cpu")
            torch.cuda.empty_cache()

    with ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="pose") as pool:
        futures = [pool.submit(run, worker_index) for worker_index in range(worker_count)]
        try:
            for future in as_completed(futures):
                future.result()
        except BaseException:
            stopped.set()
            for future in futures:
                future.cancel()
            raise

    pose_encoders.clear()
    gc.collect()
    return PoseFeatureCache(
        rcp=None if rcp_builder is None else rcp_builder.build(),
        target=target_builder.build(),
    )