Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

10.9 KB
Raw Blame
"""Per-view VAE execution and publication for generation stages."""

from __future__ import annotations

import gc
import logging
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
from copy import deepcopy
from fractions import Fraction
from pathlib import Path
from threading import BoundedSemaphore, Event
from typing import TYPE_CHECKING, TypeVar

import numpy as np

from fdanyone.config import INFERENCE
from fdanyone.errors import FourDAnyoneError
from fdanyone.video import write_video

if TYPE_CHECKING:
    from torch import Tensor

    from fdanyone.vendor.diffsynth.models.wan_video_vae import WanVideoVAE38

LOGGER = logging.getLogger("fdanyone")

Output = TypeVar("Output")


def _bf16_autocast():
    import torch

    return torch.autocast(device_type="cuda", dtype=torch.bfloat16)


def _rgb_frames(video: Tensor) -> tuple[np.ndarray, ...]:
    """Freeze the released CUDA scaling boundary before CPU publication."""

    scaled = video.detach().float().add_(1.0).mul_(127.5).clamp_(0.0, 255.0).to(device="cpu")
    return tuple(
        scaled[:, frame_index].permute(1, 2, 0).numpy().astype(np.uint8) for frame_index in range(scaled.shape[1])
    )


class VaeExecutor:
    """Run independent views on an ordered CUDA device pool.

    Each worker owns one model replica and processes a deterministic strided
    subset of camera indices. Models return to CPU between stages so the same
    replicas can be reused without competing with the DiT for device memory.
    """

    def __init__(self, model: WanVideoVAE38, devices: tuple[str, ...]) -> None:
        if not devices:
            raise FourDAnyoneError("VAE execution requires at least one CUDA device.")
        self.devices = devices
        self._models = [model]
        self.last_peak_vram_bytes: dict[str, dict[str, int]] = {}

    @classmethod
    def load(cls, path: str | Path, devices: tuple[str, ...]) -> VaeExecutor:
        from fdanyone.model.loader import load_vae

        return cls(load_vae(path), devices)

    @property
    def latent_channels(self) -> int:
        return int(self._models[0].model.z_dim)

    @property
    def upsampling_factor(self) -> int:
        return int(self._models[0].upsampling_factor)

    def _ensure_models(self, count: int) -> None:
        while len(self._models) < count:
            self._models.append(deepcopy(self._models[0]))

    @staticmethod
    def _worker_count(num_views: int, num_devices: int) -> int:
        if num_views <= 0:
            raise FourDAnyoneError("VAE execution requires at least one view.")
        return min(num_views, num_devices)

    @staticmethod
    def _worker_indices(num_views: int, worker_index: int, worker_count: int) -> tuple[int, ...]:
        return tuple(range(worker_index, num_views, worker_count))

    def _run_workers(
        self,
        *,
        num_views: int,
        operation: Callable[[WanVideoVAE38, str, tuple[int, ...], Event], None],
        stopped: Event | None = None,
    ) -> None:
        import torch

        worker_count = self._worker_count(num_views, len(self.devices))
        self._ensure_models(worker_count)
        stopped = Event() if stopped is None else stopped
        peaks: dict[str, dict[str, int]] = {}

        def run(worker_index: int) -> None:
            device = self.devices[worker_index]
            device_index = int(device.removeprefix("cuda:"))
            model = self._models[worker_index]
            indices = self._worker_indices(num_views, worker_index, worker_count)
            torch.cuda.set_device(device_index)
            torch.cuda.reset_peak_memory_stats(device_index)
            try:
                model.to(device)
                operation(model, device, indices, stopped)
                torch.cuda.synchronize(device_index)
                peaks[device] = {
                    "allocated": int(torch.cuda.max_memory_allocated(device_index)),
                    "reserved": int(torch.cuda.max_memory_reserved(device_index)),
                }
            except BaseException:
                stopped.set()
                raise
            finally:
                model.to("cpu")

        with ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="vae") as pool:
            futures = [pool.submit(run, worker_index) for worker_index in range(worker_count)]
            try:
                for future in as_completed(futures):
                    future.result()
            except BaseException:
                stopped.set()
                for future in futures:
                    future.cancel()
                raise
        # Do not let a completed worker release cached mappings while a peer is
        # still executing.  This raced with expandable segments in low-memory
        # multi-GPU inference.  Join every worker before serial cache cleanup.
        for worker_index in range(worker_count):
            torch.cuda.set_device(int(self.devices[worker_index].removeprefix("cuda:")))
            torch.cuda.empty_cache()
        self.last_peak_vram_bytes = peaks

    @staticmethod
    def _encode_view(model: WanVideoVAE38, video: Tensor, device: str) -> Tensor:
        import torch

        if INFERENCE.tiled_vae:
            tile_size = tuple(size * model.upsampling_factor for size in INFERENCE.vae_tile_size)
            tile_stride = tuple(stride * model.upsampling_factor for stride in INFERENCE.vae_tile_stride)
            batched = video.unsqueeze(0).to(dtype=torch.bfloat16, device="cpu")
            return model.tiled_encode(batched, device, tile_size, tile_stride)
        batched = video.unsqueeze(0).to(dtype=torch.bfloat16, device=device)
        return model.encode_view(batched)

    @staticmethod
    def _decode_view(model: WanVideoVAE38, latent: Tensor, device: str) -> Tensor:
        import torch

        if INFERENCE.tiled_vae:
            batched = latent.unsqueeze(0).to(dtype=torch.bfloat16, device="cpu")
            return model.tiled_decode(
                batched,
                device,
                INFERENCE.vae_tile_size,
                INFERENCE.vae_tile_stride,
            )
        batched = latent.unsqueeze(0).to(dtype=torch.bfloat16, device=device)
        return model.decode_view(batched)

    def encode(self, videos: Tensor) -> Tensor:
        """Encode CPU-resident views and gather latents by input index."""

        import torch

        if videos.ndim != 5 or videos.device.type != "cpu":
            raise FourDAnyoneError(f"VAE input must be CPU [V,C,F,H,W], got {tuple(videos.shape)} on {videos.device}.")
        outputs: list[Tensor | None] = [None] * videos.shape[0]

        def operation(model: WanVideoVAE38, device: str, indices: tuple[int, ...], stopped: Event) -> None:
            with torch.inference_mode(), _bf16_autocast():
                for index in indices:
                    if stopped.is_set():
                        return
                    encoded = self._encode_view(model, videos[index], device)
                    outputs[index] = encoded[0].detach().to(device="cpu").contiguous()
                    del encoded

        self._run_workers(num_views=videos.shape[0], operation=operation)
        if any(output is None for output in outputs):
            raise RuntimeError("VAE encode completed without every canonical view.")
        return torch.stack(outputs)  # type: ignore[arg-type]

    def _decode_and_publish(
        self,
        latents: Tensor,
        sink: Callable[[int, tuple[np.ndarray, ...]], Output],
    ) -> tuple[Output, ...]:
        import torch

        if latents.ndim != 5 or latents.device.type != "cpu":
            raise FourDAnyoneError(
                f"VAE latents must be CPU [V,C,F,H,W], got {tuple(latents.shape)} on {latents.device}."
            )
        num_views = int(latents.shape[0])
        worker_count = self._worker_count(num_views, len(self.devices))
        slots = BoundedSemaphore(worker_count)
        sink_futures: list[Future[Output] | None] = [None] * num_views
        stopped = Event()

        with ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="video-codec") as codec_pool:

            def release_slot(future: Future[Output]) -> None:
                if future.exception() is not None:
                    stopped.set()
                slots.release()

            def operation(
                model: WanVideoVAE38,
                device: str,
                indices: tuple[int, ...],
                worker_stop: Event,
            ) -> None:
                with torch.inference_mode(), _bf16_autocast():
                    for index in indices:
                        if worker_stop.is_set():
                            return
                        slots.acquire()
                        try:
                            decoded = self._decode_view(model, latents[index], device)
                            device_video = decoded[0].detach()
                            rgb_frames = _rgb_frames(device_video)
                            del decoded, device_video
                            future = codec_pool.submit(
                                sink,
                                index,
                                rgb_frames,
                            )
                            del rgb_frames
                            future.add_done_callback(release_slot)
                            sink_futures[index] = future
                        except BaseException:
                            slots.release()
                            stopped.set()
                            raise

            self._run_workers(num_views=num_views, operation=operation, stopped=stopped)
            for future in sink_futures:
                if future is not None and future.done() and future.exception() is not None:
                    future.result()
            if any(future is None for future in sink_futures):
                raise RuntimeError("VAE decode completed without every canonical view.")
            return tuple(future.result() for future in sink_futures if future is not None)

    def publish_targets(self, latents: Tensor, output_dir: Path, fps: Fraction) -> tuple[Path, ...]:
        video_root = output_dir / "videos"
        video_root.mkdir(parents=True, exist_ok=False)

        def publish(camera_id: int, rgb_frames: tuple[np.ndarray, ...]) -> Path:
            LOGGER.info("Publishing target camera %02d", camera_id)
            return write_video(
                iter(rgb_frames),
                video_root / f"{camera_id:02d}.mp4",
                fps,
                crf=INFERENCE.target_h264_crf,
                preset=INFERENCE.h264_preset,
            )

        return self._decode_and_publish(latents, publish)

    def release_replicas(self) -> None:
        """Keep one CPU model while releasing stage-local parallel replicas."""

        del self._models[1:]
        gc.collect()

    def close(self) -> None:
        """Release all CPU replicas after the final VAE stage."""

        self._models.clear()
        gc.collect()