[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video
"""Hydra/Lightning-free multi-view generation.
RCP and final target generation share one source encoding and prompt embedding.
Target groups execute sequentially on one GPU or concurrently across multiple
GPUs; TCR optionally shifts their membership between denoising steps.
"""
from __future__ import annotations
import gc
import logging
import math
from dataclasses import dataclass
from fractions import Fraction
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, TypedDict
from fdanyone.assets import BaseAssets
from fdanyone.config import INFERENCE, DenoisingProfile
from fdanyone.errors import FourDAnyoneError
from fdanyone.model.conditioning import (
PoseFeatureBank,
build_pose_feature_cache,
load_prompt_context,
)
from fdanyone.model.denoise import denoise_group
from fdanyone.model.distributed import (
WorkerReport,
denoise_targets_distributed,
select_worker_devices,
)
from fdanyone.model.loader import Denoiser, load_denoiser
from fdanyone.model.metrics import GenerationMetrics
from fdanyone.model.routing import Routes, routing_steps, validate_routes
from fdanyone.model.vae import VaeExecutor
from fdanyone.skeleton.pipeline import Conditioning
from fdanyone.views import ViewPlan
if TYPE_CHECKING:
from torch import Tensor
LOGGER = logging.getLogger("fdanyone")
class ParallelismReport(TypedDict):
"""The target-denoising topology and measurements written to metadata."""
backend: str
candidate_devices: list[str]
used_devices: list[str]
groups_per_step: int
waves_per_step: int
workers: list[WorkerReport]
@dataclass(frozen=True)
class GeneratedViews:
"""Paths and measurements produced by one resolved view plan."""
target_videos: tuple[Path, ...]
view_plan: ViewPlan
denoising_profile: DenoisingProfile
seed: int
device: str
attention_backend: str
elapsed_seconds: dict[str, float]
stage_peak_vram_bytes: dict[str, dict[str, int]]
peak_vram_allocated_bytes: int
peak_vram_reserved_bytes: int
parallelism: ParallelismReport | None = None
@dataclass(frozen=True)
class _GenerationPlan:
view_plan: ViewPlan
candidate_devices: tuple[str, ...]
primary_device: str
dit_devices: tuple[str, ...]
@property
def primary_device_index(self) -> int:
return int(self.primary_device.removeprefix("cuda:"))
@property
def distributed(self) -> bool:
return len(self.dit_devices) > 1
@property
def needs_primary_denoiser(self) -> bool:
return self.view_plan.enable_rcp or not self.distributed
def _empty_cuda_cache() -> None:
import torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
def _bf16_autocast():
"""Match Lightning's ``bf16-mixed`` inference context without Lightning."""
import torch
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
def _channels_last_source_layout(video):
"""Preserve the frozen source tensor's VFHWC-backed VCFHW layout."""
import torch
if video.ndim != 5:
raise FourDAnyoneError(f"Expected a 5D video tensor, got shape {tuple(video.shape)}.")
return video.contiguous(memory_format=torch.channels_last_3d)
def _noise(
*,
vae: VaeExecutor,
num_views: int,
num_frames: int,
seed: int,
device: str,
) -> Tensor:
import torch
shape = (
num_views,
vae.latent_channels,
(num_frames - 1) // 4 + 1,
INFERENCE.height // vae.upsampling_factor,
INFERENCE.width // vae.upsampling_factor,
)
generator = torch.Generator("cpu").manual_seed(seed)
return torch.randn(shape, generator=generator, device="cpu", dtype=torch.float32).to(
dtype=torch.bfloat16, device=device
)
def _denoise_rcp(
*,
denoiser: Denoiser,
vae: VaeExecutor,
src_latents: Tensor,
context: Tensor,
camera_ids: tuple[int, ...],
pose_features: PoseFeatureBank,
seed: int,
device: str,
) -> Tensor:
import torch
from tqdm.auto import tqdm
if pose_features.num_features != len(camera_ids):
raise FourDAnyoneError(f"RCP requires {len(camera_ids)} pose features, got {pose_features.num_features}.")
latents = _noise(
vae=vae,
num_views=len(camera_ids),
num_frames=INFERENCE.num_frames,
seed=seed,
device=device,
)
source = src_latents.to(dtype=denoiser.dtype, device=device)
context = context.to(dtype=denoiser.dtype, device=device)
# Keep the reusable group on the host. The DiT stages one feature at a
# time while patch tokens are built, then releases that device scratch
# before the transformer blocks reach their peak allocation.
pose_feature_batch = pose_features.allocate_group(len(camera_ids), "cpu")
pose_features.copy_group(tuple(range(len(camera_ids))), pose_feature_batch)
null_pose_feature = pose_features.null_features
with torch.inference_mode(), _bf16_autocast():
for step_index, _ in enumerate(tqdm(denoiser.scheduler.timesteps, desc=f"RCP 1-to-{len(camera_ids)}")):
latents = denoise_group(
denoiser,
latents,
source,
context,
pose_feature_batch,
null_pose_feature,
step_index,
)
return latents.detach().to("cpu")
def _denoise_targets_single(
*,
denoiser: Denoiser,
src_latents: Tensor,
context: Tensor,
pose_features: PoseFeatureBank,
initial_latents: Tensor,
routes: Routes,
device: str,
) -> Tensor:
"""Denoise view groups on the GPU while the full target state stays on the CPU."""
import torch
from tqdm.auto import tqdm
num_views = initial_latents.shape[0]
if initial_latents.device.type != "cpu":
raise FourDAnyoneError("Canonical target latents must remain on the CPU between denoising groups.")
if pose_features.num_features != num_views:
raise FourDAnyoneError(
f"Target generation requires {num_views} pose features, got {pose_features.num_features}."
)
validate_routes(routes, num_views)
num_timesteps = len(denoiser.scheduler.timesteps)
if len(routes) != num_timesteps:
raise FourDAnyoneError(
f"Denoising requires one route per scheduler timestep; got "
f"{len(routes)} routes for {num_timesteps} timesteps."
)
latents = initial_latents
source = src_latents.to(dtype=denoiser.dtype, device=device)
context = context.to(dtype=denoiser.dtype, device=device)
null_pose_feature = pose_features.null_features
group_size = len(routes[0][0])
pose_feature_batch = pose_features.allocate_group(group_size, "cpu")
with torch.inference_mode(), _bf16_autocast():
for step_index, groups in enumerate(tqdm(routes, desc=f"Generate {num_views} target views")):
for view_indices in groups:
index = torch.tensor(view_indices, dtype=torch.long, device="cpu")
local_latents = torch.index_select(latents, 0, index).to(device)
pose_features.copy_group(view_indices, pose_feature_batch)
local_latents = denoise_group(
denoiser,
local_latents,
source,
context,
pose_feature_batch,
null_pose_feature,
step_index,
)
latents.index_copy_(0, index, local_latents.to("cpu"))
del local_latents
return latents
def _resolve_generation_plan(
*,
conditioning: Conditioning,
devices: tuple[str, ...],
) -> _GenerationPlan:
import torch
if conditioning.num_frames != INFERENCE.num_frames:
raise FourDAnyoneError("Generation requires the frozen 121-frame contract.")
if not devices:
raise FourDAnyoneError("Generation requires at least one CUDA device.")
view_plan = conditioning.view_plan
if len(conditioning.target_skeletons) != view_plan.num_target_views:
raise FourDAnyoneError("Target skeleton count does not match the resolved view plan.")
if len(conditioning.rcp_skeletons) != len(view_plan.rcp_camera_ids):
raise FourDAnyoneError("RCP skeleton count does not match the resolved view plan.")
primary_device = devices[0]
primary_device_index = int(primary_device.removeprefix("cuda:"))
dit_devices = select_worker_devices(devices, view_plan.num_groups)
LOGGER.info(
"Using %s (%s)",
primary_device,
torch.cuda.get_device_name(primary_device_index),
)
if len(dit_devices) < len(devices):
LOGGER.info(
"Using %d of %d candidate GPUs for DiT and all %d for independent view stages",
len(dit_devices),
len(devices),
len(devices),
)
return _GenerationPlan(
view_plan=view_plan,
candidate_devices=devices,
primary_device=primary_device,
dit_devices=dit_devices,
)
def _merge_view_stage_peak(metrics: GenerationMetrics, stage: str, vae: VaeExecutor) -> None:
if not vae.last_peak_vram_bytes:
return
metrics.merge_cuda_peak(
stage,
allocated_bytes=max(peak["allocated"] for peak in vae.last_peak_vram_bytes.values()),
reserved_bytes=max(peak["reserved"] for peak in vae.last_peak_vram_bytes.values()),
)
def _target_routes(*, view_plan: ViewPlan, profile: DenoisingProfile) -> Routes:
return routing_steps(
view_plan=view_plan,
num_steps=profile.num_inference_steps,
tcr_stride=profile.tcr_stride,
freeze_after_one_cycle=profile.freeze_tcr_after_one_cycle,
)
def _denoise_targets_multi_gpu(
*,
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
attention_backend: str,
plan: _GenerationPlan,
target_sources: Tensor,
context: Tensor,
initial_latents: Tensor,
pose_features: PoseFeatureBank,
routes: Routes,
root: Path,
) -> tuple[Tensor, ParallelismReport]:
with TemporaryDirectory(prefix=".distributed-", dir=root) as work_dir:
target_latents, workers = denoise_targets_distributed(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
denoising_profile=denoising_profile,
attention_backend=attention_backend,
src_latents=target_sources,
context=context,
initial_latents=initial_latents,
pose_features=pose_features,
routes=routes,
work_dir=work_dir,
devices=plan.dit_devices,
)
return target_latents, {
"backend": "nccl",
"candidate_devices": list(plan.candidate_devices),
"used_devices": list(plan.dit_devices),
"groups_per_step": plan.view_plan.num_groups,
"waves_per_step": math.ceil(plan.view_plan.num_groups / len(plan.dit_devices)),
"workers": workers,
}
def generate_views(
*,
conditioning: Conditioning,
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
attention_backend: str,
assets: BaseAssets,
output_dir: str | Path,
devices: tuple[str, ...],
seed: int,
) -> GeneratedViews:
"""Generate proposal and target views with the pipeline's resolved backend."""
import torch
if seed < 0:
raise FourDAnyoneError(f"seed must be non-negative, got {seed}.")
plan = _resolve_generation_plan(
conditioning=conditioning,
devices=devices,
)
root = Path(output_dir).expanduser().resolve()
root.mkdir(parents=True, exist_ok=False)
metrics = GenerationMetrics(plan.primary_device_index)
with metrics.stage("prompt"):
context = load_prompt_context(assets.prompt_context)
with metrics.stage("pose_conditioning"):
pose_cache = build_pose_feature_cache(
conditioning=conditioning,
checkpoint_path=checkpoint_path,
devices=plan.candidate_devices,
)
rcp_pose_features = pose_cache.rcp
target_pose_features = pose_cache.target
del pose_cache
denoiser = None
vae = None
try:
with metrics.stage("model_load"):
denoiser = (
load_denoiser(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
profile=denoising_profile,
attention_backend=attention_backend,
)
if plan.needs_primary_denoiser
else None
)
vae = VaeExecutor.load(assets.vae, plan.candidate_devices)
with metrics.stage("source_encode"):
source_video = _channels_last_source_layout(conditioning.load_source_tensor())
source_latents = vae.encode(source_video)
del source_video
_merge_view_stage_peak(metrics, "source_encode", vae)
target_sources = source_latents
if plan.view_plan.enable_rcp:
if denoiser is None or rcp_pose_features is None:
raise RuntimeError("RCP requires a primary denoiser and proposal pose features.")
with metrics.stage("rcp_denoise"):
denoiser.prepare_on_device(plan.primary_device)
_empty_cuda_cache()
rcp_latents = _denoise_rcp(
denoiser=denoiser,
vae=vae,
src_latents=source_latents,
context=context,
camera_ids=plan.view_plan.rcp_camera_ids,
pose_features=rcp_pose_features,
seed=seed,
device=plan.primary_device,
)
# Target references use null pose features, not the RCP bank.
del rcp_pose_features
# Only single-GPU target generation reuses the parent's DiT.
if plan.distributed:
denoiser = None
_empty_cuda_cache()
with metrics.stage("rcp_latent_handoff"):
target_sources = torch.cat([source_latents, rcp_latents[:4]], dim=0)
del rcp_latents
del source_latents
# Parallel VAE replicas are stage-local. Retaining only the prototype
# bounds parent host memory while distributed DiT workers load.
vae.release_replicas()
parallelism = None
with metrics.stage("target_denoise"):
routes = _target_routes(view_plan=plan.view_plan, profile=denoising_profile)
initial_latents = _noise(
vae=vae,
num_views=plan.view_plan.num_target_views,
num_frames=INFERENCE.num_frames,
seed=seed,
device="cpu",
)
if plan.distributed:
target_latents, parallelism = _denoise_targets_multi_gpu(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
denoising_profile=denoising_profile,
attention_backend=attention_backend,
plan=plan,
target_sources=target_sources,
context=context,
initial_latents=initial_latents,
pose_features=target_pose_features,
routes=routes,
root=root,
)
else:
if denoiser is None:
raise RuntimeError("Single-GPU target generation requires a primary-process denoiser.")
if not plan.view_plan.enable_rcp:
denoiser.prepare_on_device(plan.primary_device)
_empty_cuda_cache()
target_latents = _denoise_targets_single(
denoiser=denoiser,
src_latents=target_sources,
context=context,
pose_features=target_pose_features,
initial_latents=initial_latents,
routes=routes,
device=plan.primary_device,
)
denoiser = None
_empty_cuda_cache()
del initial_latents
if parallelism is not None:
workers = parallelism["workers"]
metrics.merge_cuda_peak(
"target_denoise",
allocated_bytes=max(int(worker["peak_vram_allocated_bytes"]) for worker in workers),
reserved_bytes=max(int(worker["peak_vram_reserved_bytes"]) for worker in workers),
)
del target_pose_features, target_sources, context
with metrics.stage("target_decode_and_publish"):
target_root = root / "target"
target_root.mkdir()
target_videos = vae.publish_targets(
target_latents, target_root, Fraction(conditioning.fps_num, conditioning.fps_den)
)
_merge_view_stage_peak(metrics, "target_decode_and_publish", vae)
result = GeneratedViews(
target_videos=target_videos,
view_plan=plan.view_plan,
denoising_profile=denoising_profile,
seed=seed,
device=plan.primary_device,
attention_backend=attention_backend,
elapsed_seconds=metrics.elapsed_seconds,
stage_peak_vram_bytes=metrics.stage_peak_vram_bytes,
peak_vram_allocated_bytes=metrics.peak_vram_allocated_bytes,
peak_vram_reserved_bytes=metrics.peak_vram_reserved_bytes,
parallelism=parallelism,
)
finally:
denoiser = None
if vae is not None:
vae.close()
_empty_cuda_cache()
return result