[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video
Release: default to SDPA on 24 GB GPUs (#31)
Yudong Jinc4fb0fab4d7c
diff --git a/fdanyone/attention.py b/fdanyone/attention.pynew file mode 100644index 0000000..e0c1435--- /dev/null+++ b/fdanyone/attention.py@@ -0,0 +1,29 @@+"""Pure validation and selection of the inference attention backend."""++from __future__ import annotations++from collections.abc import Mapping++from fdanyone.errors import ConfigurationError++ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")+ATTENTION_BACKENDS = ("auto", *ATTENTION_BACKEND_PRIORITY)+++def validate_attention_backend(backend: str) -> None:+ if not isinstance(backend, str) or backend not in ATTENTION_BACKENDS:+ raise ConfigurationError(f"attention_backend must be one of {', '.join(ATTENTION_BACKENDS)}, got {backend!r}.")+++def resolve_attention_backend(backend: str, availability: Mapping[str, bool]) -> str:+ """Honor explicit selection, otherwise use the installed-backend priority."""++ validate_attention_backend(backend)+ if backend == "auto":+ return next(candidate for candidate in ATTENTION_BACKEND_PRIORITY if availability[candidate])+ if not availability[backend]:+ raise ConfigurationError(+ f"Requested attention backend {backend!r} is not available. "+ "Install that backend or choose --attention_backend=sdpa."+ )+ return backenddiff --git a/fdanyone/device.py b/fdanyone/device.pyindex 390da71..1c49f39 100644--- a/fdanyone/device.py+++ b/fdanyone/device.py@@ -10,7 +10,7 @@ from fdanyone.errors import ConfigurationErrorCUDA_ALLOCATOR_CONF = "PYTORCH_CUDA_ALLOC_CONF"CUDA_MAX_SPLIT_SIZE_MB = 4096-CUDA_EXPANDABLE_SEGMENT_MAX_MEMORY_BYTES = 24 * 1024**3+LOW_MEMORY_GPU_MAX_BYTES = 24 * 1024**3def _visible_gpu_identifiers(environment: MutableMapping[str, str]) -> tuple[str, ...] | None:@@ -66,7 +66,7 @@ def _query_total_memory_bytes(identifiers: tuple[str, ...] | None) -> tuple[int,return tuple(total_mib * 1024**2 for total_mib in totals_mib)-def selected_gpus_need_expandable_segments(+def has_low_memory_gpu(gpu_ids: Sequence[int] | None,environment: MutableMapping[str, str] | None = None,) -> bool:@@ -75,7 +75,7 @@ def selected_gpus_need_expandable_segments(environment = os.environ if environment is None else environmentidentifiers = _selected_gpu_identifiers(gpu_ids, environment)totals = _query_total_memory_bytes(identifiers)- return bool(totals) and any(total <= CUDA_EXPANDABLE_SEGMENT_MAX_MEMORY_BYTES for total in totals)+ return any(total <= LOW_MEMORY_GPU_MAX_BYTES for total in totals)def configure_inference_cuda_allocator(diff --git a/fdanyone/model/distributed.py b/fdanyone/model/distributed.pyindex fe59daf..ee35ad5 100644--- a/fdanyone/model/distributed.py+++ b/fdanyone/model/distributed.py@@ -51,6 +51,7 @@ class DistributedDenoiseRequest:checkpoint_path: strturbo_lora_path: str | Nonedenoising_profile: DenoisingProfile+ attention_backend: strroutes: Routesdevices: tuple[str, ...]work_dir: str@@ -276,7 +277,6 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:import torch.distributed as distfrom fdanyone.model.loader import load_denoiser- from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backendlogging.basicConfig(level=logging.INFO,@@ -289,13 +289,13 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:device_index = int(device.removeprefix("cuda:"))torch.cuda.set_device(device_index)torch.cuda.reset_peak_memory_stats(device_index)- resolved_backend = get_attention_backend()model_started = time.monotonic()denoiser = load_denoiser(checkpoint_path=request.checkpoint_path,turbo_lora_path=request.turbo_lora_path,profile=request.denoising_profile,+ attention_backend=request.attention_backend,)denoiser.prepare_on_device(device)torch.cuda.synchronize(device_index)@@ -320,7 +320,7 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:"rank": rank,"device": device,"device_name": torch.cuda.get_device_name(device_index),- "attention_backend": resolved_backend,+ "attention_backend": denoiser.model.attention_backend,"model_load_seconds": model_load_seconds,"denoise_seconds": denoise_seconds,"peak_vram_allocated_bytes": int(torch.cuda.max_memory_allocated(device_index)),@@ -338,6 +338,7 @@ def denoise_targets_distributed(checkpoint_path: str | Path,turbo_lora_path: str | Path | None,denoising_profile: DenoisingProfile,+ attention_backend: str,routes: Routes,src_latents: Tensor,context: Tensor,@@ -382,6 +383,7 @@ def denoise_targets_distributed(checkpoint_path=str(Path(checkpoint_path).expanduser().resolve()),turbo_lora_path=(None if turbo_lora_path is None else str(Path(turbo_lora_path).expanduser().resolve())),denoising_profile=denoising_profile,+ attention_backend=attention_backend,routes=routes,devices=devices,work_dir=str(root),diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.pyindex 3f0e8c6..3ebe16b 100644--- a/fdanyone/model/inference.py+++ b/fdanyone/model/inference.py@@ -67,6 +67,7 @@ class GeneratedViews:denoising_profile: DenoisingProfileseed: intdevice: str+ attention_backend: strelapsed_seconds: dict[str, float]stage_peak_vram_bytes: dict[str, dict[str, int]]peak_vram_allocated_bytes: int@@ -376,6 +377,7 @@ def _denoise_targets_multi_gpu(checkpoint_path: str | Path,turbo_lora_path: str | Path | None,denoising_profile: DenoisingProfile,+ attention_backend: str,plan: _GenerationPlan,target_sources: Tensor,context: Tensor,@@ -389,6 +391,7 @@ def _denoise_targets_multi_gpu(checkpoint_path=checkpoint_path,turbo_lora_path=turbo_lora_path,denoising_profile=denoising_profile,+ attention_backend=attention_backend,src_latents=target_sources,context=context,initial_latents=initial_latents,@@ -414,12 +417,13 @@ def generate_views(checkpoint_path: str | Path,turbo_lora_path: str | Path | None,denoising_profile: DenoisingProfile,+ attention_backend: str,assets: BaseAssets,output_dir: str | Path,devices: tuple[str, ...],seed: int,) -> GeneratedViews:- """Generate the proposal (when enabled) and the requested target views."""+ """Generate proposal and target views with the pipeline's resolved backend."""if seed < 0:raise FourDAnyoneError(f"seed must be non-negative, got {seed}.")@@ -448,6 +452,7 @@ def generate_views(checkpoint_path=checkpoint_path,turbo_lora_path=turbo_lora_path,profile=denoising_profile,+ attention_backend=attention_backend,)if plan.needs_primary_denoiserelse None@@ -503,6 +508,7 @@ def generate_views(checkpoint_path=checkpoint_path,turbo_lora_path=turbo_lora_path,denoising_profile=denoising_profile,+ attention_backend=attention_backend,plan=plan,target_sources=target_sources,context=context,@@ -549,6 +555,7 @@ def generate_views(denoising_profile=denoising_profile,seed=seed,device=plan.primary_device,+ attention_backend=attention_backend,elapsed_seconds=metrics.elapsed_seconds,stage_peak_vram_bytes=metrics.stage_peak_vram_bytes,peak_vram_allocated_bytes=metrics.peak_vram_allocated_bytes,diff --git a/fdanyone/model/loader.py b/fdanyone/model/loader.pyindex d6a68fb..ecb2758 100644--- a/fdanyone/model/loader.py+++ b/fdanyone/model/loader.py@@ -79,7 +79,7 @@ def _strict_assign(module, state_dict: dict, label: str) -> None:)-def _load_dit(checkpoint_path: Path):+def _load_dit(checkpoint_path: Path, attention_backend: str):import torchfrom fdanyone.vendor.diffsynth.models.wan_video_dit import (@@ -90,7 +90,7 @@ def _load_dit(checkpoint_path: Path):)with torch.device("meta"):- dit = FourDAnyoneDiT()+ dit = FourDAnyoneDiT(attention_backend=attention_backend)state_dict, metadata = _load_checkpoint(checkpoint_path, exclude_prefixes=(POSE_ENCODER_PREFIX,))_strict_assign(dit, state_dict, "4DAnyone DiT checkpoint")del state_dict@@ -148,6 +148,7 @@ def load_denoiser(checkpoint_path: str | Path,turbo_lora_path: str | Path | None,profile: DenoisingProfile,+ attention_backend: str,) -> Denoiser:"""Load one DiT and configure its denoising trajectory."""@@ -162,7 +163,7 @@ def load_denoiser(turbo_path = None if turbo_lora_path is None else Path(turbo_lora_path).expanduser().resolve()if turbo_path is not None and not turbo_path.is_file():raise AssetError(f"Turbo LoRA does not exist: {turbo_path}")- model, metadata = _load_dit(checkpoint)+ model, metadata = _load_dit(checkpoint, attention_backend)if turbo_path is not None:from fdanyone.model.turbo_lora import validate_turbo_base_metadatadiff --git a/fdanyone/output.py b/fdanyone/output.pyindex 3e7fa55..c31e614 100644--- a/fdanyone/output.py+++ b/fdanyone/output.py@@ -112,10 +112,8 @@ def export_result() -> dict:"""Publish proposal, target, skeleton, camera, and metadata artifacts."""- from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backend-root = Path(destination).expanduser().resolve()- attention_backend = get_attention_backend()+ attention_backend = generated.attention_backendview_plan = generated.view_planif conditioning.view_plan != view_plan:raise FourDAnyoneError("Conditioning and generation resolved different view plans.")diff --git a/fdanyone/pipeline.py b/fdanyone/pipeline.pyindex f03c32e..ec52e8f 100644--- a/fdanyone/pipeline.py+++ b/fdanyone/pipeline.py@@ -182,6 +182,7 @@ def run_pipeline(mhr70_regressor_path: str | None,gvhmr_root: str,gpu_ids: list[int] | None,+ attention_backend: str,start_time: float,target_fps: str | int | float,seed: int,@@ -224,6 +225,13 @@ def run_pipeline(devices = select_cuda_devices(gpu_ids)device = devices[0]+ from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backend++ # Resolve once, before downloading assets or preparing conditioning. Every+ # DiT, including spawned replicas, receives this concrete backend.+ attention_backend = get_attention_backend(attention_backend)+ LOGGER.info("Using attention backend: %s", attention_backend)+ensure_example_video(video_path)# Resolve the licensed body model before starting the much larger public# model download. Interactive use continues automatically after setup;@@ -323,6 +331,7 @@ def run_pipeline(checkpoint_path=checkpoint,turbo_lora_path=turbo_lora,denoising_profile=denoising_profile,+ attention_backend=attention_backend,assets=base_assets,output_dir=scratch / "generation",devices=devices,diff --git a/fdanyone/vendor/diffsynth/models/wan_video_dit.py b/fdanyone/vendor/diffsynth/models/wan_video_dit.pyindex 2c083bb..dba175b 100644--- a/fdanyone/vendor/diffsynth/models/wan_video_dit.py+++ b/fdanyone/vendor/diffsynth/models/wan_video_dit.py@@ -16,6 +16,8 @@ import torch.nn as nnimport torch.nn.functional as Ffrom einops import rearrange, repeat+from fdanyone.attention import resolve_attention_backend+try:import flash_attn_interface@@ -53,24 +55,22 @@ RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2# linear consumes them as BF16. Bound the three coexisting FP32 tensors while# preserving that final cast boundary.NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2-ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")-def get_attention_backend() -> str:- """Return the implementation selected by the release auto policy."""+def get_attention_backend(backend: str = "auto") -> str:+ """Resolve a requested backend against the installed implementations."""availability = {"flash_attn_3": FLASH_ATTN_3_AVAILABLE,"sageattention": SAGE_ATTN_AVAILABLE,"sdpa": True,}- return next(backend for backend in ATTENTION_BACKEND_PRIORITY if availability[backend])+ return resolve_attention_backend(backend, availability)-def attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int) -> torch.Tensor:- """Evaluate attention with the backend selected by the release policy."""+def attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, backend: str) -> torch.Tensor:+ """Evaluate attention with the owning model's resolved backend."""- backend = get_attention_backend()if backend == "flash_attn_3":q = rearrange(q, "b s (n d) -> b s n d", n=num_heads)k = rearrange(k, "b s (n d) -> b s n d", n=num_heads)@@ -240,9 +240,10 @@ class RMSNorm(nn.Module):class SelfAttention(nn.Module):- def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:+ def __init__(self, dim: int, num_heads: int, *, attention_backend: str, eps: float = NORM_EPSILON) -> None:super().__init__()self.num_heads = num_heads+ self.attention_backend = attention_backendself.q = nn.Linear(dim, dim)self.k = nn.Linear(dim, dim)self.v = nn.Linear(dim, dim)@@ -256,13 +257,14 @@ class SelfAttention(nn.Module):v = self.v(x)q = rope_apply(q, freqs, self.num_heads)k = rope_apply(k, freqs, self.num_heads)- return self.o(attention(q, k, v, self.num_heads))+ return self.o(attention(q, k, v, self.num_heads, self.attention_backend))class CrossAttention(nn.Module):- def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:+ def __init__(self, dim: int, num_heads: int, *, attention_backend: str, eps: float = NORM_EPSILON) -> None:super().__init__()self.num_heads = num_heads+ self.attention_backend = attention_backendself.q = nn.Linear(dim, dim)self.k = nn.Linear(dim, dim)self.v = nn.Linear(dim, dim)@@ -274,16 +276,16 @@ class CrossAttention(nn.Module):q = self.norm_q.forward_inplace(self.q(x))k = self.norm_k.forward_inplace(self.k(context))v = self.v(context)- return self.o(attention(q, k, v, self.num_heads))+ return self.o(attention(q, k, v, self.num_heads, self.attention_backend))class DiTBlock(nn.Module):"""One frozen video + multiview + prompt + FFN transformer block."""- def __init__(self) -> None:+ def __init__(self, *, attention_backend: str) -> None:super().__init__()- self.self_attn = SelfAttention(MODEL_DIM, NUM_HEADS)- self.cross_attn = CrossAttention(MODEL_DIM, NUM_HEADS)+ self.self_attn = SelfAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)+ self.cross_attn = CrossAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)self.norm1 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)self.norm2 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)self.norm3 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON)@@ -296,7 +298,7 @@ class DiTBlock(nn.Module):self.modulation_mvs = nn.Parameter(torch.randn(1, 3, MODEL_DIM) / MODEL_DIM**0.5)self.norm1_mvs = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)- self.self_attn_mvs = SelfAttention(MODEL_DIM, NUM_HEADS)+ self.self_attn_mvs = SelfAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)def _feed_forward(self, x: torch.Tensor) -> torch.Tensor:hidden = self.ffn[0](x)@@ -424,8 +426,9 @@ class ViewPackEmbedding(nn.Module):class FourDAnyoneDiT(nn.Module):"""Exact immutable inference graph for the released model checkpoint."""- def __init__(self) -> None:+ def __init__(self, *, attention_backend: str = "auto") -> None:super().__init__()+ self.attention_backend = get_attention_backend(attention_backend)self.patch_embedding = nn.Conv3d(LATENT_CHANNELS,MODEL_DIM,@@ -443,7 +446,7 @@ class FourDAnyoneDiT(nn.Module):nn.Linear(MODEL_DIM, MODEL_DIM),)self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(MODEL_DIM, MODEL_DIM * 6))- self.blocks = nn.ModuleList(DiTBlock() for _ in range(NUM_LAYERS))+ self.blocks = nn.ModuleList(DiTBlock(attention_backend=self.attention_backend) for _ in range(NUM_LAYERS))self.head = Head()self.viewpack_embedding = ViewPackEmbedding()self.freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor] = ()diff --git a/inference.py b/inference.pyindex ae2929b..2c3a7df 100644--- a/inference.py+++ b/inference.py@@ -4,7 +4,8 @@ from __future__ import annotationsimport sys-from fdanyone.device import configure_inference_cuda_allocator, selected_gpus_need_expandable_segments+from fdanyone.attention import validate_attention_backend+from fdanyone.device import configure_inference_cuda_allocator, has_low_memory_gpufrom fdanyone.errors import FourDAnyoneError@@ -24,6 +25,7 @@ def inference(mhr70_regressor_path: str | None = None,gvhmr_root: str = "third_party/GVHMR",gpu_ids: list[int] | None = None,+ attention_backend: str = "auto",target_fps: str | int | float = "auto",start_time: float = 0.0,seed: int = 42,@@ -55,6 +57,9 @@ def inference(gvhmr_root: Path to the GVHMR source checkout.gpu_ids: GPU IDs used for parallel pose/VAE view stages and targetdenoising. Omit to use all visible GPUs.+ attention_backend: auto uses SDPA when any selected GPU has at most+ 24 GiB; otherwise it prefers FlashAttention-3, then SageAttention,+ then SDPA. Override with sdpa, sageattention, or flash_attn_3.target_fps: auto preserves the input clock unless it divides evenlyto 24, 25, or 30 FPS; a positive number requests an explicit FPS.start_time: Clip start time on the input timeline, in seconds.@@ -63,9 +68,11 @@ def inference(# This must run before the first model/PyTorch import. It protects the# reusable 5--6 GiB DiT FFN allocation from allocator fragmentation.- configure_inference_cuda_allocator(- use_expandable_segments=selected_gpus_need_expandable_segments(gpu_ids),- )+ validate_attention_backend(attention_backend)+ low_memory = has_low_memory_gpu(gpu_ids)+ configure_inference_cuda_allocator(use_expandable_segments=low_memory)+ if attention_backend == "auto" and low_memory:+ attention_backend = "sdpa"# Keep model imports out of module scope so ``--help`` stays lightweight.from fdanyone.pipeline import run_pipeline@@ -85,6 +92,7 @@ def inference(mhr70_regressor_path=mhr70_regressor_path,gvhmr_root=gvhmr_root,gpu_ids=gpu_ids,+ attention_backend=attention_backend,target_fps=target_fps,start_time=start_time,seed=seed,