Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

Release: default to SDPA on 24 GB GPUs (#31)

Yudong Jinc4fb0fab4d7c
diff --git a/fdanyone/attention.py b/fdanyone/attention.py
new file mode 100644
index 0000000..e0c1435
--- /dev/null
+++ b/fdanyone/attention.py
@@ -0,0 +1,29 @@
+"""Pure validation and selection of the inference attention backend."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from fdanyone.errors import ConfigurationError
+
+ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")
+ATTENTION_BACKENDS = ("auto", *ATTENTION_BACKEND_PRIORITY)
+
+
+def validate_attention_backend(backend: str) -> None:
+ if not isinstance(backend, str) or backend not in ATTENTION_BACKENDS:
+ raise ConfigurationError(f"attention_backend must be one of {', '.join(ATTENTION_BACKENDS)}, got {backend!r}.")
+
+
+def resolve_attention_backend(backend: str, availability: Mapping[str, bool]) -> str:
+ """Honor explicit selection, otherwise use the installed-backend priority."""
+
+ validate_attention_backend(backend)
+ if backend == "auto":
+ return next(candidate for candidate in ATTENTION_BACKEND_PRIORITY if availability[candidate])
+ if not availability[backend]:
+ raise ConfigurationError(
+ f"Requested attention backend {backend!r} is not available. "
+ "Install that backend or choose --attention_backend=sdpa."
+ )
+ return backend
diff --git a/fdanyone/device.py b/fdanyone/device.py
index 390da71..1c49f39 100644
--- a/fdanyone/device.py
+++ b/fdanyone/device.py
@@ -10,7 +10,7 @@ from fdanyone.errors import ConfigurationError
CUDA_ALLOCATOR_CONF = "PYTORCH_CUDA_ALLOC_CONF"
CUDA_MAX_SPLIT_SIZE_MB = 4096
-CUDA_EXPANDABLE_SEGMENT_MAX_MEMORY_BYTES = 24 * 1024**3
+LOW_MEMORY_GPU_MAX_BYTES = 24 * 1024**3
def _visible_gpu_identifiers(environment: MutableMapping[str, str]) -> tuple[str, ...] | None:
@@ -66,7 +66,7 @@ def _query_total_memory_bytes(identifiers: tuple[str, ...] | None) -> tuple[int,
return tuple(total_mib * 1024**2 for total_mib in totals_mib)
-def selected_gpus_need_expandable_segments(
+def has_low_memory_gpu(
gpu_ids: Sequence[int] | None,
environment: MutableMapping[str, str] | None = None,
) -> bool:
@@ -75,7 +75,7 @@ def selected_gpus_need_expandable_segments(
environment = os.environ if environment is None else environment
identifiers = _selected_gpu_identifiers(gpu_ids, environment)
totals = _query_total_memory_bytes(identifiers)
- return bool(totals) and any(total <= CUDA_EXPANDABLE_SEGMENT_MAX_MEMORY_BYTES for total in totals)
+ return any(total <= LOW_MEMORY_GPU_MAX_BYTES for total in totals)
def configure_inference_cuda_allocator(
diff --git a/fdanyone/model/distributed.py b/fdanyone/model/distributed.py
index fe59daf..ee35ad5 100644
--- a/fdanyone/model/distributed.py
+++ b/fdanyone/model/distributed.py
@@ -51,6 +51,7 @@ class DistributedDenoiseRequest:
checkpoint_path: str
turbo_lora_path: str | None
denoising_profile: DenoisingProfile
+ attention_backend: str
routes: Routes
devices: tuple[str, ...]
work_dir: str
@@ -276,7 +277,6 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:
import torch.distributed as dist
from fdanyone.model.loader import load_denoiser
- from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backend
logging.basicConfig(
level=logging.INFO,
@@ -289,13 +289,13 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:
device_index = int(device.removeprefix("cuda:"))
torch.cuda.set_device(device_index)
torch.cuda.reset_peak_memory_stats(device_index)
- resolved_backend = get_attention_backend()
model_started = time.monotonic()
denoiser = load_denoiser(
checkpoint_path=request.checkpoint_path,
turbo_lora_path=request.turbo_lora_path,
profile=request.denoising_profile,
+ attention_backend=request.attention_backend,
)
denoiser.prepare_on_device(device)
torch.cuda.synchronize(device_index)
@@ -320,7 +320,7 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:
"rank": rank,
"device": device,
"device_name": torch.cuda.get_device_name(device_index),
- "attention_backend": resolved_backend,
+ "attention_backend": denoiser.model.attention_backend,
"model_load_seconds": model_load_seconds,
"denoise_seconds": denoise_seconds,
"peak_vram_allocated_bytes": int(torch.cuda.max_memory_allocated(device_index)),
@@ -338,6 +338,7 @@ def denoise_targets_distributed(
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
+ attention_backend: str,
routes: Routes,
src_latents: Tensor,
context: Tensor,
@@ -382,6 +383,7 @@ def denoise_targets_distributed(
checkpoint_path=str(Path(checkpoint_path).expanduser().resolve()),
turbo_lora_path=(None if turbo_lora_path is None else str(Path(turbo_lora_path).expanduser().resolve())),
denoising_profile=denoising_profile,
+ attention_backend=attention_backend,
routes=routes,
devices=devices,
work_dir=str(root),
diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.py
index 3f0e8c6..3ebe16b 100644
--- a/fdanyone/model/inference.py
+++ b/fdanyone/model/inference.py
@@ -67,6 +67,7 @@ class GeneratedViews:
denoising_profile: DenoisingProfile
seed: int
device: str
+ attention_backend: str
elapsed_seconds: dict[str, float]
stage_peak_vram_bytes: dict[str, dict[str, int]]
peak_vram_allocated_bytes: int
@@ -376,6 +377,7 @@ def _denoise_targets_multi_gpu(
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
+ attention_backend: str,
plan: _GenerationPlan,
target_sources: Tensor,
context: Tensor,
@@ -389,6 +391,7 @@ def _denoise_targets_multi_gpu(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
denoising_profile=denoising_profile,
+ attention_backend=attention_backend,
src_latents=target_sources,
context=context,
initial_latents=initial_latents,
@@ -414,12 +417,13 @@ def generate_views(
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
+ attention_backend: str,
assets: BaseAssets,
output_dir: str | Path,
devices: tuple[str, ...],
seed: int,
) -> GeneratedViews:
- """Generate the proposal (when enabled) and the requested target views."""
+ """Generate proposal and target views with the pipeline's resolved backend."""
if seed < 0:
raise FourDAnyoneError(f"seed must be non-negative, got {seed}.")
@@ -448,6 +452,7 @@ def generate_views(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
profile=denoising_profile,
+ attention_backend=attention_backend,
)
if plan.needs_primary_denoiser
else None
@@ -503,6 +508,7 @@ def generate_views(
checkpoint_path=checkpoint_path,
turbo_lora_path=turbo_lora_path,
denoising_profile=denoising_profile,
+ attention_backend=attention_backend,
plan=plan,
target_sources=target_sources,
context=context,
@@ -549,6 +555,7 @@ def generate_views(
denoising_profile=denoising_profile,
seed=seed,
device=plan.primary_device,
+ attention_backend=attention_backend,
elapsed_seconds=metrics.elapsed_seconds,
stage_peak_vram_bytes=metrics.stage_peak_vram_bytes,
peak_vram_allocated_bytes=metrics.peak_vram_allocated_bytes,
diff --git a/fdanyone/model/loader.py b/fdanyone/model/loader.py
index d6a68fb..ecb2758 100644
--- a/fdanyone/model/loader.py
+++ b/fdanyone/model/loader.py
@@ -79,7 +79,7 @@ def _strict_assign(module, state_dict: dict, label: str) -> None:
)
-def _load_dit(checkpoint_path: Path):
+def _load_dit(checkpoint_path: Path, attention_backend: str):
import torch
from fdanyone.vendor.diffsynth.models.wan_video_dit import (
@@ -90,7 +90,7 @@ def _load_dit(checkpoint_path: Path):
)
with torch.device("meta"):
- dit = FourDAnyoneDiT()
+ dit = FourDAnyoneDiT(attention_backend=attention_backend)
state_dict, metadata = _load_checkpoint(checkpoint_path, exclude_prefixes=(POSE_ENCODER_PREFIX,))
_strict_assign(dit, state_dict, "4DAnyone DiT checkpoint")
del state_dict
@@ -148,6 +148,7 @@ def load_denoiser(
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
profile: DenoisingProfile,
+ attention_backend: str,
) -> Denoiser:
"""Load one DiT and configure its denoising trajectory."""
@@ -162,7 +163,7 @@ def load_denoiser(
turbo_path = None if turbo_lora_path is None else Path(turbo_lora_path).expanduser().resolve()
if turbo_path is not None and not turbo_path.is_file():
raise AssetError(f"Turbo LoRA does not exist: {turbo_path}")
- model, metadata = _load_dit(checkpoint)
+ model, metadata = _load_dit(checkpoint, attention_backend)
if turbo_path is not None:
from fdanyone.model.turbo_lora import validate_turbo_base_metadata
diff --git a/fdanyone/output.py b/fdanyone/output.py
index 3e7fa55..c31e614 100644
--- a/fdanyone/output.py
+++ b/fdanyone/output.py
@@ -112,10 +112,8 @@ def export_result(
) -> dict:
"""Publish proposal, target, skeleton, camera, and metadata artifacts."""
- from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backend
-
root = Path(destination).expanduser().resolve()
- attention_backend = get_attention_backend()
+ attention_backend = generated.attention_backend
view_plan = generated.view_plan
if conditioning.view_plan != view_plan:
raise FourDAnyoneError("Conditioning and generation resolved different view plans.")
diff --git a/fdanyone/pipeline.py b/fdanyone/pipeline.py
index f03c32e..ec52e8f 100644
--- a/fdanyone/pipeline.py
+++ b/fdanyone/pipeline.py
@@ -182,6 +182,7 @@ def run_pipeline(
mhr70_regressor_path: str | None,
gvhmr_root: str,
gpu_ids: list[int] | None,
+ attention_backend: str,
start_time: float,
target_fps: str | int | float,
seed: int,
@@ -224,6 +225,13 @@ def run_pipeline(
devices = select_cuda_devices(gpu_ids)
device = devices[0]
+ from fdanyone.vendor.diffsynth.models.wan_video_dit import get_attention_backend
+
+ # Resolve once, before downloading assets or preparing conditioning. Every
+ # DiT, including spawned replicas, receives this concrete backend.
+ attention_backend = get_attention_backend(attention_backend)
+ LOGGER.info("Using attention backend: %s", attention_backend)
+
ensure_example_video(video_path)
# Resolve the licensed body model before starting the much larger public
# model download. Interactive use continues automatically after setup;
@@ -323,6 +331,7 @@ def run_pipeline(
checkpoint_path=checkpoint,
turbo_lora_path=turbo_lora,
denoising_profile=denoising_profile,
+ attention_backend=attention_backend,
assets=base_assets,
output_dir=scratch / "generation",
devices=devices,
diff --git a/fdanyone/vendor/diffsynth/models/wan_video_dit.py b/fdanyone/vendor/diffsynth/models/wan_video_dit.py
index 2c083bb..dba175b 100644
--- a/fdanyone/vendor/diffsynth/models/wan_video_dit.py
+++ b/fdanyone/vendor/diffsynth/models/wan_video_dit.py
@@ -16,6 +16,8 @@ import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange, repeat
+from fdanyone.attention import resolve_attention_backend
+
try:
import flash_attn_interface
@@ -53,24 +55,22 @@ RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2
# linear consumes them as BF16. Bound the three coexisting FP32 tensors while
# preserving that final cast boundary.
NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2
-ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")
-def get_attention_backend() -> str:
- """Return the implementation selected by the release auto policy."""
+def get_attention_backend(backend: str = "auto") -> str:
+ """Resolve a requested backend against the installed implementations."""
availability = {
"flash_attn_3": FLASH_ATTN_3_AVAILABLE,
"sageattention": SAGE_ATTN_AVAILABLE,
"sdpa": True,
}
- return next(backend for backend in ATTENTION_BACKEND_PRIORITY if availability[backend])
+ return resolve_attention_backend(backend, availability)
-def attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int) -> torch.Tensor:
- """Evaluate attention with the backend selected by the release policy."""
+def attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, num_heads: int, backend: str) -> torch.Tensor:
+ """Evaluate attention with the owning model's resolved backend."""
- backend = get_attention_backend()
if backend == "flash_attn_3":
q = rearrange(q, "b s (n d) -> b s n d", n=num_heads)
k = rearrange(k, "b s (n d) -> b s n d", n=num_heads)
@@ -240,9 +240,10 @@ class RMSNorm(nn.Module):
class SelfAttention(nn.Module):
- def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:
+ def __init__(self, dim: int, num_heads: int, *, attention_backend: str, eps: float = NORM_EPSILON) -> None:
super().__init__()
self.num_heads = num_heads
+ self.attention_backend = attention_backend
self.q = nn.Linear(dim, dim)
self.k = nn.Linear(dim, dim)
self.v = nn.Linear(dim, dim)
@@ -256,13 +257,14 @@ class SelfAttention(nn.Module):
v = self.v(x)
q = rope_apply(q, freqs, self.num_heads)
k = rope_apply(k, freqs, self.num_heads)
- return self.o(attention(q, k, v, self.num_heads))
+ return self.o(attention(q, k, v, self.num_heads, self.attention_backend))
class CrossAttention(nn.Module):
- def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:
+ def __init__(self, dim: int, num_heads: int, *, attention_backend: str, eps: float = NORM_EPSILON) -> None:
super().__init__()
self.num_heads = num_heads
+ self.attention_backend = attention_backend
self.q = nn.Linear(dim, dim)
self.k = nn.Linear(dim, dim)
self.v = nn.Linear(dim, dim)
@@ -274,16 +276,16 @@ class CrossAttention(nn.Module):
q = self.norm_q.forward_inplace(self.q(x))
k = self.norm_k.forward_inplace(self.k(context))
v = self.v(context)
- return self.o(attention(q, k, v, self.num_heads))
+ return self.o(attention(q, k, v, self.num_heads, self.attention_backend))
class DiTBlock(nn.Module):
"""One frozen video + multiview + prompt + FFN transformer block."""
- def __init__(self) -> None:
+ def __init__(self, *, attention_backend: str) -> None:
super().__init__()
- self.self_attn = SelfAttention(MODEL_DIM, NUM_HEADS)
- self.cross_attn = CrossAttention(MODEL_DIM, NUM_HEADS)
+ self.self_attn = SelfAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)
+ self.cross_attn = CrossAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)
self.norm1 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)
self.norm2 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)
self.norm3 = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON)
@@ -296,7 +298,7 @@ class DiTBlock(nn.Module):
self.modulation_mvs = nn.Parameter(torch.randn(1, 3, MODEL_DIM) / MODEL_DIM**0.5)
self.norm1_mvs = nn.LayerNorm(MODEL_DIM, eps=NORM_EPSILON, elementwise_affine=False)
- self.self_attn_mvs = SelfAttention(MODEL_DIM, NUM_HEADS)
+ self.self_attn_mvs = SelfAttention(MODEL_DIM, NUM_HEADS, attention_backend=attention_backend)
def _feed_forward(self, x: torch.Tensor) -> torch.Tensor:
hidden = self.ffn[0](x)
@@ -424,8 +426,9 @@ class ViewPackEmbedding(nn.Module):
class FourDAnyoneDiT(nn.Module):
"""Exact immutable inference graph for the released model checkpoint."""
- def __init__(self) -> None:
+ def __init__(self, *, attention_backend: str = "auto") -> None:
super().__init__()
+ self.attention_backend = get_attention_backend(attention_backend)
self.patch_embedding = nn.Conv3d(
LATENT_CHANNELS,
MODEL_DIM,
@@ -443,7 +446,7 @@ class FourDAnyoneDiT(nn.Module):
nn.Linear(MODEL_DIM, MODEL_DIM),
)
self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(MODEL_DIM, MODEL_DIM * 6))
- self.blocks = nn.ModuleList(DiTBlock() for _ in range(NUM_LAYERS))
+ self.blocks = nn.ModuleList(DiTBlock(attention_backend=self.attention_backend) for _ in range(NUM_LAYERS))
self.head = Head()
self.viewpack_embedding = ViewPackEmbedding()
self.freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor] = ()
diff --git a/inference.py b/inference.py
index ae2929b..2c3a7df 100644
--- a/inference.py
+++ b/inference.py
@@ -4,7 +4,8 @@ from __future__ import annotations
import sys
-from fdanyone.device import configure_inference_cuda_allocator, selected_gpus_need_expandable_segments
+from fdanyone.attention import validate_attention_backend
+from fdanyone.device import configure_inference_cuda_allocator, has_low_memory_gpu
from fdanyone.errors import FourDAnyoneError
@@ -24,6 +25,7 @@ def inference(
mhr70_regressor_path: str | None = None,
gvhmr_root: str = "third_party/GVHMR",
gpu_ids: list[int] | None = None,
+ attention_backend: str = "auto",
target_fps: str | int | float = "auto",
start_time: float = 0.0,
seed: int = 42,
@@ -55,6 +57,9 @@ def inference(
gvhmr_root: Path to the GVHMR source checkout.
gpu_ids: GPU IDs used for parallel pose/VAE view stages and target
denoising. Omit to use all visible GPUs.
+ attention_backend: auto uses SDPA when any selected GPU has at most
+ 24 GiB; otherwise it prefers FlashAttention-3, then SageAttention,
+ then SDPA. Override with sdpa, sageattention, or flash_attn_3.
target_fps: auto preserves the input clock unless it divides evenly
to 24, 25, or 30 FPS; a positive number requests an explicit FPS.
start_time: Clip start time on the input timeline, in seconds.
@@ -63,9 +68,11 @@ def inference(
# This must run before the first model/PyTorch import. It protects the
# reusable 5--6 GiB DiT FFN allocation from allocator fragmentation.
- configure_inference_cuda_allocator(
- use_expandable_segments=selected_gpus_need_expandable_segments(gpu_ids),
- )
+ validate_attention_backend(attention_backend)
+ low_memory = has_low_memory_gpu(gpu_ids)
+ configure_inference_cuda_allocator(use_expandable_segments=low_memory)
+ if attention_backend == "auto" and low_memory:
+ attention_backend = "sdpa"
# Keep model imports out of module scope so ``--help`` stays lightweight.
from fdanyone.pipeline import run_pipeline
@@ -85,6 +92,7 @@ def inference(
mhr70_regressor_path=mhr70_regressor_path,
gvhmr_root=gvhmr_root,
gpu_ids=gpu_ids,
+ attention_backend=attention_backend,
target_fps=target_fps,
start_time=start_time,
seed=seed,