Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

Release: reduce inference memory and clarify models (#26)

Yudong Jinf6599d89a256
diff --git a/README.md b/README.md
index fb81f9d..225333a 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ python scripts/download_example.py
## Inference
-This release provides two models: **4DAnyone-Base**, corresponding to the original model described in the paper, and **4DAnyone-Turbo**, the default accelerated model. Set `--enable_turbo=False` to use **4DAnyone-Base**.
+We provide two models: **4DAnyone-Turbo** for faster four-step denoising and **4DAnyone-Base** with the standard denoising schedule. Both models achieve comparable generation quality. 4DAnyone-Turbo is enabled by default. Set `--enable_turbo=False` to use 4DAnyone-Base.
4DAnyone supports flexible target-view counts, pitch layers, and yaw coverage. Here are several common camera configurations:
diff --git a/fdanyone/model/conditioning.py b/fdanyone/model/conditioning.py
index 0eb2cfa..28891a6 100644
--- a/fdanyone/model/conditioning.py
+++ b/fdanyone/model/conditioning.py
@@ -157,7 +157,7 @@ class PoseFeatureBank:
return int(self.null_features.shape[0])
def allocate_group(self, size: int, device: str) -> Tensor:
- """Allocate one reusable GPU group buffer."""
+ """Allocate one reusable group buffer on the requested device."""
import torch
diff --git a/fdanyone/model/distributed.py b/fdanyone/model/distributed.py
index e1358c9..fe59daf 100644
--- a/fdanyone/model/distributed.py
+++ b/fdanyone/model/distributed.py
@@ -250,7 +250,7 @@ def _load_worker_state(rank: int, request: DistributedDenoiseRequest, denoiser:
pose_feature_batch=torch.empty(
(group_size, *request.pose_feature_shape[1:]),
dtype=denoiser.dtype,
- device=device,
+ device="cpu",
),
latent_tail=tuple(payload["initial_latents"].shape[1:]),
)
diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.py
index 1429acb..3f0e8c6 100644
--- a/fdanyone/model/inference.py
+++ b/fdanyone/model/inference.py
@@ -181,7 +181,10 @@ def _denoise_rcp(
)
source = src_latents.to(dtype=denoiser.dtype, device=device)
context = context.to(dtype=denoiser.dtype, device=device)
- pose_feature_batch = pose_features.allocate_group(len(camera_ids), device)
+ # Keep the reusable group on the host. The DiT stages one feature at a
+ # time while patch tokens are built, then releases that device scratch
+ # before the transformer blocks reach their peak allocation.
+ pose_feature_batch = pose_features.allocate_group(len(camera_ids), "cpu")
pose_features.copy_group(tuple(range(len(camera_ids))), pose_feature_batch)
null_pose_feature = pose_features.null_on(device)
@@ -229,7 +232,7 @@ def _denoise_targets_single(
context = context.to(dtype=denoiser.dtype, device=device)
null_pose_feature = pose_features.null_on(device)
group_size = len(routes[0][0])
- pose_feature_batch = pose_features.allocate_group(group_size, device)
+ pose_feature_batch = pose_features.allocate_group(group_size, "cpu")
with torch.inference_mode(), _bf16_autocast():
for step_index, groups in enumerate(tqdm(routes, desc=f"Generate {num_views} target views")):
diff --git a/fdanyone/vendor/diffsynth/models/wan_video_dit.py b/fdanyone/vendor/diffsynth/models/wan_video_dit.py
index cb78f25..9dcb71f 100644
--- a/fdanyone/vendor/diffsynth/models/wan_video_dit.py
+++ b/fdanyone/vendor/diffsynth/models/wan_video_dit.py
@@ -45,6 +45,14 @@ NORM_EPSILON = 1e-6
# gives the same one-view production chunk as the original 768 MiB
# single-allocation limit, while naming the actual temporary budget.
ROPE_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2
+# RMSNorm keeps its FP32 input alive while materializing either the squared
+# values or normalized result. Bound those two full-width temporaries without
+# changing the reduction dimension or arithmetic used by the checkpoint.
+RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2
+# CUDA autocast keeps LayerNorm and the following modulation in FP32 until a
+# linear consumes them as BF16. Bound the three coexisting FP32 tensors while
+# preserving that final cast boundary.
+NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2
ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")
@@ -82,6 +90,48 @@ def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch
return x * (1 + scale) + shift
+def _normalized_modulation_chunk(
+ norm: nn.Module,
+ x: torch.Tensor,
+ shift: torch.Tensor,
+ scale: torch.Tensor,
+ output: torch.Tensor,
+) -> None:
+ """Finish one FP32 normalization/modulation slice in a local scope."""
+
+ output.copy_(modulate(norm(x), shift, scale))
+
+
+def normalized_modulation(
+ norm: nn.Module,
+ x: torch.Tensor,
+ shift: torch.Tensor,
+ scale: torch.Tensor,
+) -> torch.Tensor:
+ """Apply LayerNorm modulation with bounded inference temporaries."""
+
+ if torch.is_grad_enabled():
+ return modulate(norm(x), shift, scale)
+
+ output = torch.empty_like(x)
+ fp32_bytes_per_batch = x[0].numel() * torch.empty((), dtype=torch.float32).element_size()
+ temporary_bytes_per_batch = 3 * fp32_bytes_per_batch
+ batch_chunk = max(
+ 1,
+ min(x.shape[0], NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES // temporary_bytes_per_batch),
+ )
+ for start in range(0, x.shape[0], batch_chunk):
+ end = start + batch_chunk
+ _normalized_modulation_chunk(
+ norm,
+ x[start:end],
+ shift[start:end],
+ scale[start:end],
+ output[start:end],
+ )
+ return output
+
+
def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:
sinusoid = torch.outer(
position.to(torch.float64),
@@ -164,6 +214,30 @@ class RMSNorm(nn.Module):
dtype = x.dtype
return self._normalize_float(x.float()).to(dtype) * self.weight
+ def _normalize_chunk_inplace(self, chunk: torch.Tensor) -> None:
+ """Normalize one projection slice and end its temporary lifetime here."""
+
+ chunk.copy_(self.forward(chunk))
+
+ def forward_inplace(self, x: torch.Tensor) -> torch.Tensor:
+ """Normalize an inference-owned projection with bounded FP32 storage."""
+
+ if torch.is_grad_enabled():
+ return self.forward(x)
+ if not x.is_contiguous():
+ raise ValueError("In-place RMSNorm requires a contiguous projection buffer.")
+
+ rows = x.view(-1, x.shape[-1])
+ fp32_bytes_per_row = x.shape[-1] * torch.empty((), dtype=torch.float32).element_size()
+ temporary_bytes_per_row = 2 * fp32_bytes_per_row
+ rows_per_chunk = max(
+ 1,
+ min(rows.shape[0], RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES // temporary_bytes_per_row),
+ )
+ for start in range(0, rows.shape[0], rows_per_chunk):
+ self._normalize_chunk_inplace(rows[start : start + rows_per_chunk])
+ return x
+
class SelfAttention(nn.Module):
def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:
@@ -177,8 +251,8 @@ class SelfAttention(nn.Module):
self.norm_k = RMSNorm(dim, eps=eps)
def forward(self, x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
- q = self.norm_q(self.q(x))
- k = self.norm_k(self.k(x))
+ q = self.norm_q.forward_inplace(self.q(x))
+ k = self.norm_k.forward_inplace(self.k(x))
v = self.v(x)
q = rope_apply(q, freqs, self.num_heads)
k = rope_apply(k, freqs, self.num_heads)
@@ -197,8 +271,8 @@ class CrossAttention(nn.Module):
self.norm_k = RMSNorm(dim, eps=eps)
def forward(self, x: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
- q = self.norm_q(self.q(x))
- k = self.norm_k(self.k(context))
+ q = self.norm_q.forward_inplace(self.q(x))
+ k = self.norm_k.forward_inplace(self.k(context))
v = self.v(context)
return self.o(attention(q, k, v, self.num_heads))
@@ -239,7 +313,7 @@ class DiTBlock(nn.Module):
) -> torch.Tensor:
views, frames, height, width = shape
x = rearrange(
- modulate(self.norm1_mvs(x), shift, scale),
+ normalized_modulation(self.norm1_mvs, x, shift, scale),
"v (f h w) c -> f (v h w) c",
v=views,
f=frames,
@@ -269,7 +343,10 @@ class DiTBlock(nn.Module):
self.modulation.to(dtype=time_modulation.dtype, device=time_modulation.device) + time_modulation
).chunk(6, dim=1)
- x = x + gate_msa * self.self_attn(modulate(self.norm1(x), shift_msa, scale_msa), spatial_freqs)
+ x = x + gate_msa * self.self_attn(
+ normalized_modulation(self.norm1, x, shift_msa, scale_msa),
+ spatial_freqs,
+ )
shift_mvs, scale_mvs, gate_mvs = (
self.modulation_mvs.to(dtype=time_modulation.dtype, device=time_modulation.device)
@@ -285,7 +362,7 @@ class DiTBlock(nn.Module):
x = x + self.cross_attn(self.norm3(x), repeat(context, "1 l c -> v l c", v=x.shape[0]))
- residual = self._feed_forward(modulate(self.norm2(x), shift_mlp, scale_mlp))
+ residual = self._feed_forward(normalized_modulation(self.norm2, x, shift_mlp, scale_mlp))
return x + gate_mlp * residual
@@ -457,6 +534,26 @@ class FourDAnyoneDiT(nn.Module):
packed_views += 1
return x, packed_views
+ @staticmethod
+ def _add_target_pose_features_streamed(
+ x: torch.Tensor,
+ pose_features: torch.Tensor,
+ target_views: int,
+ grid_size: tuple[int, int, int],
+ ) -> None:
+ """Stage one CPU pose feature at a time into patch-token storage."""
+
+ frames, height, width = grid_size
+ staging = torch.empty(
+ (MODEL_DIM, frames, height, width),
+ dtype=x.dtype,
+ device=x.device,
+ )
+ for view_index in range(target_views):
+ staging.copy_(pose_features[view_index])
+ pose_tokens = rearrange(staging, "c f h w -> (f h w) c")
+ x[view_index].add_(pose_tokens)
+
@staticmethod
def _add_pose_features(
x: torch.Tensor,
@@ -473,11 +570,24 @@ class FourDAnyoneDiT(nn.Module):
raise ValueError(f"Expected pose features {expected_pose}, got {tuple(pose_features.shape)}.")
if tuple(null_pose_feature.shape) != expected_null:
raise ValueError(f"Expected null pose features {expected_null}, got {tuple(null_pose_feature.shape)}.")
- pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")
null_tokens = rearrange(null_pose_feature, "v c f h w -> v (f h w) c")
if torch.is_grad_enabled():
+ if pose_features.device != x.device:
+ raise ValueError("Training requires pose features on the same device as patch tokens.")
+ pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")
return torch.cat([x[:target_views] + pose_tokens, x[target_views:] + null_tokens], dim=0)
- x[:target_views].add_(pose_tokens)
+ if pose_features.device == x.device:
+ pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")
+ x[:target_views].add_(pose_tokens)
+ else:
+ if pose_features.device.type != "cpu":
+ raise ValueError(f"Inference pose features must be on CPU or {x.device}, got {pose_features.device}.")
+ FourDAnyoneDiT._add_target_pose_features_streamed(
+ x,
+ pose_features,
+ target_views,
+ grid_size,
+ )
x[target_views:].add_(null_tokens)
return x