[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video
Release: reduce inference memory and clarify models (#26)
Yudong Jinf6599d89a256
diff --git a/README.md b/README.mdindex fb81f9d..225333a 100644--- a/README.md+++ b/README.md@@ -36,7 +36,7 @@ python scripts/download_example.py## Inference-This release provides two models: **4DAnyone-Base**, corresponding to the original model described in the paper, and **4DAnyone-Turbo**, the default accelerated model. Set `--enable_turbo=False` to use **4DAnyone-Base**.+We provide two models: **4DAnyone-Turbo** for faster four-step denoising and **4DAnyone-Base** with the standard denoising schedule. Both models achieve comparable generation quality. 4DAnyone-Turbo is enabled by default. Set `--enable_turbo=False` to use 4DAnyone-Base.4DAnyone supports flexible target-view counts, pitch layers, and yaw coverage. Here are several common camera configurations:diff --git a/fdanyone/model/conditioning.py b/fdanyone/model/conditioning.pyindex 0eb2cfa..28891a6 100644--- a/fdanyone/model/conditioning.py+++ b/fdanyone/model/conditioning.py@@ -157,7 +157,7 @@ class PoseFeatureBank:return int(self.null_features.shape[0])def allocate_group(self, size: int, device: str) -> Tensor:- """Allocate one reusable GPU group buffer."""+ """Allocate one reusable group buffer on the requested device."""import torchdiff --git a/fdanyone/model/distributed.py b/fdanyone/model/distributed.pyindex e1358c9..fe59daf 100644--- a/fdanyone/model/distributed.py+++ b/fdanyone/model/distributed.py@@ -250,7 +250,7 @@ def _load_worker_state(rank: int, request: DistributedDenoiseRequest, denoiser:pose_feature_batch=torch.empty((group_size, *request.pose_feature_shape[1:]),dtype=denoiser.dtype,- device=device,+ device="cpu",),latent_tail=tuple(payload["initial_latents"].shape[1:]),)diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.pyindex 1429acb..3f0e8c6 100644--- a/fdanyone/model/inference.py+++ b/fdanyone/model/inference.py@@ -181,7 +181,10 @@ def _denoise_rcp()source = src_latents.to(dtype=denoiser.dtype, device=device)context = context.to(dtype=denoiser.dtype, device=device)- pose_feature_batch = pose_features.allocate_group(len(camera_ids), device)+ # Keep the reusable group on the host. The DiT stages one feature at a+ # time while patch tokens are built, then releases that device scratch+ # before the transformer blocks reach their peak allocation.+ pose_feature_batch = pose_features.allocate_group(len(camera_ids), "cpu")pose_features.copy_group(tuple(range(len(camera_ids))), pose_feature_batch)null_pose_feature = pose_features.null_on(device)@@ -229,7 +232,7 @@ def _denoise_targets_single(context = context.to(dtype=denoiser.dtype, device=device)null_pose_feature = pose_features.null_on(device)group_size = len(routes[0][0])- pose_feature_batch = pose_features.allocate_group(group_size, device)+ pose_feature_batch = pose_features.allocate_group(group_size, "cpu")with torch.inference_mode(), _bf16_autocast():for step_index, groups in enumerate(tqdm(routes, desc=f"Generate {num_views} target views")):diff --git a/fdanyone/vendor/diffsynth/models/wan_video_dit.py b/fdanyone/vendor/diffsynth/models/wan_video_dit.pyindex cb78f25..9dcb71f 100644--- a/fdanyone/vendor/diffsynth/models/wan_video_dit.py+++ b/fdanyone/vendor/diffsynth/models/wan_video_dit.py@@ -45,6 +45,14 @@ NORM_EPSILON = 1e-6# gives the same one-view production chunk as the original 768 MiB# single-allocation limit, while naming the actual temporary budget.ROPE_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2+# RMSNorm keeps its FP32 input alive while materializing either the squared+# values or normalized result. Bound those two full-width temporaries without+# changing the reduction dimension or arithmetic used by the checkpoint.+RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2+# CUDA autocast keeps LayerNorm and the following modulation in FP32 until a+# linear consumes them as BF16. Bound the three coexisting FP32 tensors while+# preserving that final cast boundary.+NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES = 1536 * 1024**2ATTENTION_BACKEND_PRIORITY = ("flash_attn_3", "sageattention", "sdpa")@@ -82,6 +90,48 @@ def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torchreturn x * (1 + scale) + shift+def _normalized_modulation_chunk(+ norm: nn.Module,+ x: torch.Tensor,+ shift: torch.Tensor,+ scale: torch.Tensor,+ output: torch.Tensor,+) -> None:+ """Finish one FP32 normalization/modulation slice in a local scope."""++ output.copy_(modulate(norm(x), shift, scale))+++def normalized_modulation(+ norm: nn.Module,+ x: torch.Tensor,+ shift: torch.Tensor,+ scale: torch.Tensor,+) -> torch.Tensor:+ """Apply LayerNorm modulation with bounded inference temporaries."""++ if torch.is_grad_enabled():+ return modulate(norm(x), shift, scale)++ output = torch.empty_like(x)+ fp32_bytes_per_batch = x[0].numel() * torch.empty((), dtype=torch.float32).element_size()+ temporary_bytes_per_batch = 3 * fp32_bytes_per_batch+ batch_chunk = max(+ 1,+ min(x.shape[0], NORM_MODULATION_FP32_TEMPORARY_BUDGET_BYTES // temporary_bytes_per_batch),+ )+ for start in range(0, x.shape[0], batch_chunk):+ end = start + batch_chunk+ _normalized_modulation_chunk(+ norm,+ x[start:end],+ shift[start:end],+ scale[start:end],+ output[start:end],+ )+ return output++def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:sinusoid = torch.outer(position.to(torch.float64),@@ -164,6 +214,30 @@ class RMSNorm(nn.Module):dtype = x.dtypereturn self._normalize_float(x.float()).to(dtype) * self.weight+ def _normalize_chunk_inplace(self, chunk: torch.Tensor) -> None:+ """Normalize one projection slice and end its temporary lifetime here."""++ chunk.copy_(self.forward(chunk))++ def forward_inplace(self, x: torch.Tensor) -> torch.Tensor:+ """Normalize an inference-owned projection with bounded FP32 storage."""++ if torch.is_grad_enabled():+ return self.forward(x)+ if not x.is_contiguous():+ raise ValueError("In-place RMSNorm requires a contiguous projection buffer.")++ rows = x.view(-1, x.shape[-1])+ fp32_bytes_per_row = x.shape[-1] * torch.empty((), dtype=torch.float32).element_size()+ temporary_bytes_per_row = 2 * fp32_bytes_per_row+ rows_per_chunk = max(+ 1,+ min(rows.shape[0], RMS_NORM_FP32_TEMPORARY_BUDGET_BYTES // temporary_bytes_per_row),+ )+ for start in range(0, rows.shape[0], rows_per_chunk):+ self._normalize_chunk_inplace(rows[start : start + rows_per_chunk])+ return x+class SelfAttention(nn.Module):def __init__(self, dim: int, num_heads: int, eps: float = NORM_EPSILON) -> None:@@ -177,8 +251,8 @@ class SelfAttention(nn.Module):self.norm_k = RMSNorm(dim, eps=eps)def forward(self, x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:- q = self.norm_q(self.q(x))- k = self.norm_k(self.k(x))+ q = self.norm_q.forward_inplace(self.q(x))+ k = self.norm_k.forward_inplace(self.k(x))v = self.v(x)q = rope_apply(q, freqs, self.num_heads)k = rope_apply(k, freqs, self.num_heads)@@ -197,8 +271,8 @@ class CrossAttention(nn.Module):self.norm_k = RMSNorm(dim, eps=eps)def forward(self, x: torch.Tensor, context: torch.Tensor) -> torch.Tensor:- q = self.norm_q(self.q(x))- k = self.norm_k(self.k(context))+ q = self.norm_q.forward_inplace(self.q(x))+ k = self.norm_k.forward_inplace(self.k(context))v = self.v(context)return self.o(attention(q, k, v, self.num_heads))@@ -239,7 +313,7 @@ class DiTBlock(nn.Module):) -> torch.Tensor:views, frames, height, width = shapex = rearrange(- modulate(self.norm1_mvs(x), shift, scale),+ normalized_modulation(self.norm1_mvs, x, shift, scale),"v (f h w) c -> f (v h w) c",v=views,f=frames,@@ -269,7 +343,10 @@ class DiTBlock(nn.Module):self.modulation.to(dtype=time_modulation.dtype, device=time_modulation.device) + time_modulation).chunk(6, dim=1)- x = x + gate_msa * self.self_attn(modulate(self.norm1(x), shift_msa, scale_msa), spatial_freqs)+ x = x + gate_msa * self.self_attn(+ normalized_modulation(self.norm1, x, shift_msa, scale_msa),+ spatial_freqs,+ )shift_mvs, scale_mvs, gate_mvs = (self.modulation_mvs.to(dtype=time_modulation.dtype, device=time_modulation.device)@@ -285,7 +362,7 @@ class DiTBlock(nn.Module):x = x + self.cross_attn(self.norm3(x), repeat(context, "1 l c -> v l c", v=x.shape[0]))- residual = self._feed_forward(modulate(self.norm2(x), shift_mlp, scale_mlp))+ residual = self._feed_forward(normalized_modulation(self.norm2, x, shift_mlp, scale_mlp))return x + gate_mlp * residual@@ -457,6 +534,26 @@ class FourDAnyoneDiT(nn.Module):packed_views += 1return x, packed_views+ @staticmethod+ def _add_target_pose_features_streamed(+ x: torch.Tensor,+ pose_features: torch.Tensor,+ target_views: int,+ grid_size: tuple[int, int, int],+ ) -> None:+ """Stage one CPU pose feature at a time into patch-token storage."""++ frames, height, width = grid_size+ staging = torch.empty(+ (MODEL_DIM, frames, height, width),+ dtype=x.dtype,+ device=x.device,+ )+ for view_index in range(target_views):+ staging.copy_(pose_features[view_index])+ pose_tokens = rearrange(staging, "c f h w -> (f h w) c")+ x[view_index].add_(pose_tokens)+@staticmethoddef _add_pose_features(x: torch.Tensor,@@ -473,11 +570,24 @@ class FourDAnyoneDiT(nn.Module):raise ValueError(f"Expected pose features {expected_pose}, got {tuple(pose_features.shape)}.")if tuple(null_pose_feature.shape) != expected_null:raise ValueError(f"Expected null pose features {expected_null}, got {tuple(null_pose_feature.shape)}.")- pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")null_tokens = rearrange(null_pose_feature, "v c f h w -> v (f h w) c")if torch.is_grad_enabled():+ if pose_features.device != x.device:+ raise ValueError("Training requires pose features on the same device as patch tokens.")+ pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")return torch.cat([x[:target_views] + pose_tokens, x[target_views:] + null_tokens], dim=0)- x[:target_views].add_(pose_tokens)+ if pose_features.device == x.device:+ pose_tokens = rearrange(pose_features, "v c f h w -> v (f h w) c")+ x[:target_views].add_(pose_tokens)+ else:+ if pose_features.device.type != "cpu":+ raise ValueError(f"Inference pose features must be on CPU or {x.device}, got {pose_features.device}.")+ FourDAnyoneDiT._add_target_pose_features_streamed(+ x,+ pose_features,+ target_views,+ grid_size,+ )x[target_views:].add_(null_tokens)return x