[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video
import torch
import torch.nn as nn
import numpy as np
from torch.nn import init
# PoseEncoder is a 3D version of PoseNet in MimicMotion. Copyright Tencent;
# the source is licensed under Apache-2.0:
# https://github.com/Tencent/MimicMotion/blob/c053153a1d124abae8c08568925ae88debc63001/mimicmotion/modules/pose_net.py
class PoseEncoder(nn.Module):
temporal_prefix = 3
def __init__(self, out_dim=5120, in_channels=3):
super().__init__()
if out_dim in (5120, 1536):
# Wan2.1-T2V-14B / 1.3B
t_strides = (1, 1, 1, 2, 2) # downsampled by 4
s_strides = (2, 2, 1, 2, 2) # downsampled by 16
kernel_size = (3, 3, 3)
elif out_dim == 3072:
# Wan2.2-TI2V-5B
t_strides = (1, 1, 1, 2, 2) # downsampled by 4
s_strides = (2, 2, 2, 2, 2) # downsampled by 32
kernel_size = (3, 4, 4)
else:
raise ValueError(f"Invalid out_dim: {out_dim}")
strides = [(t, s, s) for t, s in zip(t_strides, s_strides)]
self.conv_layers = nn.Sequential(
nn.Conv3d(in_channels, in_channels, kernel_size=3, stride=1, padding=1),
nn.SiLU(inplace=True),
nn.Conv3d(in_channels, 16, kernel_size=kernel_size, stride=strides[0], padding=(1, 1, 1)),
nn.SiLU(inplace=True),
nn.Conv3d(16, 16, kernel_size=3, stride=1, padding=1),
nn.SiLU(inplace=True),
nn.Conv3d(16, 32, kernel_size=kernel_size, stride=strides[1], padding=(1, 1, 1)),
nn.SiLU(inplace=True),
nn.Conv3d(32, 32, kernel_size=3, stride=1, padding=1),
nn.SiLU(inplace=True),
nn.Conv3d(32, 64, kernel_size=kernel_size, stride=strides[2], padding=(1, 1, 1)),
nn.SiLU(inplace=True),
nn.Conv3d(64, 64, kernel_size=3, stride=1, padding=1),
nn.SiLU(inplace=True),
nn.Conv3d(64, 128, kernel_size=kernel_size, stride=strides[3], padding=(1, 1, 1)),
nn.SiLU(inplace=True),
nn.Conv3d(128, 128, kernel_size=3, stride=1, padding=1),
nn.SiLU(inplace=True),
nn.Conv3d(128, 256, kernel_size=kernel_size, stride=strides[4], padding=(1, 1, 1)),
nn.SiLU(inplace=True),
)
self.final_proj = nn.Conv3d(256, out_dim, kernel_size=1)
self.scale = nn.Parameter(torch.ones(1) * 2.0)
self._initialize_weights()
def _initialize_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv3d):
# He (Kaiming) initialization in fan‑in mode
receptive = np.prod(m.kernel_size) * m.in_channels
init.normal_(m.weight, mean=0.0, std=np.sqrt(2.0 / receptive))
if m.bias is not None:
init.zeros_(m.bias)
# start with zero output so model behaves like unconditional
init.zeros_(self.final_proj.weight)
if self.final_proj.bias is not None:
init.zeros_(self.final_proj.bias)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
x: contiguous (B, C, F + 3, H, W), with three copies of the first
frame prepended by the conditioning input builder. Preparing this
final layout directly avoids retaining a second full video batch.
Wan2.1 uses F/4, H/16, W/16; Wan2.2-TI2V-5B uses F/4, H/32, W/32.
"""
x = self.conv_layers(x)
x = self.final_proj(x)
return x * self.scale