Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

Release: fit 48-view inference on RTX 4090 (#34)

Yudong Jinf9422eabd24c
diff --git a/fdanyone/model/distributed.py b/fdanyone/model/distributed.py
index ee35ad5..85b27e8 100644
--- a/fdanyone/model/distributed.py
+++ b/fdanyone/model/distributed.py
@@ -125,7 +125,7 @@ def _write_pose_feature_file(pose_features: PoseFeatureBank, path: Path) -> tupl
@dataclass
class _WorkerState:
- """CUDA tensors and collectives owned by one spawned NCCL rank."""
+ """Per-rank CUDA workspace, with canonical latents kept on the primary CPU."""
rank: int
request: DistributedDenoiseRequest
@@ -163,7 +163,7 @@ class _WorkerState:
raise RuntimeError("The primary distributed rank does not own canonical latents.")
scatter_list = [torch.zeros_like(local_input) for _ in range(self.world_size)]
for worker_index, group in enumerate(wave):
- index = torch.tensor(group, dtype=torch.long, device=self.device)
+ index = torch.tensor(group, dtype=torch.long, device="cpu")
scatter_list[worker_index].copy_(torch.index_select(self.latents, 0, index))
dist.scatter(local_input, scatter_list=scatter_list, src=0)
return local_input
@@ -201,8 +201,8 @@ class _WorkerState:
if self.latents is None or gathered is None:
raise RuntimeError("The primary distributed rank cannot commit gathered latents.")
for group, result in zip(wave, gathered[: len(wave)], strict=True):
- index = torch.tensor(group, dtype=torch.long, device=self.device)
- self.latents.index_copy_(0, index, result)
+ index = torch.tensor(group, dtype=torch.long, device="cpu")
+ self.latents.index_copy_(0, index, result.to("cpu"))
def denoise(self) -> None:
"""Run every route step, committing a complete step before TCR moves on."""
@@ -247,7 +247,7 @@ def _load_worker_state(rank: int, request: DistributedDenoiseRequest, denoiser:
context=payload["context"].to(dtype=denoiser.dtype, device=device),
null_pose_feature=payload["null_pose_feature"].to(dtype=denoiser.dtype, device=device),
pose_features=pose_features,
- latents=payload["initial_latents"].to(dtype=denoiser.dtype, device=device) if rank == 0 else None,
+ latents=payload["initial_latents"].to(dtype=denoiser.dtype, copy=True) if rank == 0 else None,
pose_feature_batch=torch.empty(
(group_size, *request.pose_feature_shape[1:]),
dtype=denoiser.dtype,
@@ -265,7 +265,7 @@ def _publish_worker_result(state: _WorkerState, report: WorkerReport) -> None:
if state.latents is None:
raise RuntimeError("The primary distributed rank has no target latents to publish.")
temporary = root / ".target_latents.pt.tmp"
- torch.save(state.latents.detach().to("cpu"), temporary)
+ torch.save(state.latents, temporary)
os.replace(temporary, root / "target_latents.pt")
(root / f"rank-{state.rank}.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
@@ -308,29 +308,28 @@ def _worker(rank: int, request: DistributedDenoiseRequest) -> None:
world_size=len(request.devices),
timeout=timedelta(minutes=30),
)
- try:
- state = _load_worker_state(rank, request, denoiser)
- dist.barrier(device_ids=[device_index])
- denoise_started = time.monotonic()
- state.denoise()
- torch.cuda.synchronize(device_index)
- denoise_seconds = time.monotonic() - denoise_started
-
- report: WorkerReport = {
- "rank": rank,
- "device": device,
- "device_name": torch.cuda.get_device_name(device_index),
- "attention_backend": denoiser.model.attention_backend,
- "model_load_seconds": model_load_seconds,
- "denoise_seconds": denoise_seconds,
- "peak_vram_allocated_bytes": int(torch.cuda.max_memory_allocated(device_index)),
- "peak_vram_reserved_bytes": int(torch.cuda.max_memory_reserved(device_index)),
- }
- _publish_worker_result(state, report)
- dist.barrier(device_ids=[device_index])
- finally:
- if dist.is_initialized():
- dist.destroy_process_group()
+ # A failed worker must exit so mp.spawn can terminate its peers. Normal
+ # NCCL destruction can wait forever for collectives those peers cannot finish.
+ state = _load_worker_state(rank, request, denoiser)
+ dist.barrier(device_ids=[device_index])
+ denoise_started = time.monotonic()
+ state.denoise()
+ torch.cuda.synchronize(device_index)
+ denoise_seconds = time.monotonic() - denoise_started
+
+ report: WorkerReport = {
+ "rank": rank,
+ "device": device,
+ "device_name": torch.cuda.get_device_name(device_index),
+ "attention_backend": denoiser.model.attention_backend,
+ "model_load_seconds": model_load_seconds,
+ "denoise_seconds": denoise_seconds,
+ "peak_vram_allocated_bytes": int(torch.cuda.max_memory_allocated(device_index)),
+ "peak_vram_reserved_bytes": int(torch.cuda.max_memory_reserved(device_index)),
+ }
+ _publish_worker_result(state, report)
+ dist.barrier(device_ids=[device_index])
+ dist.destroy_process_group()
def denoise_targets_distributed(
diff --git a/fdanyone/model/inference.py b/fdanyone/model/inference.py
index 3ebe16b..6dfddc8 100644
--- a/fdanyone/model/inference.py
+++ b/fdanyone/model/inference.py
@@ -213,10 +213,14 @@ def _denoise_targets_single(
routes: Routes,
device: str,
) -> Tensor:
+ """Denoise view groups on the GPU while the full target state stays on the CPU."""
+
import torch
from tqdm.auto import tqdm
num_views = initial_latents.shape[0]
+ if initial_latents.device.type != "cpu":
+ raise FourDAnyoneError("Canonical target latents must remain on the CPU between denoising groups.")
if pose_features.num_features != num_views:
raise FourDAnyoneError(
f"Target generation requires {num_views} pose features, got {pose_features.num_features}."
@@ -238,8 +242,8 @@ def _denoise_targets_single(
with torch.inference_mode(), _bf16_autocast():
for step_index, groups in enumerate(tqdm(routes, desc=f"Generate {num_views} target views")):
for view_indices in groups:
- index = torch.tensor(view_indices, dtype=torch.long, device=device)
- local_latents = torch.index_select(latents, 0, index)
+ index = torch.tensor(view_indices, dtype=torch.long, device="cpu")
+ local_latents = torch.index_select(latents, 0, index).to(device)
pose_features.copy_group(view_indices, pose_feature_batch)
local_latents = denoise_group(
denoiser,
@@ -250,9 +254,9 @@ def _denoise_targets_single(
null_pose_feature,
step_index,
)
- latents.index_copy_(0, index, local_latents)
+ latents.index_copy_(0, index, local_latents.to("cpu"))
del local_latents
- return latents.detach().to("cpu")
+ return latents
def _resolve_generation_plan(
@@ -498,11 +502,10 @@ def generate_views(
num_views=plan.view_plan.num_target_views,
num_frames=INFERENCE.num_frames,
seed=seed,
- device=plan.primary_device,
+ device="cpu",
)
if plan.distributed:
denoiser = None
- initial_latents = initial_latents.to("cpu")
_empty_cuda_cache()
target_latents, parallelism = _denoise_targets_multi_gpu(
checkpoint_path=checkpoint_path,