[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video
"""Single-node distributed target denoising.
The public pipeline keeps preprocessing, RCP, and result publication in the
primary process. This module gives each CUDA worker one DiT replica and uses
NCCL collectives to evaluate independent camera groups in parallel. Every
TCR step is fully gathered before the next routing step begins.
"""
from __future__ import annotations
import json
import logging
import math
import os
import socket
import time
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING, TypedDict
from fdanyone.errors import FourDAnyoneError
from fdanyone.model.routing import CameraGroup, Routes, StepGroups, validate_routes
if TYPE_CHECKING:
from torch import Tensor
from fdanyone.config import DenoisingProfile
from fdanyone.model.conditioning import PoseFeatureBank
from fdanyone.model.loader import Denoiser
LOGGER = logging.getLogger("fdanyone")
class WorkerReport(TypedDict):
"""Per-rank measurements published by a distributed denoising worker."""
rank: int
device: str
device_name: str
attention_backend: str
model_load_seconds: float
denoise_seconds: float
peak_vram_allocated_bytes: int
peak_vram_reserved_bytes: int
@dataclass(frozen=True)
class DistributedDenoiseRequest:
checkpoint_path: str
turbo_lora_path: str | None
denoising_profile: DenoisingProfile
attention_backend: str
routes: Routes
devices: tuple[str, ...]
work_dir: str
pose_feature_shape: tuple[int, ...]
init_method: str
def worker_count_for_groups(num_groups: int, max_workers: int) -> int:
"""Use fewer workers when that balances groups without adding waves."""
if num_groups <= 0 or max_workers <= 0:
raise ValueError(f"num_groups and max_workers must be positive, got {num_groups} and {max_workers}.")
workers = min(num_groups, max_workers)
waves = math.ceil(num_groups / workers)
for candidate in range(workers, 0, -1):
if num_groups % candidate == 0 and num_groups // candidate == waves:
return candidate
return workers
def select_worker_devices(devices: Sequence[str], num_groups: int) -> tuple[str, ...]:
"""Choose the largest balanced GPU prefix that does not add a wave."""
if not devices:
raise ValueError("At least one candidate GPU is required.")
return tuple(devices[: worker_count_for_groups(num_groups, len(devices))])
def group_waves(groups: Sequence[CameraGroup], num_workers: int) -> tuple[StepGroups, ...]:
"""Split one routing step into waves that fit the available workers."""
if num_workers <= 0:
raise ValueError(f"num_workers must be positive, got {num_workers}.")
return tuple(tuple(groups[start : start + num_workers]) for start in range(0, len(groups), num_workers))
def _free_local_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def _resolve_empty_workspace(path: str | Path) -> Path:
"""Validate the caller-owned transient directory used by spawned ranks."""
root = Path(path).expanduser().resolve()
if not root.is_dir():
raise ValueError(f"Distributed work_dir must be an existing directory: {root}")
if any(root.iterdir()):
raise ValueError(f"Distributed work_dir must be empty: {root}")
return root
def _write_pose_feature_file(pose_features: PoseFeatureBank, path: Path) -> tuple[int, ...]:
"""Write precomputed target pose features to a worker-shared BF16 tensor."""
import torch
features = pose_features.features
if features.ndim != 5 or features.shape[0] <= 0:
raise FourDAnyoneError("Distributed target denoising requires at least one pose feature.")
shape = tuple(features.shape)
numel = math.prod(shape)
with path.open("xb") as handle:
handle.truncate(numel * features.element_size())
mapped = torch.from_file(str(path), shared=True, size=numel, dtype=torch.bfloat16).view(shape)
mapped.copy_(features)
del mapped
return shape
@dataclass
class _WorkerState:
"""Per-rank CUDA workspace, with canonical latents kept on the primary CPU."""
rank: int
request: DistributedDenoiseRequest
denoiser: Denoiser
device: str
device_index: int
source: Tensor
context: Tensor
null_pose_feature: Tensor
pose_features: Tensor
latents: Tensor | None
pose_feature_batch: Tensor
latent_tail: tuple[int, ...]
@property
def is_primary(self) -> bool:
return self.rank == 0
@property
def world_size(self) -> int:
return len(self.request.devices)
def _scatter_latents(self, wave: StepGroups) -> Tensor:
import torch
import torch.distributed as dist
local_input = torch.empty(
(self.pose_feature_batch.shape[0], *self.latent_tail),
dtype=self.denoiser.dtype,
device=self.device,
)
scatter_list = None
if self.is_primary:
if self.latents is None:
raise RuntimeError("The primary distributed rank does not own canonical latents.")
scatter_list = [torch.zeros_like(local_input) for _ in range(self.world_size)]
for worker_index, group in enumerate(wave):
index = torch.tensor(group, dtype=torch.long, device="cpu")
scatter_list[worker_index].copy_(torch.index_select(self.latents, 0, index))
dist.scatter(local_input, scatter_list=scatter_list, src=0)
return local_input
def _denoise_local_group(self, local_input: Tensor, wave: StepGroups, step_index: int) -> Tensor:
import torch
if self.rank >= len(wave):
return torch.zeros_like(local_input)
from fdanyone.model.denoise import denoise_group
group = wave[self.rank]
for output_index, camera_id in enumerate(group):
self.pose_feature_batch[output_index].copy_(self.pose_features[camera_id])
with torch.inference_mode(), torch.autocast(device_type="cuda", dtype=torch.bfloat16):
return denoise_group(
self.denoiser,
local_input,
self.source,
self.context,
self.pose_feature_batch,
self.null_pose_feature,
step_index,
)
def _gather_and_commit(self, local_result: Tensor, wave: StepGroups) -> None:
import torch
import torch.distributed as dist
gathered = [torch.empty_like(local_result) for _ in range(self.world_size)] if self.is_primary else None
dist.gather(local_result, gather_list=gathered, dst=0)
if not self.is_primary:
return
if self.latents is None or gathered is None:
raise RuntimeError("The primary distributed rank cannot commit gathered latents.")
for group, result in zip(wave, gathered[: len(wave)], strict=True):
index = torch.tensor(group, dtype=torch.long, device="cpu")
self.latents.index_copy_(0, index, result.to("cpu"))
def denoise(self) -> None:
"""Run every route step, committing a complete step before TCR moves on."""
import torch.distributed as dist
for step_index, groups in enumerate(self.request.routes):
for wave in group_waves(groups, self.world_size):
local_input = self._scatter_latents(wave)
local_result = self._denoise_local_group(local_input, wave, step_index)
self._gather_and_commit(local_result, wave)
del local_input, local_result
dist.barrier(device_ids=[self.device_index])
if self.is_primary:
LOGGER.info("Completed target denoising step %d/%d", step_index + 1, len(self.request.routes))
def _load_worker_state(rank: int, request: DistributedDenoiseRequest, denoiser: Denoiser) -> _WorkerState:
import torch
device = request.devices[rank]
payload = torch.load(
Path(request.work_dir) / "inputs.pt",
map_location="cpu",
weights_only=True,
mmap=True,
)
group_size = len(request.routes[0][0])
pose_features = torch.from_file(
str(Path(request.work_dir) / "pose_features.bf16"),
shared=False,
size=math.prod(request.pose_feature_shape),
dtype=torch.bfloat16,
).view(request.pose_feature_shape)
return _WorkerState(
rank=rank,
request=request,
denoiser=denoiser,
device=device,
device_index=int(device.removeprefix("cuda:")),
source=payload["source"].to(dtype=denoiser.dtype, device=device),
context=payload["context"].to(dtype=denoiser.dtype, device=device),
null_pose_feature=payload["null_pose_feature"],
pose_features=pose_features,
latents=payload["initial_latents"].to(dtype=denoiser.dtype, copy=True) if rank == 0 else None,
pose_feature_batch=torch.empty(
(group_size, *request.pose_feature_shape[1:]),
dtype=denoiser.dtype,
device="cpu",
),
latent_tail=tuple(payload["initial_latents"].shape[1:]),
)
def _publish_worker_result(state: _WorkerState, report: WorkerReport) -> None:
import torch
root = Path(state.request.work_dir)
if state.is_primary:
if state.latents is None:
raise RuntimeError("The primary distributed rank has no target latents to publish.")
temporary = root / ".target_latents.pt.tmp"
torch.save(state.latents, temporary)
os.replace(temporary, root / "target_latents.pt")
(root / f"rank-{state.rank}.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
def _worker(rank: int, request: DistributedDenoiseRequest) -> None:
"""Run one NCCL rank. Rank zero owns and publishes the canonical latents."""
import torch
import torch.distributed as dist
from fdanyone.model.loader import load_denoiser
logging.basicConfig(
level=logging.INFO,
format=f"%(asctime)s | %(levelname)s | GPU rank {rank} | %(message)s",
)
os.environ.setdefault("TORCH_NCCL_ASYNC_ERROR_HANDLING", "1")
os.environ.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1")
device = request.devices[rank]
device_index = int(device.removeprefix("cuda:"))
torch.cuda.set_device(device_index)
torch.cuda.reset_peak_memory_stats(device_index)
model_started = time.monotonic()
denoiser = load_denoiser(
checkpoint_path=request.checkpoint_path,
turbo_lora_path=request.turbo_lora_path,
profile=request.denoising_profile,
attention_backend=request.attention_backend,
)
denoiser.prepare_on_device(device)
torch.cuda.synchronize(device_index)
model_load_seconds = time.monotonic() - model_started
dist.init_process_group(
backend="nccl",
init_method=request.init_method,
rank=rank,
world_size=len(request.devices),
timeout=timedelta(minutes=30),
)
# A failed worker must exit so mp.spawn can terminate its peers. Normal
# NCCL destruction can wait forever for collectives those peers cannot finish.
state = _load_worker_state(rank, request, denoiser)
dist.barrier(device_ids=[device_index])
denoise_started = time.monotonic()
state.denoise()
torch.cuda.synchronize(device_index)
denoise_seconds = time.monotonic() - denoise_started
report: WorkerReport = {
"rank": rank,
"device": device,
"device_name": torch.cuda.get_device_name(device_index),
"attention_backend": denoiser.model.attention_backend,
"model_load_seconds": model_load_seconds,
"denoise_seconds": denoise_seconds,
"peak_vram_allocated_bytes": int(torch.cuda.max_memory_allocated(device_index)),
"peak_vram_reserved_bytes": int(torch.cuda.max_memory_reserved(device_index)),
}
_publish_worker_result(state, report)
dist.barrier(device_ids=[device_index])
dist.destroy_process_group()
def denoise_targets_distributed(
*,
checkpoint_path: str | Path,
turbo_lora_path: str | Path | None,
denoising_profile: DenoisingProfile,
attention_backend: str,
routes: Routes,
src_latents: Tensor,
context: Tensor,
initial_latents: Tensor,
pose_features: PoseFeatureBank,
work_dir: str | Path,
devices: Sequence[str],
) -> tuple[Tensor, list[WorkerReport]]:
"""Prepare shared inputs, launch NCCL workers, and return canonical latents."""
import torch
import torch.multiprocessing as mp
devices = tuple(devices)
if len(devices) < 2:
raise ValueError("Distributed denoising requires at least two GPUs.")
if not torch.distributed.is_available() or not torch.distributed.is_nccl_available():
raise FourDAnyoneError("Multi-GPU inference requires a PyTorch build with NCCL support.")
validate_routes(routes, int(initial_latents.shape[0]))
if len(routes) != denoising_profile.num_inference_steps:
raise ValueError(
f"Denoising profile {denoising_profile.name!r} requires "
f"{denoising_profile.num_inference_steps} routing steps, got {len(routes)}."
)
num_groups = len(routes[0])
root = _resolve_empty_workspace(work_dir)
torch.save(
{
"source": src_latents.detach().to("cpu").contiguous(),
"context": context.detach().to("cpu").contiguous(),
"null_pose_feature": pose_features.null_features,
"initial_latents": initial_latents.detach().to("cpu").contiguous(),
},
root / "inputs.pt",
)
pose_feature_shape = _write_pose_feature_file(
pose_features,
root / "pose_features.bf16",
)
request = DistributedDenoiseRequest(
checkpoint_path=str(Path(checkpoint_path).expanduser().resolve()),
turbo_lora_path=(None if turbo_lora_path is None else str(Path(turbo_lora_path).expanduser().resolve())),
denoising_profile=denoising_profile,
attention_backend=attention_backend,
routes=routes,
devices=devices,
work_dir=str(root),
pose_feature_shape=pose_feature_shape,
init_method=f"tcp://127.0.0.1:{_free_local_port()}",
)
LOGGER.info(
"Starting distributed target denoising on %d GPUs (%d groups, up to %d waves per step)",
len(devices),
num_groups,
math.ceil(num_groups / len(devices)),
)
try:
mp.spawn(_worker, args=(request,), nprocs=len(devices), join=True)
except Exception as exc:
raise FourDAnyoneError(f"Multi-GPU target denoising failed: {exc}") from exc
output_path = root / "target_latents.pt"
if not output_path.is_file():
raise FourDAnyoneError("Multi-GPU target denoising finished without publishing target latents.")
latents = torch.load(output_path, map_location="cpu", weights_only=True)
reports: list[WorkerReport] = []
for rank in range(len(devices)):
report_path = root / f"rank-{rank}.json"
if not report_path.is_file():
raise FourDAnyoneError(f"Multi-GPU worker {rank} did not publish its runtime report.")
reports.append(json.loads(report_path.read_text()))
return latents, reports