Skip to content

[SIGGRAPH Asia 2026] 4DAnyone: Create Anyone in 4D from a Casual Monocular Video

7.0 KB
Raw Blame
"""VGG-19 perceptual loss for Nerfstudio reconstruction."""

from __future__ import annotations

from collections.abc import Mapping
from contextlib import AbstractContextManager, nullcontext
from pathlib import Path

import torch
import torch.nn.functional as F
from safetensors.torch import load_file
from torch import nn

from fdanyone.errors import AssetError

# The reference objective reads these layers from the MatConvNet VGG-19
# checkpoint and returns features after conv{1,2,3,4,5}_2. The final two VGG
# convolutions and classifier are not evaluated by its forward pass.
VGG19_CONV_CHANNELS = (
    (3, 64),
    (64, 64),
    (64, 128),
    (128, 128),
    (128, 256),
    (256, 256),
    (256, 256),
    (256, 256),
    (256, 512),
    (512, 512),
    (512, 512),
    (512, 512),
    (512, 512),
    (512, 512),
)
VGG19_FEATURE_LAYERS = (2, 4, 6, 10, 14)
VGG19_POOL_AFTER = frozenset((2, 4, 8, 12))
PERCEPTUAL_FEATURE_DIVISORS = (2.6, 4.8, 3.7, 5.6, 0.15)
PERCEPTUAL_COMPUTE_DTYPES = ("float32", "bfloat16")


def _autocast_context(compute_dtype: str, device: torch.device) -> AbstractContextManager:
    if compute_dtype not in PERCEPTUAL_COMPUTE_DTYPES:
        choices = ", ".join(PERCEPTUAL_COMPUTE_DTYPES)
        raise ValueError(f"perceptual compute dtype must be one of: {choices}.")
    if compute_dtype == "float32":
        return nullcontext()
    if device.type != "cuda":
        raise RuntimeError("bfloat16 perceptual computation currently requires CUDA.")
    if not torch.cuda.is_bf16_supported():
        raise RuntimeError("This CUDA device does not support bfloat16 convolution.")
    return torch.autocast(device_type="cuda", dtype=torch.bfloat16)


def _validated_weights(path: Path) -> Mapping[str, torch.Tensor]:
    try:
        state = load_file(str(path), device="cpu")
    except (OSError, ValueError) as exc:
        raise AssetError(f"Cannot load perceptual VGG-19 weights at {path}: {exc}") from exc

    expected = set()
    for index, (in_channels, out_channels) in enumerate(VGG19_CONV_CHANNELS, start=1):
        weight_name = f"conv{index}.weight"
        bias_name = f"conv{index}.bias"
        expected.update((weight_name, bias_name))
        weight = state.get(weight_name)
        bias = state.get(bias_name)
        expected_weight_shape = (out_channels, in_channels, 3, 3)
        if weight is None or tuple(weight.shape) != expected_weight_shape:
            shape = None if weight is None else tuple(weight.shape)
            raise AssetError(
                f"Perceptual VGG-19 tensor {weight_name} has shape {shape}; expected {expected_weight_shape}."
            )
        if bias is None or tuple(bias.shape) != (out_channels,):
            shape = None if bias is None else tuple(bias.shape)
            raise AssetError(f"Perceptual VGG-19 tensor {bias_name} has shape {shape}; expected {(out_channels,)}.")
    unexpected = set(state) - expected
    if unexpected:
        raise AssetError(f"Perceptual VGG-19 weights contain unexpected tensors: {sorted(unexpected)}")
    return state


class MatConvNetVGG19Features(nn.Module):
    """Frozen MatConvNet VGG-19 features used by the perceptual objective.

    The weights are non-persistent buffers: they move with the Nerfstudio model
    but remain in the separately downloaded asset instead of being duplicated
    into every Splatfacto checkpoint.
    """

    def __init__(self, weights_path: str | Path) -> None:
        super().__init__()
        path = Path(weights_path).expanduser().resolve()
        if not path.is_file():
            raise AssetError(
                f"Perceptual VGG-19 weights do not exist: {path}. "
                "Run `python scripts/download_model.py` to install them."
            )
        state = _validated_weights(path)
        for index in range(1, len(VGG19_CONV_CHANNELS) + 1):
            self.register_buffer(
                f"conv{index}_weight",
                state[f"conv{index}.weight"].float().contiguous(),
                persistent=False,
            )
            self.register_buffer(
                f"conv{index}_bias",
                state[f"conv{index}.bias"].float().contiguous(),
                persistent=False,
            )

    def forward(self, image: torch.Tensor) -> tuple[torch.Tensor, ...]:
        features = []
        value = image.float().contiguous()
        for index in range(1, len(VGG19_CONV_CHANNELS) + 1):
            weight = getattr(self, f"conv{index}_weight")
            bias = getattr(self, f"conv{index}_bias")
            value = F.relu(F.conv2d(value, weight, bias, padding=1), inplace=False)
            if index in VGG19_FEATURE_LAYERS:
                features.append(value)
            if index in VGG19_POOL_AFTER:
                value = F.avg_pool2d(value, kernel_size=2, stride=2)
        return tuple(features)


def perceptual_distance(
    prediction: torch.Tensor,
    target: torch.Tensor,
    prediction_features: tuple[torch.Tensor, ...],
    target_features: tuple[torch.Tensor, ...],
) -> torch.Tensor:
    """Apply the reference pixel/feature weighting to scaled images."""

    if len(prediction_features) != 5 or len(target_features) != 5:
        raise ValueError("The perceptual loss requires exactly five VGG feature tensors.")
    terms = [torch.mean(torch.abs(target - prediction))]
    terms.extend(
        torch.mean(torch.abs(target_features[index] - prediction_features[index])) / divisor
        for index, divisor in enumerate(PERCEPTUAL_FEATURE_DIVISORS)
    )
    return torch.stack(terms).sum() / 255.0


class VGG19PerceptualLoss(nn.Module):
    """The reconstruction objective's VGG-19 loss, not standard LPIPS."""

    def __init__(self, weights_path: str | Path, compute_dtype: str = "float32") -> None:
        super().__init__()
        if compute_dtype not in PERCEPTUAL_COMPUTE_DTYPES:
            choices = ", ".join(PERCEPTUAL_COMPUTE_DTYPES)
            raise ValueError(f"perceptual compute dtype must be one of: {choices}.")
        self.compute_dtype = compute_dtype
        self.features = MatConvNetVGG19Features(weights_path)
        self.register_buffer(
            "imagenet_mean",
            torch.tensor((123.6800, 116.7790, 103.9390), dtype=torch.float32).reshape(1, 3, 1, 1),
            persistent=False,
        )

    def forward(self, prediction: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
        if prediction.shape != target.shape or prediction.ndim != 4 or prediction.shape[1] != 3:
            raise ValueError("VGG-19 perceptual loss expects prediction and target with matching [B, 3, H, W] shapes.")
        prediction_scaled = prediction.float() * 255.0 - self.imagenet_mean
        target_scaled = target.float() * 255.0 - self.imagenet_mean
        with _autocast_context(self.compute_dtype, prediction.device):
            with torch.no_grad():
                target_features = self.features(target_scaled)
            prediction_features = self.features(prediction_scaled)
        return perceptual_distance(
            prediction_scaled,
            target_scaled,
            prediction_features,
            target_features,
        )