Skip to content

Idol: a single image to 3D Gaussians. Inference SDK and demo.

Python
15.3 KB
Raw Blame
import importlib.util
import os
from typing import Optional

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.attention import SDPBackend, sdpa_kernel

# flash-attn when it is installed, PyTorch SDPA otherwise (same result, flash-attn is faster).
# Override with ATTN_BACKEND=flash_attn|sdpa|sdpa_cudnn (sdpa_cudnn: PyTorch SDPA restricted to the cuDNN kernel,
# which IdolPipeline(optimize=True) selects for the denoiser while sampling).
ATTN_BACKEND = os.environ.get('ATTN_BACKEND',
                              'flash_attn' if importlib.util.find_spec('flash_attn') is not None else 'sdpa')
if ATTN_BACKEND not in ('flash_attn', 'sdpa', 'sdpa_cudnn'):
    raise ValueError(f'Unknown attention backend: {ATTN_BACKEND}')
if ATTN_BACKEND == 'flash_attn':
    import flash_attn


def scaled_dot_product_attention(*args) -> torch.Tensor:
    """(qkv [N, L, 3, H, C]) or (q [N, L, H, C], k [N, S, H, C], v [N, S, H, C]) -> [N, L, H, C]."""
    if ATTN_BACKEND == 'flash_attn':
        if len(args) == 1:
            return flash_attn.flash_attn_qkvpacked_func(args[0])
        return flash_attn.flash_attn_func(*args)
    q, k, v = args[0].unbind(dim=2) if len(args) == 1 else args
    q, k, v = q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3)
    if ATTN_BACKEND == 'sdpa_cudnn':
        with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
            out = F.scaled_dot_product_attention(q, k, v)
    else:
        out = F.scaled_dot_product_attention(q, k, v)
    return out.permute(0, 2, 1, 3)


def convert_module_to_f16(module):
    if isinstance(module, nn.Linear):
        for parameter in module.parameters():
            parameter.data = parameter.data.half()


class LayerNorm32(nn.LayerNorm):
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return super().forward(x.float()).type(x.dtype)


class MultiHeadRMSNorm(nn.Module):
    def __init__(self, dim: int, heads: int):
        super().__init__()
        self.scale = dim ** 0.5
        self.gamma = nn.Parameter(torch.ones(heads, dim))

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return (F.normalize(x.float(), dim=-1) * self.gamma * self.scale).to(x.dtype)


class FeedForwardNet(nn.Module):
    def __init__(self, channels: int, mlp_ratio: float = 4.0):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.Linear(channels, int(channels * mlp_ratio)),
            nn.GELU(approximate="tanh"),
            nn.Linear(int(channels * mlp_ratio), channels),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.mlp(x)


class MLP(nn.Module):
    def __init__(self, channels: int, inner_channels: int, channels_out: int, mlp_layer_num: int = 2):
        super().__init__()
        layers = []
        for i in range(mlp_layer_num - 1):
            layers.append(nn.Linear(channels if i == 0 else inner_channels, inner_channels))
            layers.append(nn.GELU(approximate="tanh"))
        layers.append(nn.Linear(inner_channels, channels_out))
        self.mlp = nn.Sequential(*layers)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.mlp(x)


# -- Positional and scalar embeddings ----------------------------------------

class PcdAbsolutePositionEmbedder(nn.Module):
    """Sinusoidal embedding of 3D positions in [0, 1] (the denoiser's latent token positions)."""
    def __init__(self, channels: int, in_channels: int = 3, max_res=16):
        super().__init__()
        self.channels = channels
        self.in_channels = in_channels
        self.freq_dim = channels // in_channels // 2
        freqs_2exp = torch.arange(max_res, dtype=torch.float32)
        res_dim = max(0, self.freq_dim - max_res)
        freqs_res = torch.arange(res_dim, dtype=torch.float32) / res_dim * max_res
        self.freqs = 2 ** torch.cat([freqs_2exp, freqs_res], dim=0)[:self.freq_dim]

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        *dims, _ = x.shape
        self.freqs = self.freqs.to(x.device)
        out = torch.outer(x.reshape(-1), self.freqs) * 2 * torch.pi
        embed = torch.cat([torch.sin(out), torch.cos(out)], dim=-1).reshape(*dims, -1)
        if embed.shape[-1] < self.channels:
            embed = torch.cat([embed, torch.zeros(*dims, self.channels - embed.shape[-1], device=embed.device)], dim=-1)
        return embed


class PcdAbsolutePositionEmbedderv2(nn.Module):
    """Sinusoidal embedding of 3D points in [0, 1] (the decoders' query points)."""
    def __init__(self, channels: int, in_channels: int = 3, max_res: int = 10):
        super().__init__()
        self.channels = channels
        self.freq_dim = channels // in_channels // 2
        logs = torch.linspace(0.0, float(max_res), steps=self.freq_dim, dtype=torch.float32)
        self.register_buffer("freqs", torch.pow(2.0, logs), persistent=False)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        N = x.shape[0]
        freqs = self.freqs.to(x.device, dtype=x.dtype)
        ang = x.unsqueeze(-1) * freqs * torch.pi
        embed = torch.cat([torch.sin(ang), torch.cos(ang)], dim=-1).reshape(N, -1)
        if embed.shape[1] < self.channels:
            embed = torch.cat([embed, torch.zeros(N, self.channels - embed.shape[1], device=embed.device,
                                                  dtype=embed.dtype)], dim=-1)
        return embed


class SinusoidalEmbedder(nn.Module):
    """Embeds a scalar per batch element (diffusion time, octree level) into a vector."""
    def __init__(self, hidden_size, frequency_embedding_size=256, max_period=10000, two_pi=False):
        super().__init__()
        self.mlp = nn.Sequential(
            nn.Linear(frequency_embedding_size, hidden_size, bias=True),
            nn.SiLU(),
            nn.Linear(hidden_size, hidden_size, bias=True),
        )
        self.frequency_embedding_size = frequency_embedding_size
        self.max_period = max_period
        self.two_pi = two_pi

    def forward(self, t):
        half = self.frequency_embedding_size // 2
        freqs = torch.exp(
            -np.log(self.max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
        ).to(device=t.device)
        args = t[:, None].float() * freqs[None]
        if self.two_pi:
            args = args * 2 * torch.pi
        return self.mlp(torch.cat([torch.cos(args), torch.sin(args)], dim=-1))


def apply_rotary_emb(hidden_states: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
    """hidden_states [B, L, H, D] rotated by the complex frequencies freqs [B, L, H or 1, D/2], or by the same
    rotation as real (cos, sin) pairs [B, L, H, D/2, 2]."""
    if not freqs.is_complex():
        # the complex product written out, so torch.compile can fuse it
        x0, x1 = hidden_states.float().reshape(*hidden_states.shape[:-1], -1, 2).unbind(-1)
        cos, sin = freqs.unbind(-1)
        return torch.stack([x0 * cos - x1 * sin, x0 * sin + x1 * cos], dim=-1).flatten(-2).type_as(hidden_states)
    x_rotated = torch.view_as_complex(hidden_states.float().reshape(*hidden_states.shape[:-1], -1, 2))
    x_rotated = x_rotated * freqs
    x_out = torch.view_as_real(x_rotated).reshape(*x_rotated.shape[:-1], -1)
    return x_out.type_as(hidden_states)


def clamp_mul(x, f):
    f_t = f.tanh()
    return x * f_t + x.detach() * (f - f_t)


class RePo3DRotaryEmbedding(nn.Module):
    """Learned rotary position: each token's features predict a 3D offset per head, turned into RoPE angles."""
    # True: return (cos, sin) pairs instead of a complex tensor, which torch.compile cannot fuse (optimize=True sets it)
    real_valued = False

    def __init__(self, model_channels: int, num_heads: int, head_dim: int, repo_hidden_ratio: float = 0.125,
                 max_freq: float = 16.0):
        super().__init__()
        self.num_heads = num_heads
        repo_hidden_size = int(model_channels * repo_hidden_ratio)
        self.norm = LayerNorm32(model_channels)
        self.gate_map = nn.Linear(model_channels, repo_hidden_size, bias=False)
        self.content_map = nn.Linear(model_channels, repo_hidden_size, bias=False)
        self.act = nn.SiLU()
        self.final_map = nn.Linear(repo_hidden_size, 3 * num_heads, bias=False)
        # head_dim split into 3 parts for x, y, z
        dim_0 = dim_1 = 2 * (head_dim // 6)
        dims = [dim_0, dim_1, head_dim - dim_0 - dim_1]
        freqs = [torch.linspace(1.0, float(max_freq), steps=d // 2, dtype=torch.float32) for d in dims]
        self.freqs_0 = nn.Parameter(freqs[0])
        self.freqs_1 = nn.Parameter(freqs[1])
        self.freqs_2 = nn.Parameter(freqs[2])

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        h = self.norm(hidden_states.float())
        out = self.final_map(self.act(self.gate_map(h)) * self.content_map(h))
        B, L, _ = out.shape
        delta_pos = out.reshape(B, L, self.num_heads, 3)
        ang_0 = clamp_mul(delta_pos[..., 0].unsqueeze(-1), self.freqs_0) * torch.pi
        ang_1 = clamp_mul(delta_pos[..., 1].unsqueeze(-1), self.freqs_1) * torch.pi
        ang_2 = clamp_mul(delta_pos[..., 2].unsqueeze(-1), self.freqs_2) * torch.pi
        ang = torch.cat([ang_0, ang_1, ang_2], dim=-1)
        if self.real_valued:
            return torch.stack([ang.cos(), ang.sin()], dim=-1)
        return torch.polar(torch.ones_like(ang), ang).type(torch.complex64)


# -- Attention ---------------------------------------------------------------

class MultiHeadAttention(nn.Module):
    """Self or cross attention with RMS-normalised queries and keys (the decoders)."""
    def __init__(self, channels: int, num_heads: int, ctx_channels: Optional[int] = None, cross: bool = False):
        super().__init__()
        self.num_heads = num_heads
        self.cross = cross
        head_dim = channels // num_heads
        if cross:
            self.to_q = nn.Linear(channels, channels)
            self.to_kv = nn.Linear(ctx_channels, channels * 2)
        else:
            self.to_qkv = nn.Linear(channels, channels * 3)
        self.q_rms_norm = MultiHeadRMSNorm(head_dim, num_heads)
        self.k_rms_norm = MultiHeadRMSNorm(head_dim, num_heads)
        self.to_out = nn.Linear(channels, channels)

    def forward(self, x: torch.Tensor, context: Optional[torch.Tensor] = None) -> torch.Tensor:
        B, L, C = x.shape
        if self.cross:
            q = self.q_rms_norm(self.to_q(x).reshape(B, L, self.num_heads, -1))
            k, v = self.to_kv(context).reshape(B, context.shape[1], 2, self.num_heads, -1).unbind(dim=2)
            h = scaled_dot_product_attention(q, self.k_rms_norm(k), v)
        else:
            q, k, v = self.to_qkv(x).reshape(B, L, 3, self.num_heads, -1).unbind(dim=2)
            h = scaled_dot_product_attention(torch.stack([self.q_rms_norm(q), self.k_rms_norm(k), v], dim=2))
        return self.to_out(h.reshape(B, L, -1))


class RopeMultiHeadAttention(nn.Module):
    """Self attention with rotary embeddings and RMS-normalised queries and keys (the denoiser)."""
    def __init__(self, channels: int, num_heads: int):
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = channels // num_heads
        self.qkv = nn.Linear(channels, channels * 3)
        self.q_norm = MultiHeadRMSNorm(self.head_dim, num_heads)
        self.k_norm = MultiHeadRMSNorm(self.head_dim, num_heads)
        self.out = nn.Linear(channels, channels)

    def forward(self, x: torch.Tensor, rope_emb: torch.Tensor) -> torch.Tensor:
        B, L, C = x.shape
        q, k, v = self.qkv(x).reshape(B, L, 3, self.num_heads, self.head_dim).unbind(2)
        q = self.q_norm(apply_rotary_emb(q, rope_emb))
        k = self.k_norm(apply_rotary_emb(k, rope_emb))
        return self.out(scaled_dot_product_attention(q, k, v).reshape(B, L, C))


# -- Transformer blocks ------------------------------------------------------

class UnifiedTransformerBlock(nn.Module):
    """Denoiser block: RoPE self attention + FFN, modulated by the diffusion time through a shared adaLN plus
    this block's own shift table (modulation=False for the condition refiner, which has plain LayerNorms)."""
    def __init__(self, channels: int, num_heads: int, mlp_ratio: float = 4.0, modulation: bool = True):
        super().__init__()
        self.modulation = modulation
        self.norm1 = LayerNorm32(channels, elementwise_affine=not modulation, eps=1e-6)
        self.norm2 = LayerNorm32(channels, elementwise_affine=not modulation, eps=1e-6)
        self.attn = RopeMultiHeadAttention(channels, num_heads=num_heads)
        self.mlp = FeedForwardNet(channels, mlp_ratio=mlp_ratio)
        if modulation:
            self.shift_table = nn.Parameter(torch.zeros(1, 6 * channels))

    def forward(self, x: torch.Tensor, mod: Optional[torch.Tensor], rotary_emb: torch.Tensor) -> torch.Tensor:
        if not self.modulation:
            x = x + self.attn(self.norm1(x), rope_emb=rotary_emb)
            return x + self.mlp(self.norm2(x))
        mod = mod + self.shift_table.type(mod.dtype)
        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=1)
        h = self.norm1(x)
        h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
        h = self.attn(h, rope_emb=rotary_emb)
        x = x + h * gate_msa.unsqueeze(1)
        h = self.norm2(x)
        h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
        h = self.mlp(h)
        return x + h * gate_mlp.unsqueeze(1)


class TransformerCrossBlock(nn.Module):
    """Gaussian decoder block: self attention + cross attention to the latent + FFN."""
    def __init__(self, channels: int, ctx_channels: int, num_heads: int, mlp_ratio: float = 4.0):
        super().__init__()
        self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
        self.norm2 = LayerNorm32(channels, elementwise_affine=True, eps=1e-6)
        self.norm3 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
        self.self_attn = MultiHeadAttention(channels, num_heads=num_heads)
        self.cross_attn = MultiHeadAttention(channels, num_heads=num_heads, ctx_channels=ctx_channels, cross=True)
        self.mlp = FeedForwardNet(channels, mlp_ratio=mlp_ratio)

    def forward(self, x: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
        x = x + self.self_attn(self.norm1(x))
        x = x + self.cross_attn(self.norm2(x), context)
        return x + self.mlp(self.norm3(x))


class ModulatedTransformerCrossOnlyBlock(nn.Module):
    """Octree decoder block: cross attention to the latent + FFN, modulated by the octree level (shared adaLN)."""
    def __init__(self, channels: int, ctx_channels: int, num_heads: int, mlp_ratio: float = 4.0):
        super().__init__()
        self.norm1 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
        self.norm2 = LayerNorm32(channels, elementwise_affine=False, eps=1e-6)
        self.cross_attn = MultiHeadAttention(channels, num_heads=num_heads, ctx_channels=ctx_channels, cross=True)
        self.mlp = FeedForwardNet(channels, mlp_ratio=mlp_ratio)

    def forward(self, x: torch.Tensor, mod: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=1)
        h = self.norm1(x)
        h = h * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1)
        h = self.cross_attn(h, context)
        x = x + h * gate_msa.unsqueeze(1)
        h = self.norm2(x)
        h = h * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1)
        h = self.mlp(h)
        return x + h * gate_mlp.unsqueeze(1)