Skip to content

Idol: a single image to 3D Gaussians. Inference SDK and demo.

Python
9.7 KB
Raw Blame
from typing import List

import torch
import torch.nn as nn
import torch.nn.functional as F

from ..gaussian import Gaussian
from .layers import (ModulatedTransformerCrossOnlyBlock, PcdAbsolutePositionEmbedderv2, SinusoidalEmbedder,
                     TransformerCrossBlock, convert_module_to_f16)


class OctreeProbabilityFixedlenDecoder(nn.Module):
    def __init__(self, model_channels: int, cond_channels: int, num_blocks: int, num_heads: int,
                 mlp_ratio: float = 4.0):
        super().__init__()
        self.dtype = torch.float16
        self.in_proj = nn.Linear(3, model_channels)
        self.pos_embedder = PcdAbsolutePositionEmbedderv2(channels=model_channels, in_channels=3)
        self.input_layer = nn.Linear(model_channels, model_channels)
        self.l_embedder = SinusoidalEmbedder(model_channels, max_period=1024, two_pi=True)
        self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(model_channels, 6 * model_channels, bias=True))
        self.blocks = nn.ModuleList([
            ModulatedTransformerCrossOnlyBlock(model_channels, ctx_channels=cond_channels, num_heads=num_heads,
                                               mlp_ratio=mlp_ratio)
            for _ in range(num_blocks)
        ])
        self.out_proj = nn.Linear(model_channels, 2 ** 3)
        self.blocks.apply(convert_module_to_f16)

    def forward(self, x: torch.Tensor, level: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
        """x: cell centres [B, L, 3] in [0, 1]; level: the cells' octree resolution [B]; cond: latent [B, Lc, C].
        Returns the logits [B, L, 8] of how each cell's points split among its 8 children."""
        B, L, _ = x.shape
        h = self.in_proj(x) + self.pos_embedder(x.view(-1, 3)).view(B, L, -1)
        h = self.input_layer(h)
        level_emb = self.adaLN_modulation(self.l_embedder(level.type(h.dtype))).type(self.dtype)
        out = h.type(self.dtype)
        context = cond.type(self.dtype)
        for block in self.blocks:
            out = block(out, level_emb, context)
        out = out.type(h.dtype).type(cond.dtype)
        return self.out_proj(F.layer_norm(out, out.shape[-1:]))

    @staticmethod
    @torch.no_grad()
    def sample(model, cond: torch.Tensor, num_points: int, level: int) -> torch.Tensor:
        """Place num_points anchor points [B, num_points, 3] in [0, 1]^3, descending the octree to ``level``."""
        B = cond.shape[0]
        device = cond.device
        child_offset = torch.tensor([[i, j, k] for k in [0, 1] for j in [0, 1] for i in [0, 1]], dtype=torch.long,
                                    device=device)
        prev_coords_int = torch.zeros(B, 1, 3, dtype=torch.long, device=device)
        prev_counts = torch.full((B, 1), num_points, dtype=torch.long, device=device)
        batch_indices_range = torch.arange(B, device=device).unsqueeze(1)

        for l in range(1, level + 1):
            res_p = 1 << (l - 1)
            parent_coords_norm = (prev_coords_int.to(torch.float32) + 0.5) / res_p
            res_tensor = torch.full((B,), 1 << l, dtype=torch.long, device=device)
            pred_probs = torch.softmax(model(parent_coords_norm, res_tensor, cond), dim=-1)    # [B, Np, 8]
            sampled = systematic_sample(pred_probs, prev_counts).flatten(1, 2)                   # [B, Np*8]
            child_coords_int = (prev_coords_int[:, :, None, :] * 2 + child_offset[None, None, :, :]).flatten(1, 2)

            # keep the children that received points, packed to the front
            mask = sampled > 0
            max_valid = mask.sum(dim=1).max().item()
            valid_scatter_indices = (mask.cumsum(dim=1) - 1)[mask]
            valid_batch_indices = batch_indices_range.expand_as(mask)[mask]
            prev_coords_int = torch.zeros(B, max_valid, 3, dtype=child_coords_int.dtype, device=device)
            prev_coords_int[valid_batch_indices, valid_scatter_indices] = child_coords_int[mask]
            prev_counts = torch.zeros(B, max_valid, dtype=sampled.dtype, device=device)
            prev_counts[valid_batch_indices, valid_scatter_indices] = sampled[mask]

        # each point lands at a uniformly random position inside its leaf cell
        coords_int = torch.repeat_interleave(prev_coords_int.flatten(0, 1), prev_counts.flatten(0, 1),
                                             dim=0).reshape(B, num_points, -1)
        return (coords_int.to(torch.float32) + torch.rand_like(coords_int, dtype=torch.float32)) / (1 << level)


@torch.no_grad()
def systematic_sample(probs: torch.Tensor, counts: torch.Tensor) -> torch.Tensor:
    """Split counts[...] items among the P bins of probs[..., P] by systematic resampling -> [..., P] counts."""
    batch_shape = counts.shape
    B = counts.numel()
    P = probs.size(-1)
    device = probs.device
    probs = probs.view(B, P)
    counts = counts.view(B)

    # normalise, with a uniform fallback for zero-sum rows
    probs = probs.to(torch.float32).clamp_min_(0)
    row_sums = probs.sum(1, keepdim=True)
    zero_mask = row_sums.eq(0)
    probs = probs / row_sums.clamp_min_(1)
    if zero_mask.any():
        probs = probs.clone()
        probs[zero_mask.expand_as(probs)] = 1.0 / P

    counts = counts.to(device=device, dtype=torch.long)
    out = torch.zeros(B, P, dtype=torch.long, device=device)
    cdf = probs.cumsum(dim=1).clamp(max=1.0 - 1e-12)
    unique_n, inv = counts.unique(sorted=False, return_inverse=True)
    for index, n in enumerate(unique_n.tolist()):
        if n == 0:
            continue
        rows = (inv == index).nonzero(as_tuple=False).squeeze(1)
        r = rows.numel()
        u0 = torch.rand(r, 1, device=device) / float(n)                                   # random offset in [0, 1/n)
        grid = torch.arange(n, device=device, dtype=torch.float32)[None, :] / float(n)    # stratified points
        us = (u0 + grid).clamp(max=1.0 - 1e-12)
        idx = torch.searchsorted(cdf.index_select(0, rows), us).clamp_max(P - 1)
        buf = torch.zeros(r, P, dtype=torch.float32, device=device)
        buf.scatter_add_(1, idx, torch.ones_like(idx, dtype=buf.dtype))
        out.index_copy_(0, rows, buf.to(torch.long))
    return out.view(*batch_shape, P)


class GaussianFixedlenDecoder(nn.Module):
    def __init__(self, in_channels: int, model_channels: int, cond_channels: int, num_blocks: int, num_heads: int,
                 representation_config: dict, mlp_ratio: float = 4.0):
        super().__init__()
        self.dtype = torch.float16
        self.rep_config = representation_config
        per_anchor = representation_config['num_gaussians']
        # output channels per anchor: offset (3), colour (3), scale (3), rotation (4), opacity (1), offset scale (1)
        self.layout = {
            '_xyz': (3,),
            '_features_dc': (1, 3),
            '_scaling': (3,),
            '_rotation': (4,),
            '_opacity': (1,),
            '_offset_scale': (1,),
        }
        start = 0
        self.ranges = {}
        for name, shape in self.layout.items():
            size = per_anchor * int(torch.tensor(shape).prod())
            self.ranges[name] = (start, start + size)
            start += size

        self.in_proj = nn.Linear(in_channels, model_channels)
        self.pos_embedder = PcdAbsolutePositionEmbedderv2(channels=model_channels, in_channels=3)
        self.input_layer = nn.Linear(model_channels, model_channels)
        self.blocks = nn.ModuleList([
            TransformerCrossBlock(model_channels, ctx_channels=cond_channels, num_heads=num_heads, mlp_ratio=mlp_ratio)
            for _ in range(num_blocks)
        ])
        self.out_proj = nn.Linear(model_channels, start)
        # fixed per-Gaussian offset pattern and the base offset scale, stored with the weights
        self.register_buffer('points_offset_perturbation', torch.zeros(per_anchor, 3))
        self.register_buffer('base_offset_scale', torch.zeros(()))
        self.blocks.apply(convert_module_to_f16)

    def forward(self, points: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
        """points: anchors [B, L, 3]; cond: latent [B, Lc, C] -> per-anchor outputs [B, L, 32 x 15]"""
        B, L, C = points.shape
        h = self.in_proj(points) + self.pos_embedder(points.view(-1, C)).view(B, L, -1)
        h = self.input_layer(h)
        out = h.type(self.dtype)
        context = cond.type(self.dtype)
        for block in self.blocks:
            out = block(out, context)
        out = out.type(h.dtype).type(cond.dtype)
        return self.out_proj(F.layer_norm(out, out.shape[-1:]))

    def _slice(self, features, name):
        start, end = self.ranges[name]
        return features[:, :, start:end].reshape(features.shape[0], -1, self.rep_config['num_gaussians'],
                                                 *self.layout[name])

    def to_representation(self, points: torch.Tensor, features: torch.Tensor) -> List[Gaussian]:
        config = self.rep_config
        offset_scale = F.softplus(self._slice(features, '_offset_scale') + self.base_offset_scale)
        offset = self._slice(features, '_xyz') * config['lr']['_xyz'] + self.points_offset_perturbation
        offset = torch.tanh(offset) * 0.5 * config['perturbe_size'] * offset_scale

        objects = []
        for i in range(features.shape[0]):
            raw = {name: (self._slice(features[i:i + 1], name)[0].flatten(0, 1) * config['lr'][name])
                   for name in ('_features_dc', '_scaling', '_rotation', '_opacity')}
            objects.append(Gaussian(
                raw_xyz=(offset[i] + points[i, :, None, :]).flatten(0, 1),
                features_dc=raw['_features_dc'], raw_scaling=raw['_scaling'], raw_rotation=raw['_rotation'],
                raw_opacity=raw['_opacity'], scaling_bias=config['scaling_bias'],
                opacity_bias=config['opacity_bias'], minimum_kernel_size=config['3d_filter_kernel_size']))
        return objects