Skip to content

SUPRA-Tex: 3D Generation with Super-Aligned Texture

3.2 KB
Raw Blame
import math

import torch
import torch.nn.functional as F
from einops import rearrange
from torch import nn

from src.layers import CrossAttention, RoPE, encoder


class ImageEncoder(nn.Module):
    def __init__(
        self,
        in_channels=3,
        out_channels=256,
        heads_rope=1,
        use_encoder=True,
        rope_base=None,
        rope_rescale=None,
        img_layers=2,
    ):
        super().__init__()
        self.use_encoder = use_encoder
        self.out_channels = out_channels

        self.encoder = encoder(in_channels, out_channels // 2, kernel_size=1, ks_res=1, num_layers=img_layers)
        self.sem_encoder = encoder(in_channels, out_channels // 2, kernel_size=3, ks_res=3, num_layers=img_layers)

        self.rope = RoPE(embed_dim=out_channels, num_heads=heads_rope, base=rope_base, rescale_coords=rope_rescale)

    def forward_encoder(self, x, output_size):
        if self.use_encoder:
            x = torch.cat([self.encoder(x), self.sem_encoder(x)], dim=1)
        x = F.adaptive_avg_pool2d(x, output_size=output_size)
        return x

    def forward(self, x, output_size):
        o_size = output_size
        if x.shape[-2] > 4 * o_size[0] or x.shape[-1] > 4 * o_size[1]:
            x = F.interpolate(
                x,
                size=(
                    min(x.shape[-2], 4 * o_size[0], 4 * o_size[1]),
                    min(x.shape[-1], 4 * o_size[1], 4 * o_size[0]),
                ),
                mode="bilinear",
                align_corners=False,
            )

        x = self.forward_encoder(x, o_size)
        x = self.rope(x)
        return x


class QueryEncoder(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return x


class KeyEncoder(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x, features):
        x = F.adaptive_avg_pool2d(x, output_size=features.shape[-2:])
        return x


class NAF(nn.Module):
    def __init__(
        self,
        dim=256,
        heads_attn=4,
        heads_rope=4,
        kernel_size=9,
        # ImageEncoder options
        use_encoder=True,
        rope_base=100.0,
        rope_rescale=2.0,
        img_layers=2,
        **kwargs,
    ):
        super().__init__()

        self.image_encoder = ImageEncoder(
            in_channels=3,
            out_channels=dim,
            heads_rope=heads_rope,
            use_encoder=use_encoder,
            rope_base=rope_base,
            img_layers=img_layers,
            rope_rescale=rope_rescale,
        )

        self.query_encoder = QueryEncoder()

        self.key_encoder = KeyEncoder()

        self.upsampler = CrossAttention(dim=dim, num_heads=heads_attn, kernel_size=(kernel_size, kernel_size))

    def forward(self, image, features, output_size, return_weights=False, *args, **kwargs):
        x = self.image_encoder(image, output_size=output_size)

        queries = self.query_encoder(x)
        keys = self.key_encoder(x, features)
        values = features

        if return_weights:
            out, attn_weights = self.upsampler(queries, keys, values, image, return_weights=True)
            return out, attn_weights
        else:
            out = self.upsampler(queries, keys, values, image)
            return out