SUPRA-Tex: 3D Generation with Super-Aligned Texture
import math
from pathlib import Path
from typing import NamedTuple
import torch
from transformers import Gemma4UnifiedConfig, Gemma4UnifiedForConditionalGeneration, PreTrainedTokenizerFast
from .ltx2_common import rms_norm
from .ltx2_dit import (
Attention,
FeedForward,
LTXRopeType,
generate_freq_grid_np,
generate_freq_grid_pytorch,
precompute_freqs_cis,
)
class LTX25TextEncoder(Gemma4UnifiedForConditionalGeneration):
def __init__(self):
config = {
"architectures": ["Gemma4UnifiedForConditionalGeneration"],
"audio_config": {
"_name_or_path": "",
"architectures": None,
"audio_embed_dim": 640,
"chunk_size_feed_forward": 0,
"dtype": "bfloat16",
"id2label": {"0": "LABEL_0", "1": "LABEL_1"},
"initializer_range": 0.02,
"is_encoder_decoder": False,
"label2id": {"LABEL_0": 0, "LABEL_1": 1},
"model_type": "gemma4_unified_audio",
"output_attentions": False,
"output_hidden_states": False,
"problem_type": None,
"return_dict": True,
"rms_norm_eps": 1e-06,
},
"audio_token_id": 258881,
"boa_token_id": 256000,
"boi_token_id": 255999,
"dtype": "bfloat16",
"eoa_token_index": 258883,
"eoi_token_id": 258882,
"eos_token_id": [1, 106],
"gemma_version": "gemma4-12b-ltx-v1",
"image_token_id": 258880,
"initializer_range": 0.02,
"model_type": "gemma4_unified",
"text_config": {
"attention_bias": False,
"attention_dropout": 0.0,
"attention_k_eq_v": True,
"bos_token_id": 2,
"dtype": "bfloat16",
"enable_moe_block": False,
"eos_token_id": 1,
"final_logit_softcapping": 30.0,
"global_head_dim": 512,
"head_dim": 256,
"hidden_activation": "gelu_pytorch_tanh",
"hidden_size": 3840,
"hidden_size_per_layer_input": 0,
"initializer_range": 0.02,
"intermediate_size": 15360,
"layer_types": (["sliding_attention"] * 5 + ["full_attention"]) * 8,
"max_position_embeddings": 262144,
"model_type": "gemma4_unified_text",
"moe_intermediate_size": None,
"num_attention_heads": 16,
"num_experts": None,
"num_global_key_value_heads": 1,
"num_hidden_layers": 48,
"num_key_value_heads": 8,
"num_kv_shared_layers": 0,
"pad_token_id": 0,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"full_attention": {"partial_rotary_factor": 0.25, "rope_theta": 1000000.0, "rope_type": "proportional"},
"sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"},
},
"sliding_window": 1024,
"tie_word_embeddings": True,
"top_k_experts": None,
"use_bidirectional_attention": "vision",
"use_cache": True,
"use_double_wide_mlp": False,
"vocab_size": 262144,
"vocab_size_per_layer_input": 262144,
},
"tie_word_embeddings": True,
"transformers_version": "5.10.1",
"video_token_id": 258884,
"vision_config": {
"_name_or_path": "",
"architectures": None,
"chunk_size_feed_forward": 0,
"dtype": "bfloat16",
"id2label": {"0": "LABEL_0", "1": "LABEL_1"},
"initializer_range": 0.02,
"is_encoder_decoder": False,
"label2id": {"LABEL_0": 0, "LABEL_1": 1},
"mm_embed_dim": 3840,
"mm_posemb_size": 1120,
"model_type": "gemma4_unified_vision",
"num_soft_tokens": 280,
"output_attentions": False,
"output_hidden_states": False,
"output_proj_dims": 3840,
"patch_size": 16,
"pooling_kernel_size": 3,
"problem_type": None,
"return_dict": True,
"rms_norm_eps": 1e-06,
},
}
super().__init__(Gemma4UnifiedConfig(**config))
# Gemma4 registers a constant `layer_scalar` as a persistent buffer on each decoder layer.
# Disk-offload reload restores only named_parameters(), so a persistent buffer is never fetched
# from disk and strict load_state_dict fails; register it as a frozen parameter instead.
for module in self.modules():
if "layer_scalar" in module._buffers:
module.register_parameter("layer_scalar", torch.nn.Parameter(module._buffers.pop("layer_scalar"), requires_grad=False))
class LTX25GemmaTokenizer:
def __init__(self, tokenizer_path: str | Path, max_length: int = 1024):
self.tokenizer = PreTrainedTokenizerFast.from_pretrained(tokenizer_path, local_files_only=True, model_max_length=max_length)
self.tokenizer.model_max_length = max_length
self.tokenizer.padding_side = "left"
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
self.max_length = max_length
def tokenize_with_weights(self, text: str) -> dict[str, list[tuple[int, int]]]:
text = text.strip()
bos_id = self.tokenizer.bos_token_id
if bos_id is None:
raise ValueError("Packed Gemma tokenizer has no BOS token id.")
encoded = self.tokenizer(text, padding=False, truncation=True, max_length=self.max_length, return_tensors="pt")
input_ids = encoded.input_ids[0].tolist()
if not input_ids or input_ids[0] != bos_id:
input_ids = [bos_id, *input_ids][: self.max_length]
padded = self.tokenizer.pad(
{"input_ids": [input_ids]},
padding="max_length",
max_length=self.max_length,
return_tensors="pt",
return_attention_mask=True,
)
return {"gemma": list(zip(padded.input_ids[0].tolist(), padded.attention_mask[0].tolist(), strict=True))}
def norm_and_concat_per_token_rms(
encoded_text: torch.Tensor,
attention_mask: torch.Tensor,
) -> torch.Tensor:
batch_size, sequence_length, embedding_dim, num_layers = encoded_text.shape
variance = torch.mean(encoded_text**2, dim=2, keepdim=True)
normed = encoded_text * torch.rsqrt(variance + 1e-6)
normed = normed.reshape(batch_size, sequence_length, embedding_dim * num_layers)
return torch.where(attention_mask.bool().unsqueeze(-1), normed, torch.zeros_like(normed))
def _rescale_norm(x: torch.Tensor, target_dim: int, source_dim: int) -> torch.Tensor:
return x * math.sqrt(target_dim / source_dim)
class LTX25FeatureExtractorV2(torch.nn.Module):
def __init__(
self,
embedding_dim: int = 3840,
num_layers: int = 49,
video_out_features: int = 4096,
audio_out_features: int = 2048,
):
super().__init__()
self.video_aggregate_embed = torch.nn.Linear(embedding_dim * num_layers, video_out_features, bias=True)
self.audio_aggregate_embed = (
torch.nn.Linear(embedding_dim * num_layers, audio_out_features, bias=True)
if audio_out_features is not None
else None
)
self.embedding_dim = embedding_dim
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
padding_side: str = "left",
) -> tuple[torch.Tensor, torch.Tensor | None]:
del padding_side
encoded = torch.stack(hidden_states, dim=-1) if isinstance(hidden_states, (list, tuple)) else hidden_states
normed = norm_and_concat_per_token_rms(encoded, attention_mask).to(encoded.dtype)
video = self.video_aggregate_embed(
_rescale_norm(normed, self.video_aggregate_embed.out_features, self.embedding_dim)
)
audio = None
if self.audio_aggregate_embed is not None:
audio = self.audio_aggregate_embed(
_rescale_norm(normed, self.audio_aggregate_embed.out_features, self.embedding_dim)
)
return video, audio
class LTX25Embeddings1DConnector(torch.nn.Module):
def __init__(
self,
attention_head_dim: int,
num_attention_heads: int,
num_layers: int,
positional_embedding_theta: float = 10000.0,
positional_embedding_max_pos: list[int] | None = None,
num_learnable_registers: int | None = 128,
rope_type: LTXRopeType = LTXRopeType.SPLIT,
double_precision_rope: bool = True,
apply_gated_attention: bool = True,
ff_bias: bool = False,
):
super().__init__()
self.num_attention_heads = num_attention_heads
self.inner_dim = num_attention_heads * attention_head_dim
self.positional_embedding_theta = positional_embedding_theta
self.positional_embedding_max_pos = positional_embedding_max_pos if positional_embedding_max_pos is not None else [1]
self.rope_type = rope_type
self.double_precision_rope = double_precision_rope
self.transformer_1d_blocks = torch.nn.ModuleList(
[
LTX25BasicTransformerBlock1D(
dim=self.inner_dim,
heads=num_attention_heads,
dim_head=attention_head_dim,
rope_type=rope_type,
apply_gated_attention=apply_gated_attention,
ff_bias=ff_bias,
)
for _ in range(num_layers)
]
)
self.num_learnable_registers = num_learnable_registers
if self.num_learnable_registers:
self.learnable_registers = torch.nn.Parameter(
torch.rand(self.num_learnable_registers, self.inner_dim, dtype=torch.bfloat16) * 2.0 - 1.0
)
def _replace_padded_with_learnable_registers(
self,
hidden_states: torch.Tensor,
additive_attention_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
batch_size, sequence_length, _ = hidden_states.shape
assert sequence_length % self.num_learnable_registers == 0
registers = self.learnable_registers.to(hidden_states).repeat(sequence_length // self.num_learnable_registers, 1)
registers = registers.unsqueeze(0).expand(batch_size, -1, -1)
binary_mask = (additive_attention_mask[:, 0, 0, :].unsqueeze(-1) >= 0).to(hidden_states.dtype)
hidden_states = binary_mask * hidden_states + (1 - binary_mask) * registers
return hidden_states, torch.zeros_like(additive_attention_mask)
def forward(
self,
hidden_states: torch.Tensor,
additive_attention_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.num_learnable_registers:
hidden_states, additive_attention_mask = self._replace_padded_with_learnable_registers(
hidden_states,
additive_attention_mask,
)
indices_grid = torch.arange(hidden_states.shape[1], dtype=torch.float32, device=hidden_states.device)
indices_grid = indices_grid[None, None, :].expand(hidden_states.shape[0], -1, -1)
freq_grid_generator = generate_freq_grid_np if self.double_precision_rope else generate_freq_grid_pytorch
freqs_cis = precompute_freqs_cis(
indices_grid=indices_grid,
dim=self.inner_dim,
out_dtype=hidden_states.dtype,
theta=self.positional_embedding_theta,
max_pos=self.positional_embedding_max_pos,
num_attention_heads=self.num_attention_heads,
rope_type=self.rope_type,
freq_grid_generator=freq_grid_generator,
)
for block in self.transformer_1d_blocks:
hidden_states = block(hidden_states, additive_attention_mask=additive_attention_mask, pe=freqs_cis)
return rms_norm(hidden_states), additive_attention_mask
class LTX25BasicTransformerBlock1D(torch.nn.Module):
def __init__(
self,
dim: int,
heads: int,
dim_head: int,
rope_type: LTXRopeType,
apply_gated_attention: bool,
ff_bias: bool,
):
super().__init__()
self.attn1 = Attention(
query_dim=dim,
heads=heads,
dim_head=dim_head,
rope_type=rope_type,
apply_gated_attention=apply_gated_attention,
)
self.ff = FeedForward(dim, dim_out=dim, bias=ff_bias)
def forward(
self,
hidden_states: torch.Tensor,
additive_attention_mask: torch.Tensor | None = None,
pe: torch.Tensor | None = None,
) -> torch.Tensor:
norm_hidden_states = rms_norm(hidden_states).squeeze(1)
hidden_states = self.attn1(norm_hidden_states, mask=additive_attention_mask, pe=pe) + hidden_states
if hidden_states.ndim == 4:
hidden_states = hidden_states.squeeze(1)
hidden_states = self.ff(rms_norm(hidden_states)) + hidden_states
return hidden_states.squeeze(1) if hidden_states.ndim == 4 else hidden_states
class LTX25EmbeddingsProcessorOutput(NamedTuple):
video_encoding: torch.Tensor
audio_encoding: torch.Tensor | None
attention_mask: torch.Tensor
def _convert_to_additive_mask(attention_mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
return (attention_mask.to(torch.int64) - 1).to(dtype).reshape(
attention_mask.shape[0], 1, 1, attention_mask.shape[-1]
) * torch.finfo(dtype).max
def _right_pad_order(additive_attention_mask: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
binary = (additive_attention_mask[:, 0, 0, :] >= 0).to(torch.int32)
sort_indices = torch.argsort(binary, dim=-1, descending=True, stable=True)
reordered = torch.gather(binary, 1, sort_indices)
additive = (reordered.to(additive_attention_mask.dtype) - 1) * torch.finfo(additive_attention_mask.dtype).max
return sort_indices, additive[:, None, None, :]
class LTX25EmbeddingsConnectors(torch.nn.Module):
def __init__(
self,
video_attention_heads: int = 32,
video_attention_head_dim: int = 128,
audio_attention_heads: int = 32,
audio_attention_head_dim: int = 64,
num_connector_layers: int = 8,
connector_max_positions: list[int] | None = None,
connector_ff_bias: bool = True,
):
super().__init__()
connector_max_positions = [4096] if connector_max_positions is None else connector_max_positions
self.video_connector = LTX25Embeddings1DConnector(
attention_head_dim=video_attention_head_dim,
num_attention_heads=video_attention_heads,
num_layers=num_connector_layers,
positional_embedding_max_pos=connector_max_positions,
ff_bias=connector_ff_bias,
)
self.audio_connector = LTX25Embeddings1DConnector(
attention_head_dim=audio_attention_head_dim,
num_attention_heads=audio_attention_heads,
num_layers=num_connector_layers,
positional_embedding_max_pos=connector_max_positions,
ff_bias=connector_ff_bias,
)
class LTX25TextEncoderPostModules(torch.nn.Module):
def __init__(
self,
embedding_dim: int = 3840,
num_layers: int = 49,
video_attention_heads: int = 32,
video_attention_head_dim: int = 128,
audio_attention_heads: int = 32,
audio_attention_head_dim: int = 64,
num_connector_layers: int = 8,
connector_max_positions: list[int] | None = None,
connector_ff_bias: bool = True,
feature_extractor: LTX25FeatureExtractorV2 | None = None,
connectors: LTX25EmbeddingsConnectors | None = None,
):
super().__init__()
self.feature_extractor = (
feature_extractor
if feature_extractor is not None
else LTX25FeatureExtractorV2(
embedding_dim=embedding_dim,
num_layers=num_layers,
video_out_features=video_attention_heads * video_attention_head_dim,
audio_out_features=audio_attention_heads * audio_attention_head_dim,
)
)
self.connectors = (
connectors
if connectors is not None
else LTX25EmbeddingsConnectors(
video_attention_heads=video_attention_heads,
video_attention_head_dim=video_attention_head_dim,
audio_attention_heads=audio_attention_heads,
audio_attention_head_dim=audio_attention_head_dim,
num_connector_layers=num_connector_layers,
connector_max_positions=connector_max_positions,
connector_ff_bias=connector_ff_bias,
)
)
@property
def video_connector(self):
return self.connectors.video_connector
@property
def audio_connector(self):
return self.connectors.audio_connector
def create_embeddings(
self,
video_features: torch.Tensor,
audio_features: torch.Tensor | None,
additive_attention_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
if audio_features is None:
raise ValueError("LTX-2.5 requires audio features for the audio connector.")
sort_indices, connector_mask = _right_pad_order(additive_attention_mask)
video_features = torch.gather(video_features, 1, sort_indices.unsqueeze(-1).expand_as(video_features))
video_encoded, video_mask = self.video_connector(video_features, connector_mask)
binary_mask = (video_mask < 0.000001).to(torch.int64).reshape(video_encoded.shape[0], video_encoded.shape[1], 1)
video_encoded = video_encoded * binary_mask
audio_features = torch.gather(audio_features, 1, sort_indices.unsqueeze(-1).expand_as(audio_features))
audio_encoded, _ = self.audio_connector(audio_features, connector_mask)
return video_encoded, audio_encoded, binary_mask.squeeze(-1)
def process_hidden_states(
self,
hidden_states: tuple[torch.Tensor, ...],
attention_mask: torch.Tensor,
padding_side: str = "left",
) -> LTX25EmbeddingsProcessorOutput:
video_features, audio_features = self.feature_extractor(hidden_states, attention_mask, padding_side)
additive_attention_mask = _convert_to_additive_mask(attention_mask, video_features.dtype)
video_encoding, audio_encoding, binary_mask = self.create_embeddings(
video_features,
audio_features,
additive_attention_mask,
)
return LTX25EmbeddingsProcessorOutput(video_encoding, audio_encoding, binary_mask)