Skip to content

SUPRA-Tex: 3D Generation with Super-Aligned Texture

68.9 KB
Raw Blame
from typing import *
from functools import partial
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from ..modules.utils import convert_module_to, manual_cast, str_to_dtype
from ..modules.transformer import AbsolutePositionEmbedder
from ..modules import sparse as sp
from ..modules.sparse.transformer import ModulatedSparseTransformerCrossBlock
from .sparse_structure_flow import TimestepEmbedder


class _PartialZObsInnovationGate(nn.Module):
    """Residual, per-observed-token correction for the partial z_obs carrier."""

    def __init__(self, model_channels: int, message_channels: int, dim: int, time_dim: int, num_blocks: int):
        super().__init__()
        self.alpha = nn.Parameter(torch.zeros(num_blocks))
        self.embed_y = nn.Linear(message_channels, dim, bias=True)
        self.time_proj = nn.Linear(model_channels, time_dim, bias=True)
        self.embed_x = nn.Linear(model_channels + time_dim, dim, bias=True)
        self.mlp = nn.Sequential(
            nn.Linear(dim + 2 + time_dim, dim, bias=True),
            nn.SiLU(),
            nn.Linear(dim, 1, bias=True),
        )
        self.proj = nn.Linear(dim, model_channels, bias=True)


_LOCAL_ONE_HOP_OFFSETS = (
    (-1, 0, 0), (0, -1, 0), (0, 0, -1),
    (0, 0, 1), (0, 1, 0), (1, 0, 0),
)
_LOCAL_TWO_HOP_OFFSETS = tuple(sorted(
    (dx, dy, dz)
    for dx in range(-2, 3)
    for dy in range(-2, 3)
    for dz in range(-2, 3)
    if abs(dx) + abs(dy) + abs(dz) <= 2
))
assert len(_LOCAL_TWO_HOP_OFFSETS) == 25


def _exact_coordinate_rows(
    full_coords: torch.Tensor,
    candidate_coords: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
    """Return exact full-support rows and a validity mask for candidate coords."""
    assert full_coords.ndim == candidate_coords.ndim == 2
    assert full_coords.shape[1] == candidate_coords.shape[1]
    assert full_coords.shape[0] > 0, 'local attention requires nonempty full sparse support'
    all_coords = torch.cat([full_coords, candidate_coords], dim=0)
    _, inverse = torch.unique(all_coords, dim=0, return_inverse=True)
    full_ids = inverse[:full_coords.shape[0]]
    candidate_ids = inverse[full_coords.shape[0]:]
    assert torch.unique(full_ids).numel() == full_ids.numel(), \
        'x contains duplicate coordinates'
    ordered_ids, ordered_rows = torch.sort(full_ids)
    positions = torch.searchsorted(ordered_ids, candidate_ids)
    in_range = positions < ordered_ids.numel()
    safe_positions = positions.clamp(max=ordered_ids.numel() - 1)
    valid = in_range & (ordered_ids[safe_positions] == candidate_ids)
    rows = torch.where(valid, ordered_rows[safe_positions], -torch.ones_like(safe_positions))
    return rows, valid


def _build_two_hop_observed_topology(
    full_coords: torch.Tensor,
    observed_rows: torch.Tensor,
) -> Dict[str, torch.Tensor]:
    """Pack exact batch-local G(S) distance<=2 observed source/query pairs."""
    assert full_coords.ndim == 2 and full_coords.shape[1] == 4, \
        f'local attention requires [N,4] integer coords, got {tuple(full_coords.shape)}'
    assert not torch.is_floating_point(full_coords), 'local attention coordinates must be integer'
    assert observed_rows.ndim == 1 and observed_rows.dtype == torch.long
    # Validate full-coordinate uniqueness even for an all-absent batch.
    _exact_coordinate_rows(full_coords, full_coords.new_empty((0, full_coords.shape[1])))
    if observed_rows.numel() == 0:
        empty = torch.empty(0, dtype=torch.long, device=full_coords.device)
        return {
            'query_rows': empty,
            'source_indices': empty,
            'offset_indices': empty,
            'segment_indices': empty,
            'neighbour_counts': empty,
        }
    assert observed_rows.min() >= 0 and observed_rows.max() < full_coords.shape[0], \
        'observed row lies outside full sparse support'
    assert torch.unique(observed_rows).numel() == observed_rows.numel(), \
        'observed rows must be unique'

    device = full_coords.device
    one_hop = torch.tensor(
        _LOCAL_ONE_HOP_OFFSETS, dtype=full_coords.dtype, device=device,
    )
    source_coords = full_coords[observed_rows]
    candidate_hop1 = source_coords[:, None, :].expand(-1, len(_LOCAL_ONE_HOP_OFFSETS), -1).clone()
    candidate_hop1[:, :, 1:] += one_hop[None]
    hop1_rows, hop1_valid = _exact_coordinate_rows(
        full_coords, candidate_hop1.reshape(-1, full_coords.shape[1]),
    )
    source_ids_hop1 = torch.arange(
        observed_rows.numel(), device=device, dtype=torch.long,
    )[:, None].expand(-1, len(_LOCAL_ONE_HOP_OFFSETS)).reshape(-1)
    hop1_rows = hop1_rows[hop1_valid]
    source_ids_hop1 = source_ids_hop1[hop1_valid]

    if hop1_rows.numel() != 0:
        candidate_hop2 = full_coords[hop1_rows][:, None, :].expand(
            -1, len(_LOCAL_ONE_HOP_OFFSETS), -1,
        ).clone()
        candidate_hop2[:, :, 1:] += one_hop[None]
        hop2_rows, hop2_valid = _exact_coordinate_rows(
            full_coords, candidate_hop2.reshape(-1, full_coords.shape[1]),
        )
        source_ids_hop2 = source_ids_hop1[:, None].expand(
            -1, len(_LOCAL_ONE_HOP_OFFSETS),
        ).reshape(-1)
        hop2_rows = hop2_rows[hop2_valid]
        source_ids_hop2 = source_ids_hop2[hop2_valid]
    else:
        hop2_rows = torch.empty(0, dtype=torch.long, device=device)
        source_ids_hop2 = torch.empty(0, dtype=torch.long, device=device)

    source_ids = torch.cat([
        torch.arange(observed_rows.numel(), device=device, dtype=torch.long),
        source_ids_hop1,
        source_ids_hop2,
    ])
    query_rows = torch.cat([observed_rows, hop1_rows, hop2_rows])
    # Multiple graph paths to the same query/source pair are one neighbourhood entry.
    unique_pairs = torch.unique(torch.stack([query_rows, source_ids], dim=1), dim=0)
    query_rows = unique_pairs[:, 0]
    source_ids = unique_pairs[:, 1]
    query_coords = full_coords[query_rows]
    selected_source_coords = source_coords[source_ids]
    assert torch.equal(query_coords[:, 0], selected_source_coords[:, 0]), \
        'local attention constructed a cross-sample edge'
    relative = query_coords[:, 1:] - selected_source_coords[:, 1:]
    offset_table = torch.tensor(
        _LOCAL_TWO_HOP_OFFSETS, dtype=full_coords.dtype, device=device,
    )
    offset_matches = (relative[:, None, :] == offset_table[None, :, :]).all(dim=-1)
    assert torch.equal(offset_matches.sum(dim=1), torch.ones_like(offset_matches[:, 0], dtype=torch.long)), \
        'local attention produced an invalid two-hop offset'
    offset_indices = offset_matches.to(dtype=torch.long).argmax(dim=1)

    # Canonical per-query real-KV order is lexicographic relative offset, independent
    # of either full-support or observed-message row ordering.
    order = torch.argsort(query_rows * len(_LOCAL_TWO_HOP_OFFSETS) + offset_indices)
    query_rows = query_rows[order]
    source_ids = source_ids[order]
    offset_indices = offset_indices[order]
    active_query_rows, segment_indices, counts = torch.unique_consecutive(
        query_rows, return_inverse=True, return_counts=True,
    )
    assert counts.max() <= len(_LOCAL_TWO_HOP_OFFSETS), \
        'local observed neighbourhood exceeds frozen 25-source cap'
    return {
        'query_rows': active_query_rows,
        'source_indices': source_ids,
        'offset_indices': offset_indices,
        'segment_indices': segment_indices,
        'neighbour_counts': counts,
    }


class _PartialZObsLocalAttention(nn.Module):
    """Shared packed two-hop observed-innovation attention for V12-A."""

    def __init__(
        self,
        model_channels: int,
        message_channels: int,
        dim: int,
        time_dim: int,
        num_heads: int,
        num_active_blocks: int,
    ):
        super().__init__()
        assert dim == 128 and time_dim == 32 and num_heads == 8
        assert dim % num_heads == 0
        self.dim = dim
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        self.alpha = nn.Parameter(torch.zeros(num_active_blocks))
        self.time_proj = nn.Linear(model_channels, time_dim, bias=True)
        self.embed_x = nn.Linear(model_channels + time_dim, dim, bias=True)
        self.embed_y = nn.Linear(message_channels, dim, bias=True)
        self.key_proj = nn.Linear(dim, dim, bias=True)
        self.value_proj = nn.Linear(dim, dim, bias=True)
        self.relative_offset = nn.Parameter(torch.empty(len(_LOCAL_TWO_HOP_OFFSETS), dim))
        self.null_key = nn.Parameter(torch.empty(num_heads, self.head_dim))
        self.out_proj = nn.Linear(dim, model_channels, bias=False)

    @staticmethod
    def _rms_norm(value: torch.Tensor) -> torch.Tensor:
        return value * torch.rsqrt(value.square().mean(dim=-1, keepdim=True) + 1e-6)

    def forward(
        self,
        hidden: torch.Tensor,
        full_batch: torch.Tensor,
        t_base: torch.Tensor,
        message: torch.Tensor,
        observed_rows: torch.Tensor,
        topology: Dict[str, torch.Tensor],
        block_index: int,
    ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]:
        assert 0 <= block_index < self.alpha.numel(), 'terminal block has no V12-A residual'
        query_rows = topology['query_rows']
        source_indices = topology['source_indices']
        offset_indices = topology['offset_indices']
        segment_indices = topology['segment_indices']
        counts = topology['neighbour_counts']
        assert full_batch.ndim == 1 and full_batch.shape[0] == hidden.shape[0]
        assert full_batch.numel() == 0 or (
            full_batch.min() >= 0 and full_batch.max() < t_base.shape[0]
        ), 'full sparse batch coordinate is outside timestep batch'
        assert message.shape == (observed_rows.numel(), 34)

        if query_rows.numel() == 0:
            live_zero = sum(parameter.sum() for parameter in self.parameters()) * 0
            empty_delta = hidden.new_empty((0, hidden.shape[1])) + live_zero
            zero = hidden.new_zeros(())
            return empty_delta, {
                'null_attention_mean': zero,
                'real_neighbour_entropy_mean': zero,
                'effective_real_neighbour_count_mean': zero,
                'active_queries': zero,
                'singleton_queries': zero,
                'packed_pairs': zero,
            }

        branch_dtype = self.embed_x.weight.dtype
        hidden_branch = hidden.to(dtype=branch_dtype)
        time = self.time_proj(t_base.to(dtype=branch_dtype))
        x_embed = self.embed_x(torch.cat([
            F.layer_norm(hidden_branch, hidden_branch.shape[-1:], eps=1e-6),
            time[full_batch],
        ], dim=-1))
        innovation = self.embed_y(message.to(dtype=branch_dtype)) - x_embed[observed_rows]
        query = self._rms_norm(
            x_embed[query_rows].reshape(-1, self.num_heads, self.head_dim)
        )
        real_key = self.key_proj(innovation)[source_indices]
        real_key = real_key + self.relative_offset[offset_indices]
        real_key = self._rms_norm(real_key.reshape(-1, self.num_heads, self.head_dim))
        real_value = self.value_proj(innovation)[source_indices].reshape(
            -1, self.num_heads, self.head_dim,
        )
        real_logits = (query[segment_indices] * real_key).sum(dim=-1) / np.sqrt(self.head_dim)
        null_key = self._rms_norm(self.null_key)
        null_logits = (query * null_key[None]).sum(dim=-1) / np.sqrt(self.head_dim)

        segment_heads = segment_indices[:, None].expand(-1, self.num_heads)
        maxima = null_logits.clone()
        maxima.scatter_reduce_(0, segment_heads, real_logits, reduce='amax', include_self=True)
        real_exp = torch.exp(real_logits - maxima[segment_indices])
        null_exp = torch.exp(null_logits - maxima)
        real_sum = torch.zeros_like(null_exp).scatter_add(0, segment_heads, real_exp)
        denominator = null_exp + real_sum
        real_weight = real_exp / denominator[segment_indices]
        null_weight = null_exp / denominator
        attended = hidden_branch.new_zeros(
            (query_rows.numel(), self.num_heads, self.head_dim),
        )
        attended.index_add_(
            0, segment_indices, real_weight[..., None] * real_value,
        )
        delta = self.out_proj(attended.reshape(query_rows.numel(), self.dim))
        delta = delta * self.alpha[block_index].to(dtype=delta.dtype)

        # Entropy is conditional on selecting real evidence.  Null-vs-real
        # acceptance is reported independently as null_attention_mean; mixing
        # that mass into this distribution would make singleton entropy nonzero.
        real_conditional_weight = real_exp / real_sum[segment_indices].clamp_min(
            torch.finfo(real_sum.dtype).tiny
        )
        entropy_terms = -real_conditional_weight * torch.log(
            real_conditional_weight.clamp_min(
                torch.finfo(real_conditional_weight.dtype).tiny
            )
        )
        real_entropy = torch.zeros_like(null_weight).scatter_add(
            0, segment_heads, entropy_terms,
        )
        return delta, {
            'null_attention_mean': null_weight.mean(),
            'real_neighbour_entropy_mean': real_entropy.mean(),
            'effective_real_neighbour_count_mean': real_entropy.exp().mean(),
            'active_queries': delta.new_tensor(float(query_rows.numel())),
            'singleton_queries': delta.new_tensor(float((counts == 1).sum())),
            'packed_pairs': delta.new_tensor(float(source_indices.numel())),
        }


class _V12LACrossRope(nn.Module):
    """Apply the same 3D rotary basis to independently packed Q and K rows."""

    def __init__(
        self,
        head_dim: int,
        rope_freq: Tuple[float, float],
        reserved_dims: int = 2,
    ):
        super().__init__()
        content_dim = head_dim - reserved_dims
        assert content_dim > 0 and content_dim % 6 == 0
        self.head_dim = head_dim
        self.content_dim = content_dim
        freq_dim = content_dim // 2 // 3
        frequencies = torch.arange(freq_dim, dtype=torch.float32) / freq_dim
        frequencies = rope_freq[0] / (rope_freq[1] ** frequencies)
        self.register_buffer('frequencies', frequencies, persistent=False)

    def forward(self, value: torch.Tensor, coords: torch.Tensor) -> torch.Tensor:
        assert value.ndim == 3 and value.shape[-1] == self.head_dim
        assert coords.shape == (value.shape[0], 3)
        if value.shape[0] == 0:
            return value
        phases = torch.outer(
            coords.reshape(-1).float(), self.frequencies,
        ).reshape(coords.shape[0], self.content_dim // 2)
        phases = torch.polar(torch.ones_like(phases), phases).unsqueeze(1)
        content = torch.view_as_complex(
            value[..., :self.content_dim].float().reshape(
                value.shape[0], value.shape[1], -1, 2,
            )
        )
        rotated = torch.view_as_real(content * phases).reshape(
            value.shape[0], value.shape[1], self.content_dim,
        ).to(dtype=value.dtype)
        return torch.cat([rotated, value[..., self.content_dim:]], dim=-1)


class _PartialZObsLearnedAttention(nn.Module):
    """Five-site V12LA observation attention with an exact carrier anchor."""

    def __init__(
        self,
        model_channels: int,
        num_heads: int,
        sites: Sequence[int],
        rope_freq: Tuple[float, float],
        null_prior: float = 0.5,
    ):
        super().__init__()
        self.sites = tuple(int(site) for site in sites)
        assert self.sites == (5, 10, 15, 20, 25), self.sites
        assert num_heads == 12
        assert model_channels % num_heads == 0
        assert (model_channels // num_heads - 2) > 0
        assert (model_channels // num_heads - 2) % 6 == 0
        self.model_channels = model_channels
        self.num_heads = num_heads
        self.head_dim = model_channels // num_heads
        self.content_dim = self.head_dim - 2
        self.site_to_slot = {site: slot for slot, site in enumerate(self.sites)}
        self.null_prior = float(null_prior)
        assert 0.0 < self.null_prior < 1.0, \
            f'learned-attention null prior must lie in (0,1), got {self.null_prior}'
        self.null_logit_offset = float(
            np.log(self.null_prior) - np.log1p(-self.null_prior)
        )

        # K/V are source-only and shared across all five sites. Q and the
        # zero-start deviation/output matrix are site-specific.
        self.key_proj = nn.Linear(model_channels, model_channels, bias=False)
        self.value_proj = nn.Linear(model_channels, model_channels, bias=False)
        self.query_proj = nn.ModuleList([
            nn.Linear(model_channels, model_channels, bias=False)
            for _ in self.sites
        ])
        self.delta_proj = nn.ModuleList([
            nn.Linear(model_channels, model_channels, bias=False)
            for _ in self.sites
        ])
        self.null_key = nn.Parameter(torch.empty(num_heads, self.content_dim))
        self.rope = _V12LACrossRope(self.head_dim, rope_freq, reserved_dims=2)
        self.reset_parameters()

    def reset_parameters(self) -> None:
        nn.init.xavier_uniform_(self.key_proj.weight)
        # Attention mixes the already trained V11 carrier basis at warm start;
        # the value map may then adapt, but does not begin as a random rewrite.
        nn.init.eye_(self.value_proj.weight)
        nn.init.xavier_uniform_(self.null_key)
        for query_proj in self.query_proj:
            nn.init.xavier_uniform_(query_proj.weight)
        for delta_proj in self.delta_proj:
            nn.init.zeros_(delta_proj.weight)

    @staticmethod
    def _rms_norm(value: torch.Tensor) -> torch.Tensor:
        normalized = value.float() * torch.rsqrt(
            value.float().square().mean(dim=-1, keepdim=True) + 1e-6
        )
        return normalized.to(dtype=value.dtype)

    @staticmethod
    def _anchor(
        carrier: torch.Tensor,
        observed_rows: torch.Tensor,
        full_rows: int,
    ) -> torch.Tensor:
        """Evaluate fixed coordinate/null attention without an additive scatter."""
        assert carrier.ndim == 2
        assert observed_rows.ndim == 1 and observed_rows.dtype == torch.long
        assert carrier.shape[0] == observed_rows.numel()
        assert full_rows >= observed_rows.numel()
        if observed_rows.numel() != 0:
            assert observed_rows.min() >= 0 and observed_rows.max() < full_rows
            assert torch.unique(observed_rows).numel() == observed_rows.numel()
        # Index zero is the exact-zero null value. Every observed row is then
        # redirected to its own carrier row; unmatched rows keep selecting null.
        values = torch.cat([carrier.new_zeros((1, carrier.shape[1])), carrier], dim=0)
        anchor_index = torch.zeros(full_rows, dtype=torch.long, device=carrier.device)
        anchor_index[observed_rows] = torch.arange(
            1, observed_rows.numel() + 1, dtype=torch.long, device=carrier.device,
        )
        return values[anchor_index]

    def build_cache(
        self,
        full_coords: torch.Tensor,
        full_layout: List[slice],
        carrier: torch.Tensor,
        observed_rows: torch.Tensor,
    ) -> Dict[str, Any]:
        """Build the exact anchor and the shared per-sample O+null K/V cache."""
        assert full_coords.ndim == 2 and full_coords.shape[1] == 4
        assert carrier.shape == (observed_rows.numel(), self.model_channels)
        assert len(full_layout) > 0
        anchor = self._anchor(carrier, observed_rows, full_coords.shape[0])
        observed_coords = full_coords[observed_rows]
        observed_batch = observed_coords[:, 0].long()
        assert observed_batch.numel() == 0 or (
            observed_batch.min() >= 0 and observed_batch.max() < len(full_layout)
        )

        branch_dtype = self.key_proj.weight.dtype
        carrier_branch = carrier.to(dtype=branch_dtype)
        carrier_norm = F.layer_norm(
            carrier_branch, (self.model_channels,), eps=1e-6,
        )
        real_key = self.key_proj(carrier_norm).reshape(
            -1, self.num_heads, self.head_dim,
        )
        real_key = self.rope(real_key, observed_coords[:, 1:])
        real_key = self._rms_norm(real_key[..., :self.content_dim])
        real_key = torch.cat([
            real_key,
            real_key.new_zeros((real_key.shape[0], self.num_heads, 2)),
        ], dim=-1)
        real_value = self.value_proj(carrier_branch).reshape(
            -1, self.num_heads, self.head_dim,
        )
        null_content = self._rms_norm(
            self.null_key.to(dtype=real_key.dtype)
        )

        packed_kv = []
        observed_counts = []
        for batch_index in range(len(full_layout)):
            source_indices = torch.nonzero(
                observed_batch == batch_index, as_tuple=False,
            ).flatten()
            observed_count = int(source_indices.numel())
            observed_counts.append(observed_count)
            # One null competes with O real keys.  log(O) cancels its cardinality
            # disadvantage; the constant logit offset sets the null mass only in
            # the equal-content-logit reference case, not after learned Q/K drift.
            null_reserved = real_key.new_zeros((self.num_heads, 2))
            null_reserved[:, 0] = np.sqrt(self.head_dim) * (
                np.log(max(observed_count, 1)) + self.null_logit_offset
            )
            null_key = torch.cat([null_content, null_reserved], dim=-1).unsqueeze(0)
            null_value = real_value.new_zeros((1, self.num_heads, self.head_dim))
            keys = torch.cat([real_key[source_indices], null_key], dim=0)
            values = torch.cat([real_value[source_indices], null_value], dim=0)
            packed_kv.append(torch.stack([keys, values], dim=1))

        kv = sp.VarLenTensor.from_tensor_list(packed_kv)
        return {
            'anchor': anchor,
            'kv': kv,
            'observed_rows': observed_rows,
            'observed_counts': tuple(observed_counts),
        }

    def forward(
        self,
        hidden: torch.Tensor,
        full_coords: torch.Tensor,
        full_layout: List[slice],
        cache: Dict[str, Any],
        block_index: int,
    ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]:
        assert block_index in self.site_to_slot, block_index
        assert hidden.shape == (full_coords.shape[0], self.model_channels)
        slot = self.site_to_slot[block_index]
        branch_dtype = self.query_proj[slot].weight.dtype
        hidden_branch = hidden.to(dtype=branch_dtype)
        query = self.query_proj[slot](F.layer_norm(
            hidden_branch, (self.model_channels,), eps=1e-6,
        )).reshape(-1, self.num_heads, self.head_dim)
        query = self.rope(query, full_coords[:, 1:])
        query_content = self._rms_norm(query[..., :self.content_dim])
        query_reserved = query.new_zeros((query.shape[0], self.num_heads, 2))
        query_reserved[..., 0] = 1
        query = torch.cat([query_content, query_reserved], dim=-1)
        query_varlen = sp.VarLenTensor(query, full_layout)
        attended = sp.sparse_scaled_dot_product_attention(
            query_varlen, cache['kv'],
        ).feats.reshape(-1, self.model_channels)

        anchor = cache['anchor'].to(dtype=attended.dtype)
        deviation_input = attended - anchor
        correction = self.delta_proj[slot](deviation_input)
        inject = anchor + correction
        eps = torch.finfo(inject.dtype).eps
        observed_rows = cache['observed_rows']
        observed_mask = torch.zeros(
            inject.shape[0], dtype=torch.bool, device=inject.device,
        )
        observed_mask[observed_rows] = True
        return inject, {
            'correction_to_anchor_norm_ratio': correction.norm() / anchor.norm().clamp_min(eps),
            'correction_to_hidden_norm_ratio': correction.norm() / hidden_branch.norm().clamp_min(eps),
            'observed_correction_l2_norm': correction[observed_mask].float().norm(),
            'unobserved_correction_l2_norm': correction[~observed_mask].float().norm(),
            'attention_output_l2_norm': attended.float().norm(),
            'anchor_l2_norm': anchor.float().norm(),
            'delta_weight_l2_norm': self.delta_proj[slot].weight.float().norm(),
        }

class SLatFlowModel(nn.Module):
    """
    Structured Latent Flow Model for 3D generation.
    
    Supports two conditioning modes:
    - "cross": Standard cross-attention with image features
    - "proj": View-aligned projection attention with camera-aware features
    """
    def __init__(
        self,
        resolution: int,
        in_channels: int,
        model_channels: int,
        cond_channels: int,
        out_channels: int,
        num_blocks: int,
        num_heads: Optional[int] = None,
        num_head_channels: Optional[int] = 64,
        mlp_ratio: float = 4,
        pe_mode: Literal["ape", "rope"] = "ape",
        rope_freq: Tuple[float, float] = (1.0, 10000.0),
        dtype: str = 'float32',
        use_checkpoint: bool = False,
        share_mod: bool = False,
        initialization: str = 'vanilla',
        qk_rms_norm: bool = False,
        qk_rms_norm_cross: bool = False,
        image_attn_mode: Literal["cross", "proj", "gated_proj"] = "cross",
        proj_in_channels: Optional[int] = None,
        vae_in_channels: Optional[int] = None,
        zobs_inject: bool = False,
        zobs_inject_in_channels: int = 33,
        zobs_inject_scale: float = 1.0,
        partial_zobs_inject: bool = False,
        partial_zobs_in_channels: int = 34,
        partial_zobs_inject_position: Literal['post_block', 'pre_mlp'] = 'post_block',
        legacy_zobs_zero_input: bool = False,
        partial_zobs_innovation_gate: bool = False,
        innovation_gate_dim: int = 128,
        innovation_gate_time_dim: int = 32,
        partial_zobs_innovation_gate_train_only: bool = False,
        partial_zobs_local_attention: bool = False,
        local_attention_dim: int = 128,
        local_attention_time_dim: int = 32,
        local_attention_heads: int = 8,
        partial_zobs_local_attention_train_only: bool = False,
        partial_zobs_learned_attention: bool = False,
        learned_attention_sites: Sequence[int] = (5, 10, 15, 20, 25),
        learned_attention_heads: int = 12,
        learned_attention_null_prior: float = 0.5,
        partial_zobs_learned_attention_train_only: bool = False,
        **kwargs
    ):
        super().__init__()
        self.resolution = resolution
        self.in_channels = in_channels
        self.model_channels = model_channels
        self.cond_channels = cond_channels
        self.out_channels = out_channels
        self.num_blocks = num_blocks
        self.num_heads = num_heads or model_channels // num_head_channels
        self.mlp_ratio = mlp_ratio
        self.pe_mode = pe_mode
        self.use_checkpoint = use_checkpoint
        self.share_mod = share_mod
        self.initialization = initialization
        self.qk_rms_norm = qk_rms_norm
        self.qk_rms_norm_cross = qk_rms_norm_cross
        self.image_attn_mode = image_attn_mode
        self.proj_in_channels = proj_in_channels
        self.vae_in_channels = vae_in_channels
        self.dtype = str_to_dtype(dtype)

        self.t_embedder = TimestepEmbedder(model_channels)
        if share_mod:
            self.adaLN_modulation = nn.Sequential(
                nn.SiLU(),
                nn.Linear(model_channels, 6 * model_channels, bias=True)
            )

        if pe_mode == "ape":
            self.pos_embedder = AbsolutePositionEmbedder(model_channels)

        self.input_layer = sp.SparseLinear(in_channels, model_channels)
        # Trained-bridge: single shared per-block injector of [z_obs (x) vis (32) | vis (1)] -> model_channels.
        self.zobs_inject = zobs_inject
        self.zobs_inject_scale = float(zobs_inject_scale)
        self.zobs_injector = sp.SparseLinear(zobs_inject_in_channels, model_channels) if zobs_inject else None
        # v8-partial: the observation lives only on its observed support.  This adapter is
        # deliberately bias-free: absent rows never acquire a learned content/fill value.
        self.partial_zobs_inject = partial_zobs_inject
        self.partial_zobs_in_channels = partial_zobs_in_channels
        assert partial_zobs_inject_position in ('post_block', 'pre_mlp'), \
            f'partial_zobs_inject_position must be post_block or pre_mlp, got {partial_zobs_inject_position!r}'
        self.partial_zobs_inject_position = partial_zobs_inject_position
        if partial_zobs_inject_position == 'pre_mlp':
            assert not any((
                partial_zobs_innovation_gate,
                partial_zobs_local_attention,
                partial_zobs_learned_attention,
            )), 'partial_zobs_inject_position=pre_mlp is incompatible with post-block experimental branches'
        self.legacy_zobs_zero_input = legacy_zobs_zero_input
        self.partial_zobs_injector = (
            sp.SparseLinear(partial_zobs_in_channels, model_channels, bias=False)
            if partial_zobs_inject else None
        )
        self.partial_zobs_innovation_gate = partial_zobs_innovation_gate
        self.partial_zobs_innovation_gate_train_only = partial_zobs_innovation_gate_train_only
        if partial_zobs_innovation_gate_train_only:
            assert partial_zobs_innovation_gate, \
                'partial_zobs_innovation_gate_train_only requires partial_zobs_innovation_gate=true'
        if partial_zobs_innovation_gate:
            assert partial_zobs_inject, 'partial_zobs_innovation_gate requires partial_zobs_inject=true'
            assert partial_zobs_in_channels == 34, \
                f'partial_zobs_innovation_gate requires 34 input channels, got {partial_zobs_in_channels}'
            assert innovation_gate_dim == 128, f'innovation_gate_dim is fixed at 128, got {innovation_gate_dim}'
            assert innovation_gate_time_dim == 32, \
                f'innovation_gate_time_dim is fixed at 32, got {innovation_gate_time_dim}'
            assert model_channels == 1536, f'partial_zobs_innovation_gate requires 1536 model channels, got {model_channels}'
            assert num_blocks == 30, f'partial_zobs_innovation_gate requires 30 blocks, got {num_blocks}'
            self.innovation_gate = _PartialZObsInnovationGate(
                model_channels, partial_zobs_in_channels, innovation_gate_dim,
                innovation_gate_time_dim, num_blocks,
            )
        else:
            self.innovation_gate = None
        self.partial_zobs_local_attention = partial_zobs_local_attention
        self.partial_zobs_local_attention_train_only = partial_zobs_local_attention_train_only
        if partial_zobs_local_attention_train_only:
            assert partial_zobs_local_attention, \
                'partial_zobs_local_attention_train_only requires partial_zobs_local_attention=true'
        assert sum(bool(value) for value in (
            partial_zobs_innovation_gate,
            partial_zobs_local_attention,
            partial_zobs_learned_attention,
        )) <= 1, 'partial-zobs experimental attention/gate branches are mutually exclusive'
        if partial_zobs_local_attention:
            assert partial_zobs_inject, 'partial_zobs_local_attention requires partial_zobs_inject=true'
            assert partial_zobs_in_channels == 34, \
                f'partial_zobs_local_attention requires 34 input channels, got {partial_zobs_in_channels}'
            assert local_attention_dim == 128, \
                f'local_attention_dim is fixed at 128, got {local_attention_dim}'
            assert local_attention_time_dim == 32, \
                f'local_attention_time_dim is fixed at 32, got {local_attention_time_dim}'
            assert local_attention_heads == 8, \
                f'local_attention_heads is fixed at 8, got {local_attention_heads}'
            assert model_channels == 1536, \
                f'partial_zobs_local_attention requires 1536 model channels, got {model_channels}'
            assert num_blocks == 30, \
                f'partial_zobs_local_attention requires 30 blocks, got {num_blocks}'
            self.local_obs_attention = _PartialZObsLocalAttention(
                model_channels, partial_zobs_in_channels, local_attention_dim,
                local_attention_time_dim, local_attention_heads, num_blocks - 1,
            )
        else:
            self.local_obs_attention = None

        self.partial_zobs_learned_attention = partial_zobs_learned_attention
        self.partial_zobs_learned_attention_train_only = \
            partial_zobs_learned_attention_train_only
        if partial_zobs_learned_attention_train_only:
            assert partial_zobs_learned_attention, \
                'partial_zobs_learned_attention_train_only requires partial_zobs_learned_attention=true'
        if partial_zobs_learned_attention:
            assert partial_zobs_inject, \
                'partial_zobs_learned_attention requires partial_zobs_inject=true'
            assert partial_zobs_in_channels == 34, \
                f'partial_zobs_learned_attention requires 34 input channels, got {partial_zobs_in_channels}'
            assert model_channels == 1536, \
                f'partial_zobs_learned_attention requires 1536 model channels, got {model_channels}'
            assert num_blocks == 30, \
                f'partial_zobs_learned_attention requires 30 blocks, got {num_blocks}'
            assert learned_attention_heads == 12, \
                f'partial_zobs_learned_attention requires 12 heads, got {learned_attention_heads}'
            assert tuple(learned_attention_sites) == (5, 10, 15, 20, 25), \
                f'V12LA sites are fixed at [5,10,15,20,25], got {learned_attention_sites}'
            self.learned_attention = _PartialZObsLearnedAttention(
                model_channels,
                learned_attention_heads,
                learned_attention_sites,
                rope_freq,
                learned_attention_null_prior,
            )
        else:
            self.learned_attention = None

        self.blocks = nn.ModuleList([
            ModulatedSparseTransformerCrossBlock(
                model_channels,
                cond_channels,
                num_heads=self.num_heads,
                mlp_ratio=self.mlp_ratio,
                attn_mode='full',
                use_checkpoint=self.use_checkpoint,
                use_rope=(pe_mode == "rope"),
                rope_freq=rope_freq,
                share_mod=self.share_mod,
                qk_rms_norm=self.qk_rms_norm,
                qk_rms_norm_cross=self.qk_rms_norm_cross,
                image_attn_mode=image_attn_mode,
                proj_in_channels=proj_in_channels,
                vae_in_channels=vae_in_channels,
            )
            for _ in range(num_blocks)
        ])
            
        self.out_layer = sp.SparseLinear(model_channels, out_channels)

        self.initialize_weights()
        self.convert_to(self.dtype)
        if self.partial_zobs_innovation_gate_train_only:
            self._freeze_for_partial_zobs_innovation_gate()
        if self.partial_zobs_local_attention_train_only:
            self._freeze_for_partial_zobs_local_attention()
        if self.partial_zobs_learned_attention_train_only:
            self._freeze_for_partial_zobs_learned_attention()

    @property
    def device(self) -> torch.device:
        """
        Return the device of the model.
        """
        return next(self.parameters()).device

    def convert_to(self, dtype: torch.dtype) -> None:
        """
        Convert the torso of the model to the specified dtype.
        """
        self.dtype = dtype
        self.blocks.apply(partial(convert_module_to, dtype=dtype))
        if self.learned_attention is not None:
            self.learned_attention.apply(partial(convert_module_to, dtype=dtype))

    def initialize_weights(self) -> None:
        if self.initialization == 'vanilla':
            # Initialize transformer layers:
            def _basic_init(module):
                if isinstance(module, nn.Linear):
                    torch.nn.init.xavier_uniform_(module.weight)
                    if module.bias is not None:
                        nn.init.constant_(module.bias, 0)
            self.apply(_basic_init)

            # Initialize timestep embedding MLP:
            nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
            nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)

            # Zero-out adaLN modulation layers in DiT blocks:
            if self.share_mod:
                nn.init.constant_(self.adaLN_modulation[-1].weight, 0)
                nn.init.constant_(self.adaLN_modulation[-1].bias, 0)
            else:
                for block in self.blocks:
                    nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
                    nn.init.constant_(block.adaLN_modulation[-1].bias, 0)

            # Zero-out output layers:
            nn.init.constant_(self.out_layer.weight, 0)
            nn.init.constant_(self.out_layer.bias, 0)
            
        elif self.initialization == 'scaled':
            # Initialize transformer layers:
            def _basic_init(module):
                if isinstance(module, nn.Linear):
                    torch.nn.init.normal_(module.weight, std=np.sqrt(2.0 / (5.0 * self.model_channels)))
                    if module.bias is not None:
                        nn.init.constant_(module.bias, 0)
            self.apply(_basic_init)
            
            # Scaled init for to_out and ffn2
            def _scaled_init(module):
                if isinstance(module, nn.Linear):
                    torch.nn.init.normal_(module.weight, std=1.0 / np.sqrt(5 * self.num_blocks * self.model_channels))
                    if module.bias is not None:
                        nn.init.constant_(module.bias, 0)
            for block in self.blocks:
                block.self_attn.to_out.apply(_scaled_init)
                # Handle cross, proj, and gated_proj modes
                if self.image_attn_mode in ("proj", "gated_proj"):
                    block.cross_attn.cross_attn_block.to_out.apply(_scaled_init)
                else:
                    block.cross_attn.to_out.apply(_scaled_init)
                block.mlp.mlp[2].apply(_scaled_init)
            
            # Initialize input layer to make the initial representation have variance 1
            nn.init.normal_(self.input_layer.weight, std=1.0 / np.sqrt(self.in_channels))
            nn.init.zeros_(self.input_layer.bias)
            
            # Initialize timestep embedding MLP:
            nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
            nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
            
            # Zero-out adaLN modulation layers in DiT blocks:
            if self.share_mod:
                nn.init.constant_(self.adaLN_modulation[-1].weight, 0)
                nn.init.constant_(self.adaLN_modulation[-1].bias, 0)
            else:
                for block in self.blocks:
                    nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
                    nn.init.constant_(block.adaLN_modulation[-1].bias, 0)

            # Zero-out output layers:
            nn.init.constant_(self.out_layer.weight, 0)
            nn.init.constant_(self.out_layer.bias, 0)

        # Trained-bridge: zero-init the z_obs injector LAST (after _basic_init), so step-0 injection == 0.
        if getattr(self, 'zobs_injector', None) is not None:
            nn.init.zeros_(self.zobs_injector.weight)
            nn.init.zeros_(self.zobs_injector.bias)
        if getattr(self, 'partial_zobs_injector', None) is not None:
            nn.init.zeros_(self.partial_zobs_injector.weight)
        if getattr(self, 'innovation_gate', None) is not None:
            nn.init.zeros_(self.innovation_gate.alpha)
        if getattr(self, 'local_obs_attention', None) is not None:
            # V12-A freezes vanilla Xavier initialization independently of the
            # scaled V11 backbone initialization mode. Only alpha starts at zero.
            for module in self.local_obs_attention.modules():
                if isinstance(module, nn.Linear):
                    nn.init.xavier_uniform_(module.weight)
                    if module.bias is not None:
                        nn.init.zeros_(module.bias)
            nn.init.xavier_uniform_(self.local_obs_attention.relative_offset)
            nn.init.xavier_uniform_(self.local_obs_attention.null_key)
            nn.init.zeros_(self.local_obs_attention.alpha)
        if getattr(self, 'learned_attention', None) is not None:
            self.learned_attention.reset_parameters()

    def _freeze_for_partial_zobs_innovation_gate(self) -> None:
        """Make the V12 gate the sole trainable branch before an optimizer is built."""
        assert self.innovation_gate is not None
        for parameter in self.parameters():
            parameter.requires_grad_(False)
        for parameter in self.innovation_gate.parameters():
            parameter.requires_grad_(True)
        trainable = [name for name, parameter in self.named_parameters() if parameter.requires_grad]
        expected = [f'innovation_gate.{name}' for name in self.innovation_gate.state_dict()]
        assert trainable == expected, (trainable, expected)
        assert sum(parameter.numel() for parameter in self.parameters() if parameter.requires_grad) == 473663

    def _freeze_for_partial_zobs_local_attention(self) -> None:
        """Make the V12-A local attention the sole trainable warm-start branch."""
        assert self.local_obs_attention is not None
        for parameter in self.parameters():
            parameter.requires_grad_(False)
        for parameter in self.local_obs_attention.parameters():
            parameter.requires_grad_(True)
        trainable = [name for name, parameter in self.named_parameters() if parameter.requires_grad]
        expected = [f'local_obs_attention.{name}' for name in self.local_obs_attention.state_dict()]
        assert trainable == expected, (trainable, expected)
        assert len(trainable) == 14
        assert sum(parameter.numel() for parameter in self.parameters() if parameter.requires_grad) == 487485

    def _freeze_for_partial_zobs_learned_attention(self) -> None:
        """Make the V12LA module the sole trainable warm-start branch."""
        assert self.learned_attention is not None
        for parameter in self.parameters():
            parameter.requires_grad_(False)
        for parameter in self.learned_attention.parameters():
            parameter.requires_grad_(True)
        trainable = [name for name, parameter in self.named_parameters() if parameter.requires_grad]
        expected = [
            f'learned_attention.{name}'
            for name, _ in self.learned_attention.named_parameters()
        ]
        assert trainable == expected, (trainable, expected)
        assert len(trainable) == 13, trainable

    def _partial_zobs_message(
        self,
        x: sp.SparseTensor,
        partial_z_obs: sp.SparseTensor,
        partial_z_obs_keep_mask: Optional[torch.Tensor],
    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
        """Project a variable-support observation and join its messages onto ``x`` rows.

        The return value stays variable-length: projected messages plus their exact full-row
        indices.  Neither observation content nor a zero-filled model-channel tensor is
        materialized on full support. ``torch.unique(..., dim=0)`` makes the coordinate join
        exact and rejects duplicate or non-subset partial coordinates instead of assuming a
        row order.
        """
        adapter = self.partial_zobs_injector
        assert adapter is not None
        assert partial_z_obs.feats.ndim == 2 and \
            partial_z_obs.feats.shape[-1] == self.partial_zobs_in_channels, \
            f'partial_z_obs must be [N,{self.partial_zobs_in_channels}], got {tuple(partial_z_obs.feats.shape)}'
        assert partial_z_obs.coords.ndim == 2 and partial_z_obs.coords.shape[1] == x.coords.shape[1], \
            f'partial_z_obs coordinate rank mismatch: {tuple(partial_z_obs.coords.shape)} vs {tuple(x.coords.shape)}'

        active = torch.ones(partial_z_obs.feats.shape[0], dtype=torch.bool, device=partial_z_obs.feats.device)
        if partial_z_obs_keep_mask is not None:
            assert partial_z_obs_keep_mask.ndim == 1 and partial_z_obs_keep_mask.shape[0] == x.shape[0], \
                f'partial_z_obs_keep_mask must be [B={x.shape[0]}], got {tuple(partial_z_obs_keep_mask.shape)}'
            keep = partial_z_obs_keep_mask.to(device=partial_z_obs.feats.device, dtype=torch.bool)
            active = keep[partial_z_obs.coords[:, 0].long()]

        active_coords = partial_z_obs.coords[active]
        active_feats = partial_z_obs.feats[active]
        # F.linear on a zero-row tensor is intentional: it keeps the adapter parameters in the
        # graph on all-drop batches, so DDP sees a zero (rather than None) gradient.
        active_messages = F.linear(active_feats, adapter.weight, None)

        full_rows = torch.empty(0, dtype=torch.long, device=x.feats.device)
        if active_coords.shape[0] != 0:
            all_coords = torch.cat([x.coords, active_coords], dim=0)
            _, inverse = torch.unique(all_coords, dim=0, return_inverse=True)
            x_ids = inverse[:x.coords.shape[0]]
            partial_ids = inverse[x.coords.shape[0]:]
            assert torch.unique(x_ids).numel() == x_ids.numel(), 'x contains duplicate coordinates'
            assert torch.unique(partial_ids).numel() == partial_ids.numel(), 'partial_z_obs contains duplicate coordinates'
            ordered_x_ids, ordered_x_rows = torch.sort(x_ids)
            positions = torch.searchsorted(ordered_x_ids, partial_ids)
            assert (positions < ordered_x_ids.numel()).all() and \
                torch.equal(ordered_x_ids[positions], partial_ids), \
                'partial_z_obs coordinates must be an exact subset of current x coordinates'
            full_rows = ordered_x_rows[positions]

        return active_messages, full_rows, active_feats, active_coords[:, 0].long()

    def forward(
        self,
        x: sp.SparseTensor,
        t: torch.Tensor,
        cond: Union[torch.Tensor, List[torch.Tensor], Dict[str, Union[torch.Tensor, sp.SparseTensor]], Tuple],
        concat_cond: Optional[sp.SparseTensor] = None,
        z_obs: Optional[sp.SparseTensor] = None,
        partial_z_obs: Optional[sp.SparseTensor] = None,
        partial_z_obs_keep_mask: Optional[torch.Tensor] = None,
        **kwargs
    ) -> sp.SparseTensor:
        """
        Forward pass.

        Args:
            x: SparseTensor input
            t: Timestep tensor [B]
            cond: Conditioning tensor. For "cross" mode: list of tensors or tensor.
                  For "proj" mode: dict {'global': global_cond, 'proj': proj_cond}
                  or tuple of (global_cond, proj_cond)
            concat_cond: Optional concatenation condition
            z_obs: Optional 33ch [z_obs(32)|vis(1)] sparse tensor, same coords as x. Feeds ONLY
                   the zobs_injector (v6 patchcross path: concat_cond is the plain 32ch shape
                   latent and z_obs never enters the input_layer). When None, the legacy 97ch
                   concat slice below applies — shipped/v4 behavior is bit-identical.
            partial_z_obs: Optional variable-length v8 observation
                [normalized_z_obs(32)|coverage(1)|boundary_distance(1)].
                Its coordinates must be a subset of x; it is consumed only by the dedicated,
                bias-free partial adapter and is never converted to full-support content.

        Returns:
            SparseTensor output
        """
        if concat_cond is not None:
            x = sp.sparse_cat([x, concat_cond], dim=-1)

        h = self.input_layer(x)
        h = manual_cast(h, self.dtype)

        # Trained-bridge: compute the per-block injection vector ONCE (shared zero-init linear, added each block).
        # Sources, in priority order: separate z_obs kwarg (v6), else the trailing channels of a
        # 97-ch [x_t|shape|z_obs|vis] input (v4). z_obs*vis==0 at vis=0 in both.
        zobs_inject = None
        _injector = getattr(self, 'zobs_injector', None)
        if z_obs is not None:
            # Fail fast: without an injector the kwarg would be a silent no-op, and in the 64ch
            # patchcross architecture the injector is the ONLY z_obs consumption path.
            assert _injector is not None, \
                'z_obs kwarg passed but the denoiser was built without zobs_inject=true'
            # Full coords equality is asserted at data-load; a row-count check catches gross mismatch.
            assert z_obs.feats.shape[-1] == 33 and z_obs.feats.shape[0] == x.feats.shape[0], \
                f'z_obs kwarg mismatch: {tuple(z_obs.feats.shape)} vs x rows {x.feats.shape[0]}'
            _zo = z_obs.feats[:, :32]
            _vis = z_obs.feats[:, 32:33]
            zobs_inject = manual_cast(self.zobs_injector(z_obs.replace(torch.cat([_zo * _vis, _vis], dim=-1))), self.dtype)
            if not getattr(self, '_zobs_kwarg_logged', False):  # in-log proof hook (once per process)
                print('[zobs_injector] consuming separate z_obs kwarg (patchcross path)', flush=True)
                self._zobs_kwarg_logged = True
        elif _injector is not None and x.feats.shape[-1] >= 97:
            _zo = x.feats[:, 64:96]
            _vis = x.feats[:, 96:97]
            zobs_inject = manual_cast(self.zobs_injector(x.replace(torch.cat([_zo * _vis, _vis], dim=-1))), self.dtype)
        elif _injector is not None and self.legacy_zobs_zero_input:
            # 64ch v8 shell with a legacy injector: preserve the legacy all-zero z_obs mode,
            # including its learned per-block bias, without materialising observation content.
            # Evaluate one zero row and broadcast the result: this is exactly W*0+b, keeps a
            # zero (non-None) weight gradient under DDP, and avoids an N-row 33ch GEMM.
            zero_output = F.linear(
                x.feats.new_zeros((1, _injector.in_features)),
                _injector.weight,
                _injector.bias,
            )
            zobs_inject = manual_cast(
                x.replace(zero_output.expand(x.feats.shape[0], -1)), self.dtype
            )
        elif _injector is not None and not getattr(self, '_zobs_none_warned', False):
            # Injector present but NO z_obs source (64ch input, kwarg omitted): the trained
            # per-block bias is NOT applied — a third mode distinct from "z_obs dropped" (an
            # all-zero 33ch tensor still adds the bias each block). A nozobs arm on a patchcross
            # model must pass the zero tensor, never omit the kwarg.
            print('[zobs_injector] WARNING: injector present but no z_obs source (kwarg omitted, '
                  'input < 97ch) — per-block bias NOT applied; pass an all-zero 33ch z_obs for a '
                  'true nozobs arm', flush=True)
            self._zobs_none_warned = True

        partial_zobs_inject = None
        partial_zobs_rows = None
        partial_zobs_feats = None
        partial_zobs_batch = None
        if partial_z_obs is not None:
            assert self.partial_zobs_injector is not None, \
                'partial_z_obs passed but denoiser was built without partial_zobs_inject=true'
            partial_zobs_inject, partial_zobs_rows, partial_zobs_feats, partial_zobs_batch = self._partial_zobs_message(
                x, partial_z_obs, partial_z_obs_keep_mask
            )
            partial_zobs_inject = manual_cast(partial_zobs_inject, self.dtype)
        elif self.partial_zobs_injector is not None:
            raise AssertionError('partial_zobs_inject=true requires a separate partial_z_obs tensor')

        if zobs_inject is not None and self.zobs_inject_scale != 1.0:
            zobs_inject = zobs_inject.replace(zobs_inject.feats * self.zobs_inject_scale)
            if not getattr(self, '_zobs_inject_scale_logged', False):
                print(f'[zobs_injector] output scale={self.zobs_inject_scale:g}', flush=True)
                self._zobs_inject_scale_logged = True

        # Keep the pre-AdaLN embedding for the innovation gate; blocks retain their existing
        # modulation input exactly.
        t_base = self.t_embedder(t)
        t_emb = t_base
        if self.share_mod:
            t_emb = self.adaLN_modulation(t_emb)
        t_emb = manual_cast(t_emb, self.dtype)

        innovation_time = None
        innovation_y = None
        innovation_gate_y = None
        if self.innovation_gate is not None and partial_zobs_feats is not None:
            # Observation-only terms are shared by all blocks. Empty rows deliberately pass
            # through these linears too, so an all-drop batch keeps the full gate graph live.
            gate_dtype = self.innovation_gate.embed_y.weight.dtype
            innovation_gate_y = partial_zobs_feats.to(dtype=gate_dtype)
            innovation_y = self.innovation_gate.embed_y(innovation_gate_y)
            innovation_time = self.innovation_gate.time_proj(t_base)

        if self.pe_mode == "ape":
            pe = self.pos_embedder(h.coords[:, 1:])
            h = h + manual_cast(pe, self.dtype)
            
        # Handle different conditioning modes
        if self.image_attn_mode == 'proj':
            if isinstance(cond, dict):
                global_cond = cond['global']
                proj_cond = cond['proj']
            else:
                global_cond, proj_cond = cond
            if isinstance(global_cond, list):
                global_cond = sp.VarLenTensor.from_tensor_list(global_cond)
            global_cond = manual_cast(global_cond, self.dtype)
            proj_cond = manual_cast(proj_cond, self.dtype)
            cond = (global_cond, proj_cond)
        elif self.image_attn_mode == 'gated_proj':
            global_cond = cond['global']
            if isinstance(global_cond, list):
                global_cond = sp.VarLenTensor.from_tensor_list(global_cond)
            global_cond = manual_cast(global_cond, self.dtype)
            proj_semantic = manual_cast(cond['proj_semantic'], self.dtype)
            proj_color = manual_cast(cond['proj_color'], self.dtype)
            cond = {'global': global_cond, 'proj_semantic': proj_semantic, 'proj_color': proj_color}
        else:
            if isinstance(cond, list):
                cond = sp.VarLenTensor.from_tensor_list(cond)
            cond = manual_cast(cond, self.dtype)

        # Inference-only probe hook. When the attribute is absent (the default, and the
        # only state any training path produces) this is None and the loop below is
        # byte-identical to the unhooked path: no constructor arg, no config key, and no
        # entry in the inference ABI contract. A probe sets it to a length-num_blocks
        # sequence to rescale the shared partial message per block.
        _partial_scales = getattr(self, 'partial_zobs_layer_scales', None)
        # Optional evaluator-only callable: ``fn(gate_value, active_batch, block_index)``.
        # It is an instance attribute rather than a constructor/config argument, so no model
        # state or training ABI changes; callers can implement neutral, constant, or shuffled
        # gate-value probes deterministically.
        _gate_value_transform = getattr(self, 'partial_zobs_innovation_gate_value_transform', None)
        _innovation_ratio_stats = [] if self.innovation_gate is not None else None
        _innovation_gate_mean_stats = [] if self.innovation_gate is not None else None
        _innovation_observed_rows = 0 if partial_zobs_rows is None else int(partial_zobs_rows.numel())
        _local_attention = getattr(self, 'local_obs_attention', None)
        _local_topology = None
        _local_full_batch = None
        _local_ratio_base_stats = [] if _local_attention is not None else None
        _local_ratio_hidden_stats = [] if _local_attention is not None else None
        _local_null_stats = [] if _local_attention is not None else None
        _local_entropy_stats = [] if _local_attention is not None else None
        _local_effective_neighbour_stats = [] if _local_attention is not None else None
        _local_observed_residual_stats = [] if _local_attention is not None else None
        _local_unobserved_residual_stats = [] if _local_attention is not None else None
        _local_query_is_observed = None
        _local_neighbourhood_histogram = None
        if _local_attention is not None:
            assert partial_zobs_rows is not None and partial_zobs_feats is not None
            _local_topology = _build_two_hop_observed_topology(x.coords, partial_zobs_rows)
            _local_full_batch = x.coords[:, 0].long()
            _active_observed_mask = torch.zeros(
                x.coords.shape[0], dtype=torch.bool, device=x.coords.device,
            )
            _active_observed_mask[partial_zobs_rows] = True
            _local_query_is_observed = _active_observed_mask[
                _local_topology['query_rows']
            ]
            _local_neighbourhood_histogram = torch.bincount(
                _local_topology['neighbour_counts'], minlength=26,
            )[:26]
            _local_neighbourhood_histogram[0] = (
                x.coords.shape[0] - _local_topology['query_rows'].numel()
            )
            assert int(_local_neighbourhood_histogram.sum()) == x.coords.shape[0]
            assert int(_local_neighbourhood_histogram[1:].sum()) == \
                _local_topology['query_rows'].numel()
        _learned_attention = getattr(self, 'learned_attention', None)
        _learned_cache = None
        _learned_stats = [] if _learned_attention is not None else None
        if _learned_attention is not None:
            assert partial_zobs_inject is not None and partial_zobs_rows is not None
            assert _partial_scales is None, \
                'partial_zobs_layer_scales is not defined for the five-site V12LA operator'
            _learned_cache = _learned_attention.build_cache(
                x.coords,
                x.layout,
                partial_zobs_inject,
                partial_zobs_rows,
            )
        partial_zobs_inject_position = getattr(
            self, 'partial_zobs_inject_position', 'post_block',
        )
        for _block_index, block in enumerate(self.blocks):
            if partial_zobs_inject is not None and partial_zobs_inject_position == 'pre_mlp':
                _pre_mlp_scale = 1.0 if _partial_scales is None else float(
                    _partial_scales[_block_index]
                )
                h = block(
                    h,
                    t_emb,
                    cond,
                    partial_zobs_rows=partial_zobs_rows,
                    partial_zobs_inject=partial_zobs_inject,
                    partial_zobs_inject_scale=_pre_mlp_scale,
                )
            else:
                h = block(h, t_emb, cond)
            if zobs_inject is not None:
                h = h + zobs_inject
            _innovation_ratio = None
            _innovation_gate_mean = None
            _local_delta = None
            _local_stats = None
            if _local_attention is not None and _block_index < _local_attention.alpha.numel():
                assert _local_topology is not None and _local_full_batch is not None
                _local_delta, _local_stats = _local_attention(
                    h.feats,
                    _local_full_batch,
                    t_base,
                    partial_zobs_feats,
                    partial_zobs_rows,
                    _local_topology,
                    _block_index,
                )
                _local_delta = manual_cast(_local_delta, self.dtype)
                _local_ratio_base = _local_delta.norm() / partial_zobs_inject.norm().clamp_min(
                    torch.finfo(partial_zobs_inject.dtype).eps
                )
                _local_hidden = h.feats[_local_topology['query_rows']]
                _local_ratio_hidden = _local_delta.norm() / _local_hidden.norm().clamp_min(
                    torch.finfo(_local_hidden.dtype).eps
                )
                assert _local_query_is_observed is not None
                _local_observed_residual = _local_delta[
                    _local_query_is_observed
                ].float().norm()
                _local_unobserved_residual = _local_delta[
                    ~_local_query_is_observed
                ].float().norm()
            if _learned_attention is not None and _block_index in _learned_attention.site_to_slot:
                assert _learned_cache is not None
                _learned_inject, _site_stats = _learned_attention(
                    h.feats,
                    h.coords,
                    h.layout,
                    _learned_cache,
                    _block_index,
                )
                h = h.replace(h.feats + manual_cast(_learned_inject, self.dtype))
                _learned_stats.append({
                    name: value.detach() for name, value in _site_stats.items()
                })
            if partial_zobs_inject is not None and _learned_attention is None and \
                    partial_zobs_inject_position == 'post_block':
                layer_inject = partial_zobs_inject
                if self.innovation_gate is not None:
                    gate = self.innovation_gate
                    # Every operation intentionally accepts an empty leading dimension. This
                    # keeps all innovation parameters in the all-drop DDP graph.
                    assert innovation_gate_y is not None and innovation_y is not None and innovation_time is not None
                    t_obs = innovation_time[partial_zobs_batch]
                    h_obs = h.feats[partial_zobs_rows].to(dtype=gate.embed_y.weight.dtype)
                    x_embed = gate.embed_x(torch.cat([h_obs, t_obs], dim=-1))
                    innovation = innovation_y - x_embed
                    gate_input = torch.cat([
                        innovation,
                        innovation_gate_y[:, 32:34],
                        t_obs,
                    ], dim=-1)
                    gate_value = torch.sigmoid(gate.mlp(gate_input))
                    if _gate_value_transform is not None:
                        gate_value = _gate_value_transform(
                            gate_value, partial_zobs_batch, _block_index
                        )
                        assert gate_value.shape == (innovation.shape[0], 1), \
                            f'gate value transform must return [Nobs,1], got {tuple(gate_value.shape)}'
                    delta = gate.proj(innovation) * gate_value
                    delta = delta * gate.alpha[_block_index].to(dtype=delta.dtype)
                    if _block_index == len(self.blocks) - 1:
                        # Keep the frozen V11 base carrier at the terminal site,
                        # but never add a newly learned observed-only shortcut:
                        # no attention block remains to transport/smooth it into U.
                        delta = delta * 0
                    # Detached telemetry for the training log. This is before the legacy
                    # layer-scale hook and therefore measures the learned branch itself.
                    _innovation_ratio = delta.norm() / partial_zobs_inject.norm().clamp_min(
                        torch.finfo(partial_zobs_inject.dtype).eps
                    )
                    _innovation_gate_mean = (
                        gate_value.mean() if gate_value.numel() != 0 else delta.new_zeros(())
                    )
                    layer_inject = partial_zobs_inject + manual_cast(delta, self.dtype)
                if partial_zobs_rows.numel() != 0:
                    if _partial_scales is None:
                        h = h.replace(h.feats.index_add(
                            0, partial_zobs_rows, layer_inject
                        ))
                    else:
                        h = h.replace(h.feats.index_add(
                            0, partial_zobs_rows, layer_inject,
                            alpha=float(_partial_scales[_block_index]),
                        ))
                else:
                    # Preserve every empty carrier/gate GEMM in the graph on all-drop batches.
                    h = h.replace(h.feats + layer_inject.sum() * 0)
                if _local_delta is not None:
                    _local_scale = 1.0 if _partial_scales is None else float(
                        _partial_scales[_block_index]
                    )
                    if _local_topology['query_rows'].numel() != 0:
                        h = h.replace(h.feats.index_add(
                            0, _local_topology['query_rows'], _local_delta,
                            alpha=_local_scale,
                        ))
                    else:
                        # The module returns an exact-zero parameter edge for all 14
                        # trainable tensors, so DDP observes finite numerical-zero grads.
                        h = h.replace(h.feats + _local_delta.sum() * _local_scale)
            if self.innovation_gate is not None:
                if _innovation_ratio is None:
                    _innovation_ratio = h.feats.new_zeros(())
                    _innovation_gate_mean = h.feats.new_zeros(())
                _innovation_ratio_stats.append(_innovation_ratio.detach())
                _innovation_gate_mean_stats.append(_innovation_gate_mean.detach())
            if _local_attention is not None and _block_index < _local_attention.alpha.numel():
                assert _local_stats is not None
                _local_ratio_base_stats.append(_local_ratio_base.detach())
                _local_ratio_hidden_stats.append(_local_ratio_hidden.detach())
                _local_null_stats.append(_local_stats['null_attention_mean'].detach())
                _local_entropy_stats.append(_local_stats['real_neighbour_entropy_mean'].detach())
                _local_effective_neighbour_stats.append(
                    _local_stats['effective_real_neighbour_count_mean'].detach()
                )
                _local_observed_residual_stats.append(
                    _local_observed_residual.detach()
                )
                _local_unobserved_residual_stats.append(
                    _local_unobserved_residual.detach()
                )

        if self.innovation_gate is not None:
            # Instance-only telemetry: deliberately neither buffer nor state-dict entry.
            self.partial_zobs_innovation_gate_last_stats = {
                'delta_to_base_norm_ratio': torch.stack(_innovation_ratio_stats),
                'gate_mean': torch.stack(_innovation_gate_mean_stats),
                'observed_rows': _innovation_observed_rows,
            }
        if _local_attention is not None:
            # Block 29 is structurally absent: every telemetry vector has length 29.
            assert _local_query_is_observed is not None
            assert _local_neighbourhood_histogram is not None
            _full_support_rows = int(x.coords.shape[0])
            _active_queries = int(_local_topology['query_rows'].numel())
            _empty_queries = _full_support_rows - _active_queries
            _singleton_queries = int(
                (_local_topology['neighbour_counts'] == 1).sum()
            )
            self.partial_zobs_local_attention_last_stats = {
                'delta_to_base_norm_ratio': torch.stack(_local_ratio_base_stats),
                'delta_to_hidden_norm_ratio': torch.stack(_local_ratio_hidden_stats),
                'null_attention_mean': torch.stack(_local_null_stats),
                'real_neighbour_entropy_mean': torch.stack(_local_entropy_stats),
                'effective_real_neighbour_count_mean': torch.stack(
                    _local_effective_neighbour_stats
                ),
                'observed_residual_l2_norm': torch.stack(
                    _local_observed_residual_stats
                ),
                'unobserved_residual_l2_norm': torch.stack(
                    _local_unobserved_residual_stats
                ),
                'observed_rows': _innovation_observed_rows,
                'full_support_rows': _full_support_rows,
                'active_queries': _active_queries,
                'empty_queries': _empty_queries,
                'singleton_queries': _singleton_queries,
                'observed_active_queries': int(_local_query_is_observed.sum()),
                'unobserved_active_queries': int((~_local_query_is_observed).sum()),
                'active_query_fraction': _active_queries / max(_full_support_rows, 1),
                'empty_query_fraction': _empty_queries / max(_full_support_rows, 1),
                'singleton_query_fraction': _singleton_queries / max(_active_queries, 1),
                'neighbourhood_size_histogram': _local_neighbourhood_histogram.detach(),
                'packed_pairs': int(_local_topology['source_indices'].numel()),
            }
        if _learned_attention is not None:
            assert _learned_cache is not None
            assert len(_learned_stats) == len(_learned_attention.sites)
            self.partial_zobs_learned_attention_last_stats = {
                name: torch.stack([site[name] for site in _learned_stats])
                for name in _learned_stats[0]
            }
            self.partial_zobs_learned_attention_last_stats.update({
                'sites': _learned_attention.sites,
                'observed_rows': int(partial_zobs_rows.numel()),
                'full_support_rows': int(x.coords.shape[0]),
                'observed_counts': _learned_cache['observed_counts'],
            })

        h = manual_cast(h, x.dtype)
        h = h.replace(F.layer_norm(h.feats, h.feats.shape[-1:]))
        h = self.out_layer(h)
        return h


# Existing checkpoint configs use this name; inference uses the same base model.
ElasticSLatFlowModel = SLatFlowModel