Skip to content

SUPRA-Tex: 3D Generation with Super-Aligned Texture

190 KB
Raw Blame
from __future__ import annotations

import dataclasses
import itertools
import logging
import math

from tqdm import tqdm
from collections.abc import Iterator, Sequence
from dataclasses import dataclass, replace
from enum import Enum
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Callable, Final, List, Literal, NamedTuple, Protocol, Tuple, TYPE_CHECKING

import numpy as np
import torch
import torch.nn.functional as F
from einops import rearrange
from torch import nn
from torch.nn import functional


class Disposable:
    pass
class VideoDecoder:
    pass
def _clip_generators(count, generator):
    if isinstance(generator, Sequence):
        if len(generator) != count:
            raise ValueError(f"decode_single_frames got {count} latents and {len(generator)} generators")
        return generator
    return [generator] * count


def iter_decoded_single_frames(decoder, latents, generator=None):
    generators = _clip_generators(len(latents), generator)
    for index, (latent, item_generator) in enumerate(zip(latents, generators, strict=True)):
        if latent.ndim != 5 or latent.shape[2] != 1:
            raise ValueError(
                f"decode_single_frames expects (B, C, 1, H, W) latents, got {tuple(latent.shape)} at index {index}"
            )
        chunks = list(decoder.decode_video(latent, tiling_config=None, generator=item_generator))
        if not chunks:
            raise RuntimeError(f"Decoder returned no pixels for single-frame latent {index}")
        yield torch.cat(chunks, dim=0)


def get_timestep_embedding(
    timesteps: torch.Tensor,
    embedding_dim: int,
    flip_sin_to_cos: bool = False,
    downscale_freq_shift: float = 1,
    scale: float = 1,
    max_period: int = 10000,
) -> torch.Tensor:
    assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"

    half_dim = embedding_dim // 2
    exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device)
    exponent = exponent / (half_dim - downscale_freq_shift)

    emb = torch.exp(exponent)
    emb = timesteps[:, None].float() * emb[None, :]

    # scale embeddings
    emb = scale * emb

    # concat sine and cosine embeddings
    emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)

    # flip sine and cosine embeddings
    if flip_sin_to_cos:
        emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)

    # zero pad
    if embedding_dim % 2 == 1:
        emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
    return emb


class TimestepEmbedding(torch.nn.Module):
    def __init__(
        self,
        in_channels: int,
        time_embed_dim: int,
        out_dim: int | None = None,
        post_act_fn: str | None = None,
        cond_proj_dim: int | None = None,
        sample_proj_bias: bool = True,
    ):
        super().__init__()

        self.linear_1 = torch.nn.Linear(in_channels, time_embed_dim, sample_proj_bias)

        if cond_proj_dim is not None:
            self.cond_proj = torch.nn.Linear(cond_proj_dim, in_channels, bias=False)
        else:
            self.cond_proj = None

        self.act = torch.nn.SiLU()
        time_embed_dim_out = out_dim if out_dim is not None else time_embed_dim

        self.linear_2 = torch.nn.Linear(time_embed_dim, time_embed_dim_out, sample_proj_bias)

        if post_act_fn is None:
            self.post_act = None

    def forward(self, sample: torch.Tensor, condition: torch.Tensor | None = None) -> torch.Tensor:
        if condition is not None:
            sample = sample + self.cond_proj(condition)
        sample = self.linear_1(sample)

        if self.act is not None:
            sample = self.act(sample)

        sample = self.linear_2(sample)

        if self.post_act is not None:
            sample = self.post_act(sample)
        return sample


class Timesteps(torch.nn.Module):
    def __init__(self, num_channels: int, flip_sin_to_cos: bool, downscale_freq_shift: float, scale: int = 1):
        super().__init__()
        self.num_channels = num_channels
        self.flip_sin_to_cos = flip_sin_to_cos
        self.downscale_freq_shift = downscale_freq_shift
        self.scale = scale

    def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
        t_emb = get_timestep_embedding(
            timesteps,
            self.num_channels,
            flip_sin_to_cos=self.flip_sin_to_cos,
            downscale_freq_shift=self.downscale_freq_shift,
            scale=self.scale,
        )
        return t_emb


class PixArtAlphaCombinedTimestepSizeEmbeddings(torch.nn.Module):
    def __init__(
        self,
        embedding_dim: int,
        size_emb_dim: int,
    ):
        super().__init__()

        self.outdim = size_emb_dim
        self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0)
        self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)

    def forward(
        self,
        timestep: torch.Tensor,
        hidden_dtype: torch.dtype,
    ) -> torch.Tensor:
        timesteps_proj = self.time_proj(timestep)
        timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_dtype))  # (N, D)
        return timesteps_emb


class VideoPixelShape(NamedTuple):
    batch: int
    frames: int
    height: int
    width: int
    fps: float


class SpatioTemporalScaleFactors(NamedTuple):
    time: int
    height: int
    width: int

    @classmethod
    def default(cls) -> "SpatioTemporalScaleFactors":
        return cls(time=8, height=32, width=32)

    @classmethod
    def from_blocks(cls, blocks: list, patch_size: int) -> "SpatioTemporalScaleFactors":
        spatial_steps = 0
        temporal_steps = 0
        for block_name, _ in blocks:
            if block_name.startswith(("compress_space", "compress_all")):
                spatial_steps += 1
            if block_name.startswith(("compress_time", "compress_all")):
                temporal_steps += 1
        spatial = patch_size * (2**spatial_steps)
        return cls(time=2**temporal_steps, height=spatial, width=spatial)

VIDEO_SCALE_FACTORS = SpatioTemporalScaleFactors.default()


class VideoLatentShape(NamedTuple):
    batch: int
    channels: int
    frames: int
    height: int
    width: int

    def to_torch_shape(self) -> torch.Size:
        return torch.Size([self.batch, self.channels, self.frames, self.height, self.width])

    @staticmethod
    def from_torch_shape(shape: torch.Size) -> "VideoLatentShape":
        return VideoLatentShape(
            batch=shape[0],
            channels=shape[1],
            frames=shape[2],
            height=shape[3],
            width=shape[4],
        )

    @staticmethod
    def from_pixel_shape(
        shape: VideoPixelShape,
        latent_channels: int = 128,
        scale_factors: SpatioTemporalScaleFactors = VIDEO_SCALE_FACTORS,
    ) -> "VideoLatentShape":
        frames = (shape.frames - 1) // scale_factors.time + 1
        height = shape.height // scale_factors.height
        width = shape.width // scale_factors.width

        return VideoLatentShape(
            batch=shape.batch,
            channels=latent_channels,
            frames=frames,
            height=height,
            width=width,
        )

    def upscale(self, scale_factors: SpatioTemporalScaleFactors = VIDEO_SCALE_FACTORS) -> "VideoLatentShape":
        return self._replace(
            channels=3,
            frames=(self.frames - 1) * scale_factors.time + 1,
            height=self.height * scale_factors.height,
            width=self.width * scale_factors.width,
        )


class AudioLatentShape(NamedTuple):
    batch: int
    channels: int
    frames: int
    mel_bins: int

    def to_torch_shape(self) -> torch.Size:
        return torch.Size([self.batch, self.channels, self.frames, self.mel_bins])

    @staticmethod
    def from_torch_shape(shape: torch.Size) -> "AudioLatentShape":
        return AudioLatentShape(
            batch=shape[0],
            channels=shape[1],
            frames=shape[2],
            mel_bins=shape[3],
        )

    @staticmethod
    def from_duration(
        batch: int,
        duration: float,
        channels: int = 8,
        mel_bins: int = 16,
        sample_rate: int = 16000,
        hop_length: int = 160,
        audio_latent_downsample_factor: int = 4,
    ) -> "AudioLatentShape":
        latents_per_second = float(sample_rate) / float(hop_length) / float(audio_latent_downsample_factor)

        return AudioLatentShape(
            batch=batch,
            channels=channels,
            frames=round(duration * latents_per_second),
            mel_bins=mel_bins,
        )

    @staticmethod
    def from_video_pixel_shape(
        shape: VideoPixelShape,
        channels: int = 8,
        mel_bins: int = 16,
        sample_rate: int = 16000,
        hop_length: int = 160,
        audio_latent_downsample_factor: int = 4,
    ) -> "AudioLatentShape":
        return AudioLatentShape.from_duration(
            batch=shape.batch,
            duration=float(shape.frames) / float(shape.fps),
            channels=channels,
            mel_bins=mel_bins,
            sample_rate=sample_rate,
            hop_length=hop_length,
            audio_latent_downsample_factor=audio_latent_downsample_factor,
        )


@dataclass(frozen=True)
class Audio:
    waveform: torch.Tensor
    sampling_rate: int

    def to(self, **kwargs: object) -> "Audio":
        return replace(self, waveform=self.waveform.to(**kwargs))


@dataclass(frozen=True)
class GeneratedKeyframeLayout:
    pixel_frame_indices: tuple[int, ...]
    tokens_per_keyframe: int
    first_token: int

    @property
    def num_keyframes(self) -> int:
        return len(self.pixel_frame_indices)

    @property
    def num_tokens(self) -> int:
        return self.num_keyframes * self.tokens_per_keyframe

@dataclass(frozen=True)
class LatentState:
    latent: torch.Tensor
    denoise_mask: torch.Tensor
    positions: torch.Tensor
    clean_latent: torch.Tensor
    attention_mask: torch.Tensor | None = None
    keyframes_mask: torch.Tensor | None = None
    generated_keyframe_layout: GeneratedKeyframeLayout | None = None
    generated_keyframes: torch.Tensor | None = None
    frozen: bool = False

    def clone(self) -> "LatentState":
        return LatentState(
            latent=self.latent.clone(),
            denoise_mask=self.denoise_mask.clone(),
            positions=self.positions.clone(),
            clean_latent=self.clean_latent.clone(),
            attention_mask=self.attention_mask.clone() if self.attention_mask is not None else None,
            keyframes_mask=self.keyframes_mask.clone() if self.keyframes_mask is not None else None,
            generated_keyframe_layout=self.generated_keyframe_layout,
            generated_keyframes=(self.generated_keyframes.clone() if self.generated_keyframes is not None else None),
            frozen=self.frozen,
        )


def rms_norm(x: torch.Tensor, weight: torch.Tensor | None = None, eps: float = 1e-6) -> torch.Tensor:
    return torch.nn.functional.rms_norm(x, (x.shape[-1],), weight=weight, eps=eps)


def to_velocity(
    sample: torch.Tensor,
    sigma: float | torch.Tensor,
    denoised_sample: torch.Tensor,
    calc_dtype: torch.dtype = torch.float32,
) -> torch.Tensor:
    if isinstance(sigma, torch.Tensor):
        sigma = sigma.to(calc_dtype).item()
    if sigma == 0:
        raise ValueError("Sigma can't be 0.0")
    return ((sample.to(calc_dtype) - denoised_sample.to(calc_dtype)) / sigma).to(sample.dtype)


def to_denoised(
    sample: torch.Tensor,
    velocity: torch.Tensor,
    sigma: float | torch.Tensor,
    calc_dtype: torch.dtype = torch.float32,
) -> torch.Tensor:
    if isinstance(sigma, torch.Tensor):
        sigma = sigma.to(calc_dtype)
    return (sample.to(calc_dtype) - velocity.to(calc_dtype) * sigma).to(sample.dtype)


def compute_trapezoidal_mask_1d(
    length: int,
    ramp_left: int,
    ramp_right: int,
    left_starts_from_0: bool = False,
) -> torch.Tensor:
    if length <= 0:
        raise ValueError("Mask length must be positive.")

    ramp_left = max(0, min(ramp_left, length))
    ramp_right = max(0, min(ramp_right, length))

    mask = torch.ones(length)

    if ramp_left > 0:
        interval_length = ramp_left + 1 if left_starts_from_0 else ramp_left + 2
        fade_in = torch.linspace(0.0, 1.0, interval_length)[:-1]
        if not left_starts_from_0:
            fade_in = fade_in[1:]
        mask[:ramp_left] *= fade_in

    if ramp_right > 0:
        fade_out = torch.linspace(1.0, 0.0, steps=ramp_right + 2)[1:-1]
        mask[-ramp_right:] *= fade_out

    return mask.clamp_(0, 1)


def compute_rectangular_mask_1d(
    length: int,
    left_ramp: int,
    right_ramp: int,
) -> torch.Tensor:
    if length <= 0:
        raise ValueError("Mask length must be positive.")

    mask = torch.ones(length)
    if left_ramp > 0:
        mask[:left_ramp] = 0
    if right_ramp > 0:
        mask[-right_ramp:] = 0
    return mask


@dataclass(frozen=True)
class DimensionInterval:
    start: int
    end: int
    left_ramp: int
    right_ramp: int


@dataclass(frozen=True)
class DimensionIntervals:
    intervals: list[DimensionInterval]


@dataclass(frozen=True)
class LatentIntervals:
    original_shape: torch.Size
    dimension_intervals: tuple[DimensionIntervals, ...]


SplitOperation = Callable[[int], DimensionIntervals]


MappingOperation = Callable[[DimensionIntervals], tuple[list[slice], list[torch.Tensor]]]


def default_split_operation(length: int) -> DimensionIntervals:
    return DimensionIntervals(intervals=[DimensionInterval(start=0, end=length, left_ramp=0, right_ramp=0)])


DEFAULT_SPLIT_OPERATION: SplitOperation = default_split_operation


def untiled_mask_1d() -> torch.Tensor:
    return torch.ones(1)


def default_mapping_operation(
    _intervals: DimensionIntervals,
) -> tuple[list[slice], list[torch.Tensor]]:
    return [slice(0, None)], [untiled_mask_1d()]


DEFAULT_MAPPING_OPERATION: MappingOperation = default_mapping_operation


def _grow_last_tile_to_min(intervals: list[DimensionInterval], min_tile_size: int) -> list[DimensionInterval]:
    if len(intervals) <= 1:
        return list(intervals)
    last = intervals[-1]
    if last.end - last.start >= min_tile_size:
        return list(intervals)
    new_start = last.end - min_tile_size
    prev = intervals[-2]
    new_overlap = prev.end - new_start
    return [
        *intervals[:-2],
        replace(prev, right_ramp=new_overlap),
        replace(last, start=new_start, left_ramp=new_overlap),
    ]


def _validate_tile_intervals(intervals: list[DimensionInterval], *, dim_size: int, min_tile_size: int) -> None:
    if not intervals or intervals[0].start != 0 or intervals[-1].end != dim_size:
        raise ValueError(f"tiles must cover [0, {dim_size})")
    for i, iv in enumerate(intervals):
        length = iv.end - iv.start
        if length < min_tile_size:
            raise ValueError(f"tile {i} length {length} is below min_tile_size={min_tile_size}")
        if iv.left_ramp < 0 or iv.right_ramp < 0 or iv.left_ramp > length or iv.right_ramp > length:
            raise ValueError(f"tile {i} has invalid ramps: left={iv.left_ramp}, right={iv.right_ramp}, length={length}")
        if i == 0:
            continue
        overlap = intervals[i - 1].end - iv.start
        if overlap < 0 or intervals[i - 1].right_ramp != overlap or iv.left_ramp != overlap:
            raise ValueError(f"tiles {i - 1}/{i}: ramp/overlap mismatch (overlap={overlap})")


def split_by_size(size: int, overlap: int, min_tile_size: int | None = None) -> SplitOperation:
    if size <= 0:
        raise ValueError(f"size must be > 0, got {size}")
    if overlap < 0 or overlap >= size:
        raise ValueError(f"overlap must satisfy 0 <= overlap < size, got overlap={overlap}, size={size}")
    if min_tile_size is not None and min_tile_size < 1:
        raise ValueError(f"min_tile_size must be >= 1, got {min_tile_size}")

    def split(dimension_size: int) -> DimensionIntervals:
        if min_tile_size is not None and dimension_size < min_tile_size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        if dimension_size <= size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        amount = (dimension_size + size - 2 * overlap - 1) // (size - overlap)
        intervals = [
            DimensionInterval(start=0, end=size, left_ramp=0, right_ramp=overlap),
            *(
                DimensionInterval(
                    start=i * (size - overlap),
                    end=i * (size - overlap) + size,
                    left_ramp=overlap,
                    right_ramp=overlap,
                )
                for i in range(1, amount - 1)
            ),
            DimensionInterval(
                start=(amount - 1) * (size - overlap), end=dimension_size, left_ramp=overlap, right_ramp=0
            ),
        ]
        if min_tile_size is not None:
            intervals = _grow_last_tile_to_min(intervals, min_tile_size)
            _validate_tile_intervals(intervals, dim_size=dimension_size, min_tile_size=min_tile_size)
        return DimensionIntervals(intervals=intervals)

    return split


def split_temporal_causal(size: int, overlap: int, min_tile_size: int | None = None) -> SplitOperation:
    non_causal_split = split_by_size(size, overlap, min_tile_size=min_tile_size)

    def split(dimension_size: int) -> DimensionIntervals:
        if dimension_size <= size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        dim_intervals = non_causal_split(dimension_size)
        if len(dim_intervals.intervals) <= 1:
            return dim_intervals
        modified_intervals = [dim_intervals.intervals[0]] + [
            replace(interval, start=interval.start - 1, left_ramp=interval.left_ramp + 1)
            for interval in dim_intervals.intervals[1:]
        ]
        return DimensionIntervals(intervals=modified_intervals)

    return split


def split_by_count_temporal_causal(
    num_tiles: int, overlap: int = 0, min_tile_size: int | None = None
) -> SplitOperation:
    non_causal_split = split_by_count(num_tiles, overlap, min_tile_size=min_tile_size)

    def split(dimension_size: int) -> DimensionIntervals:
        dim_intervals = non_causal_split(dimension_size)
        if len(dim_intervals.intervals) <= 1:
            return dim_intervals
        modified_intervals = [dim_intervals.intervals[0]] + [
            replace(interval, start=interval.start - 1, left_ramp=interval.left_ramp + 1)
            for interval in dim_intervals.intervals[1:]
        ]
        return DimensionIntervals(intervals=modified_intervals)

    return split


def split_at_seams(boundaries: Sequence[int], num_tiles: int, overlap: int = 0) -> SplitOperation:
    boundaries = tuple(boundaries)
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    if overlap < 0:
        raise ValueError(f"overlap must be >= 0, got {overlap}")
    if len(boundaries) < 2 or boundaries[0] != 0:
        raise ValueError(f"boundaries must start at 0 and hold at least one segment, got {list(boundaries)}")
    if any(b <= a for a, b in itertools.pairwise(boundaries)):
        raise ValueError(f"boundaries must be strictly increasing, got {list(boundaries)}")
    n_segments = len(boundaries) - 1
    n_tiles = min(num_tiles, n_segments)
    base, leftover = divmod(n_segments, n_tiles)
    counts = [base + (1 if index < leftover else 0) for index in range(n_tiles)]

    def split(dim_size: int) -> DimensionIntervals:
        if boundaries[-1] != dim_size - 1:
            raise ValueError(f"boundaries must end at the last cell ({dim_size - 1}), got {boundaries[-1]}")
        intervals: list[DimensionInterval] = []
        cursor = 0
        for tile_index, count in enumerate(counts):
            resume = boundaries[cursor] + 1
            start = 0 if tile_index == 0 else max(0, resume - overlap)
            cursor += count
            intervals.append(
                DimensionInterval(
                    start=start,
                    end=boundaries[cursor] + 1,
                    left_ramp=0 if tile_index == 0 else resume - start,
                    right_ramp=0,
                )
            )
        return DimensionIntervals(intervals=intervals)

    return split


def split_by_count(num_tiles: int, overlap: int = 0, min_tile_size: int | None = None) -> SplitOperation:
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    if overlap < 0:
        raise ValueError(f"overlap must be >= 0, got {overlap}")
    if min_tile_size is not None and min_tile_size < 1:
        raise ValueError(f"min_tile_size must be >= 1, got {min_tile_size}")

    def split(dim_size: int) -> DimensionIntervals:
        if num_tiles > dim_size:
            raise ValueError(
                f"num_tiles ({num_tiles}) exceeds dim_size ({dim_size}). Cannot assign at least 1 unit per tile."
            )
        if num_tiles == 1:
            return DEFAULT_SPLIT_OPERATION(dim_size)

        total = dim_size + overlap * (num_tiles - 1)
        tile_size = total // num_tiles
        if tile_size <= overlap:
            raise ValueError(
                f"split_by_count produced size={tile_size} <= overlap={overlap} "
                f"for dim_size={dim_size}, num_tiles={num_tiles}"
            )
        remainder = total % num_tiles

        base_intervals = split_by_size(tile_size, overlap)(dim_size - remainder).intervals

        # First `remainder` tiles each absorb 1 extra unit; shift subsequent boundaries.
        intervals: list[DimensionInterval] = []
        for i, iv in enumerate(base_intervals):
            shift = min(i, remainder)
            grow = 1 if i < remainder else 0
            intervals.append(replace(iv, start=iv.start + shift, end=iv.end + shift + grow))

        if min_tile_size is not None:
            intervals = _grow_last_tile_to_min(intervals, min_tile_size)
            _validate_tile_intervals(intervals, dim_size=dim_size, min_tile_size=min_tile_size)

        return DimensionIntervals(intervals=intervals)

    return split


def identity_mapping_operation(
    intervals: DimensionIntervals,
    *,
    rectangular: bool = False,
) -> tuple[list[slice], list[torch.Tensor]]:
    mask_1d = compute_rectangular_mask_1d if rectangular else compute_trapezoidal_mask_1d
    out_slices: list[slice] = []
    masks: list[torch.Tensor] = []
    for iv in intervals.intervals:
        out_slices.append(slice(iv.start, iv.end))
        masks.append(mask_1d(iv.end - iv.start, iv.left_ramp, iv.right_ramp))
    return out_slices, masks


class Tile(NamedTuple):
    in_coords: tuple[slice, ...]
    out_coords: tuple[slice, ...]
    masks_1d: tuple[torch.Tensor, ...]

    @property
    def blend_mask(self) -> torch.Tensor:
        num_dims = len(self.out_coords)
        per_dimension_masks: list[torch.Tensor] = []

        for dim_idx in range(num_dims):
            mask_1d = self.masks_1d[dim_idx]
            view_shape = [1] * num_dims
            # Reshape (L,) -> (1, ..., L, ..., 1) so masks across dimensions broadcast-multiply.
            view_shape[dim_idx] = mask_1d.shape[0]
            per_dimension_masks.append(mask_1d.view(*view_shape))

        # Multiply per-dimension masks to form the full N-D mask (separable blending window).
        combined_mask = per_dimension_masks[0]
        for mask in per_dimension_masks[1:]:
            combined_mask = combined_mask * mask

        return combined_mask


def scale_by_masks_1d(x: torch.Tensor, masks_1d: Sequence[torch.Tensor]) -> torch.Tensor:
    if len(masks_1d) != x.ndim:
        raise ValueError(f"masks_1d length {len(masks_1d)} != x.ndim {x.ndim}")
    out = x
    for axis, mask in enumerate(masks_1d):
        view_shape = [1] * x.ndim
        view_shape[axis] = -1
        out = out * mask.reshape(*view_shape)
    return out


def masks_are_complementary(
    tiles: Sequence[Tile],
    full_shape: Sequence[int],
    *,
    atol: float = 1e-5,
) -> bool:
    if not tiles:
        return True
    ndim = len(full_shape)
    for tile in tiles:
        if len(tile.out_coords) != ndim or len(tile.masks_1d) != ndim:
            raise ValueError(
                f"Tile out_coords/masks_1d rank {len(tile.out_coords)}/{len(tile.masks_1d)} != full_shape rank {ndim}"
            )
    for axis, length in enumerate(full_shape):
        # Explicit CPU float32: masks may live on CUDA; a non-CPU default device
        # must not place ``acc`` on GPU (device-mismatch on ``acc[sl] +=``).
        acc = torch.zeros(length, dtype=torch.float32, device="cpu")
        seen: set[tuple[int | None, int | None]] = set()
        for tile in tiles:
            sl = tile.out_coords[axis]
            key = (sl.start, sl.stop)
            if key in seen:
                continue
            seen.add(key)
            # Length-1 untiled masks broadcast over ``acc[sl]``.
            acc[sl] += tile.masks_1d[axis].detach().float().cpu()
        if not torch.allclose(acc, torch.ones(length, dtype=torch.float32), atol=atol, rtol=0.0):
            return False
    return True


def create_tiles_from_intervals_and_mappers(
    intervals: LatentIntervals,
    mappers: list[MappingOperation],
) -> list[Tile]:
    full_dim_input_slices: list[list[slice]] = []
    full_dim_output_slices: list[list[slice]] = []
    full_dim_masks_1d: list[list[torch.Tensor]] = []
    for axis_index in range(len(intervals.original_shape)):
        dimension_intervals = intervals.dimension_intervals[axis_index]
        input_slices = [slice(interval.start, interval.end) for interval in dimension_intervals.intervals]
        output_slices, masks_1d = mappers[axis_index](dimension_intervals)
        n_intervals = len(input_slices)
        if len(output_slices) != n_intervals or len(masks_1d) != n_intervals:
            raise ValueError(
                f"Axis {axis_index}: mapper produced {len(output_slices)} output slices and "
                f"{len(masks_1d)} masks for {n_intervals} input intervals"
            )
        full_dim_input_slices.append(input_slices)
        full_dim_output_slices.append(output_slices)
        full_dim_masks_1d.append(masks_1d)

    return [
        Tile(in_coords=in_coord, out_coords=out_coord, masks_1d=mask_1d)
        for in_coord, out_coord, mask_1d in zip(
            itertools.product(*full_dim_input_slices),
            itertools.product(*full_dim_output_slices),
            itertools.product(*full_dim_masks_1d),
            strict=True,
        )
    ]


def create_tiles(
    latent_shape: torch.Size,
    splitters: list[SplitOperation],
    mappers: list[MappingOperation],
) -> list[Tile]:
    if len(splitters) != len(latent_shape):
        raise ValueError(
            f"Number of splitters must be equal to number of dimensions in latent shape, "
            f"got {len(splitters)} and {len(latent_shape)}"
        )
    if len(mappers) != len(latent_shape):
        raise ValueError(
            f"Number of mappers must be equal to number of dimensions in latent shape, "
            f"got {len(mappers)} and {len(latent_shape)}"
        )
    intervals = [splitter(length) for splitter, length in zip(splitters, latent_shape, strict=True)]
    latent_intervals = LatentIntervals(original_shape=latent_shape, dimension_intervals=tuple(intervals))
    return create_tiles_from_intervals_and_mappers(latent_intervals, mappers)


def group_tiles_by_temporal_slice(tiles: list[Tile]) -> list[list[Tile]]:
    if not tiles:
        return []

    groups = []
    current_slice = tiles[0].out_coords[2]
    current_group = []

    for tile in tiles:
        tile_slice = tile.out_coords[2]
        if tile_slice == current_slice:
            current_group.append(tile)
        else:
            groups.append(current_group)
            current_slice = tile_slice
            current_group = [tile]

    if current_group:
        groups.append(current_group)

    return groups


@dataclass(frozen=True)
class DimensionTilingConfig:
    num_tiles: int = 1
    overlap: int = 0

    def __post_init__(self) -> None:
        if self.num_tiles < 1:
            raise ValueError(f"num_tiles must be >= 1, got {self.num_tiles}")
        if self.overlap < 0:
            raise ValueError(f"overlap must be >= 0, got {self.overlap}")

    def is_tiled(self) -> bool:
        return self.num_tiles > 1 or self.overlap > 0

@dataclass(frozen=True)
class DimensionSizeConfig:
    tile_size: int = 0
    overlap: int = 0

    def __post_init__(self) -> None:
        if self.tile_size < 0:
            raise ValueError(f"tile_size must be >= 0, got {self.tile_size}")
        if self.overlap < 0:
            raise ValueError(f"overlap must be >= 0, got {self.overlap}")
        if self.tile_size == 0:
            if self.overlap != 0:
                raise ValueError("untiled axis (tile_size=0) must have overlap=0")
            return
        if self.overlap >= self.tile_size:
            raise ValueError(f"Overlap must be less than tile size, got {self.overlap} and {self.tile_size}")

    def is_tiled(self) -> bool:
        return self.tile_size > 0


@dataclass(frozen=True)
class TileCountConfig:
    frames: DimensionTilingConfig = DimensionTilingConfig()
    height: DimensionTilingConfig = DimensionTilingConfig()
    width: DimensionTilingConfig = DimensionTilingConfig()

    def validate(self, scale_factors: SpatioTemporalScaleFactors, video_shape: VideoPixelShape) -> None:
        check_temporal = _assert_video_on_vae_grid(scale_factors, video_shape)
        latent_h = video_shape.height // scale_factors.height
        latent_w = video_shape.width // scale_factors.width
        _validate_count_axis(self.height, latent_h, "height")
        _validate_count_axis(self.width, latent_w, "width")
        if check_temporal:
            latent_f = (video_shape.frames - 1) // scale_factors.time + 1
            _validate_count_axis(self.frames, latent_f, "frames")

    def to_splitters(
        self,
        scale_factors: SpatioTemporalScaleFactors,
        min_tile_size: tuple[int, int, int] | None = None,
        *,
        causal_temporal: bool = True,
    ) -> tuple[SplitOperation, SplitOperation, SplitOperation]:
        del scale_factors
        min_t = min_h = min_w = None
        if min_tile_size is not None:
            min_t, min_h, min_w = min_tile_size

        def axis_split(cfg: DimensionTilingConfig, axis_min: int | None, *, temporal: bool) -> SplitOperation:
            if not cfg.is_tiled():
                return DEFAULT_SPLIT_OPERATION
            if temporal and causal_temporal:
                return split_by_count_temporal_causal(cfg.num_tiles, cfg.overlap, min_tile_size=axis_min)
            return split_by_count(cfg.num_tiles, cfg.overlap, min_tile_size=axis_min)

        return (
            axis_split(self.frames, min_t, temporal=True),
            axis_split(self.height, min_h, temporal=False),
            axis_split(self.width, min_w, temporal=False),
        )

@dataclass(frozen=True)
class TileSizeConfig:
    frames: DimensionSizeConfig = DimensionSizeConfig()
    height: DimensionSizeConfig = DimensionSizeConfig()
    width: DimensionSizeConfig = DimensionSizeConfig()

    def validate(self, scale_factors: SpatioTemporalScaleFactors, video_shape: VideoPixelShape) -> None:
        check_temporal = _assert_video_on_vae_grid(scale_factors, video_shape)
        _validate_size_axis(self.height, scale_factors.height, "height")
        _validate_size_axis(self.width, scale_factors.width, "width")
        if check_temporal:
            _validate_size_axis(self.frames, scale_factors.time, "frames")

    @classmethod
    def default(cls) -> TileSizeConfig:
        return cls(
            frames=DimensionSizeConfig(tile_size=80, overlap=24),
            height=DimensionSizeConfig(tile_size=768, overlap=64),
            width=DimensionSizeConfig(tile_size=768, overlap=64),
        )

    def to_splitters(
        self,
        scale_factors: SpatioTemporalScaleFactors,
        min_tile_size: tuple[int, int, int] | None = None,
        *,
        causal_temporal: bool = True,
    ) -> tuple[SplitOperation, SplitOperation, SplitOperation]:
        min_t = min_h = min_w = None
        if min_tile_size is not None:
            min_t, min_h, min_w = min_tile_size

        def enable_size_axis(
            factor: int,
            axis_min: int | None,
            cfg: DimensionSizeConfig,
            axis_name: str,
            *,
            temporal: bool,
        ) -> SplitOperation:
            if not cfg.is_tiled():
                return DEFAULT_SPLIT_OPERATION
            _validate_size_axis(cfg, factor, axis_name)
            size = cfg.tile_size // factor
            overlap = cfg.overlap // factor
            lower_threshold = max(2, overlap + 1)
            tile = max(lower_threshold, size)
            if temporal and causal_temporal:
                return split_temporal_causal(tile, overlap, min_tile_size=axis_min)
            return split_by_size(tile, overlap, min_tile_size=axis_min)

        return (
            enable_size_axis(scale_factors.time, min_t, self.frames, "frames", temporal=True),
            enable_size_axis(scale_factors.height, min_h, self.height, "height", temporal=False),
            enable_size_axis(scale_factors.width, min_w, self.width, "width", temporal=False),
        )

TilingConfig = TileSizeConfig | TileCountConfig




def _assert_video_on_vae_grid(
    scale_factors: SpatioTemporalScaleFactors,
    video_shape: VideoPixelShape,
) -> bool:
    if scale_factors.time < 1 or scale_factors.height < 1 or scale_factors.width < 1:
        raise ValueError(f"scale_factors must be >= 1 on each axis, got {scale_factors}")
    if video_shape.height < 1 or video_shape.width < 1:
        raise ValueError(f"video_shape height/width must be >= 1, got {video_shape.height}x{video_shape.width}")
    if video_shape.height % scale_factors.height != 0:
        raise ValueError(f"video height {video_shape.height} must be divisible by scale {scale_factors.height}")
    if video_shape.width % scale_factors.width != 0:
        raise ValueError(f"video width {video_shape.width} must be divisible by scale {scale_factors.width}")
    if video_shape.frames <= 0:
        return False
    if (video_shape.frames - 1) % scale_factors.time != 0:
        raise ValueError(f"video frames {video_shape.frames} must satisfy (frames - 1) % {scale_factors.time} == 0")
    return True


def _validate_size_axis(cfg: DimensionSizeConfig, factor: int, axis_name: str) -> None:
    if not cfg.is_tiled():
        return
    min_size = 2 * factor
    if cfg.tile_size < min_size:
        raise ValueError(f"{axis_name}.tile_size must be at least {min_size}, got {cfg.tile_size}")
    if cfg.tile_size % factor != 0:
        raise ValueError(f"{axis_name}.tile_size must be divisible by {factor}, got {cfg.tile_size}")
    if cfg.overlap % factor != 0:
        raise ValueError(f"{axis_name}.overlap must be divisible by {factor}, got {cfg.overlap}")


def _validate_count_axis(cfg: DimensionTilingConfig, latent_extent: int, axis_name: str) -> None:
    if not cfg.is_tiled():
        return
    if cfg.num_tiles > latent_extent:
        raise ValueError(f"{axis_name}.num_tiles {cfg.num_tiles} exceeds latent {axis_name} extent {latent_extent}")
    # split_by_count requires overlap < tile_size; tile_size grows with extent/n.
    max_overlap = latent_extent - cfg.num_tiles
    if cfg.overlap > max_overlap:
        raise ValueError(
            f"{axis_name}.overlap {cfg.overlap} exceeds latent bound {max_overlap} "
            f"for extent {latent_extent} with {cfg.num_tiles} tiles"
        )


def _validate_overlap(
    tiling_config: TilingConfig,
    *,
    min_overlap_frames: int,
    min_overlap_pixels: int,
) -> None:
    if not isinstance(tiling_config, TileSizeConfig):
        return

    for axis_name, cfg, recommended, unit in (
        ("frames", tiling_config.frames, min_overlap_frames, "frames"),
        ("height", tiling_config.height, min_overlap_pixels, "px"),
        ("width", tiling_config.width, min_overlap_pixels, "px"),
    ):
        if cfg.is_tiled() and cfg.overlap < recommended:
            raise ValueError(f"{axis_name} overlap {cfg.overlap} {unit} is below the required {recommended} {unit}.")


class DiffVAEMode(Enum):
    COMBINED_COMPILE = "combined_compile"
    CHUNKED_COMPILE = "chunked_compile"
    CHUNKED_EAGER = "chunked_eager"
    BLACKWELL_DSL = "blackwell_dsl"

    def resolve(self):
        return self


class NAttentionKind(Enum):
    TRITON = "triton"
    EAGER_SDPA = "eager_sdpa"


@dataclass(frozen=True)
class _ResolvedAttention:
    attention: NAttentionKind = NAttentionKind.EAGER_SDPA
    compile_blocks: bool = False


def resolve_attention_for_host(mode):
    del mode
    return _ResolvedAttention()


def frames_per_yuv_gemm(height: int, width: int) -> int:
    del height, width
    return 2**31 - 1


def patchify(x: torch.Tensor, patch_size_hw: int, patch_size_t: int = 1) -> torch.Tensor:
    if patch_size_hw == 1 and patch_size_t == 1:
        return x
    if x.dim() == 4:
        x = rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size_hw, r=patch_size_hw)
    elif x.dim() == 5:
        x = rearrange(
            x,
            "b c (f p) (h q) (w r) -> b (c p r q) f h w",
            p=patch_size_t,
            q=patch_size_hw,
            r=patch_size_hw,
        )
    else:
        raise ValueError(f"Invalid input shape: {x.shape}")

    return x


def unpatchify(x: torch.Tensor, patch_size_hw: int, patch_size_t: int = 1) -> torch.Tensor:
    if patch_size_hw == 1 and patch_size_t == 1:
        return x

    if x.dim() == 4:
        x = rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size_hw, r=patch_size_hw)
    elif x.dim() == 5:
        x = rearrange(
            x,
            "b (c p r q) f h w -> b c (f p) (h q) (w r)",
            p=patch_size_t,
            q=patch_size_hw,
            r=patch_size_hw,
        )

    return x


class PerChannelStatistics(nn.Module):
    def __init__(self, latent_channels: int = 128):
        super().__init__()
        # Frozen parameters, not buffers: disk-offload reload restores only named_parameters(),
        # so persistent buffers would be skipped and strict load_state_dict would report them missing.
        self.register_parameter("std-of-means", nn.Parameter(torch.ones(latent_channels), requires_grad=False))
        self.register_parameter("mean-of-means", nn.Parameter(torch.zeros(latent_channels), requires_grad=False))

    def un_normalize(self, x: torch.Tensor) -> torch.Tensor:
        return (x * getattr(self, "std-of-means").view(1, -1, 1, 1, 1).to(x)) + getattr(self, "mean-of-means").view(
            1, -1, 1, 1, 1
        ).to(x)

    def normalize(self, x: torch.Tensor) -> torch.Tensor:
        return (x - getattr(self, "mean-of-means").view(1, -1, 1, 1, 1).to(x)) / getattr(self, "std-of-means").view(
            1, -1, 1, 1, 1
        ).to(x)


"""Keyframe (dual-stream) inputs and coordinate math for DiffVAE decode.
A keyframe-aware decode carries two streams through the decoder: the video volume
``(B, T, H, W, C)`` and a stack of keyframe *planes* ``(B, P, H, W, C)`` whose plane
axis occupies video's temporal slot. Weights are fully shared; the streams only ever
mix inside one joint attention softmax (see ``transformer/fallback_na/joint_eager.py``).
Everything here is pure coordinate/geometry math with no module state, so the eager and
triton backends can share it and therefore agree exactly on slot selection.
Deviation from upstream worth knowing: upstream carries per-sample keyframe times and
masks (``(B, n_kf)``). Here they are batch-shared 1-D ``(P,)`` tensors, because our
decode path is single-sample and ``rope_math.rot_abs_axis_impl`` takes a 1-D position
vector per axis. That keeps the RoPE call and the slot tables batch-independent.
"""


KEYFRAME_CONTEXT_SLOTS = 2


@dataclass(frozen=True)
class DecodeKeyframes:
    latents: torch.Tensor
    pixel_frame_indices: torch.Tensor
    clip_start_frame: int = 0

    def validate(self, *, num_frames: int | None = None) -> None:
        if self.latents.ndim != 5:
            raise ValueError(f"keyframe latents must be (B, C, P, H, W), got {tuple(self.latents.shape)}")
        if self.pixel_frame_indices.ndim != 1:
            raise ValueError(f"pixel_frame_indices must be 1-D (P,), got {tuple(self.pixel_frame_indices.shape)}")
        if self.clip_start_frame < 0:
            raise ValueError(f"clip_start_frame must be non-negative, got {self.clip_start_frame}")
        planes = self.latents.shape[2]
        if planes != self.pixel_frame_indices.shape[0]:
            raise ValueError(
                f"keyframe plane count {planes} != len(pixel_frame_indices) {self.pixel_frame_indices.shape[0]}"
            )
        if planes == 0:
            # An empty stack is a plain decode wearing a keyframe decode's costs, and every
            # backend has to special-case it (the slot tables are all -1, and gathering plane 0
            # of an empty axis is an out-of-bounds read). Say so here instead.
            raise ValueError("keyframe decode needs at least one plane; use decode_video() for a plain decode")
        if planes and int(self.pixel_frame_indices.min()) < 0:
            raise ValueError("pixel_frame_indices must be non-negative (global pixel frames)")
        if num_frames is not None and num_frames < 1:
            raise ValueError(f"num_frames must be positive, got {num_frames}")
        # Planes may sit outside [clip_start_frame, clip_start_frame + num_frames): Dist tiles
        # keep the nearest plane on each side so |dt| matches a whole-clip decode. A far plane
        # on a full clip is the same geometry -- joint attention ranks it by distance.

    def crop_spatial(self, height: slice, width: slice) -> "DecodeKeyframes":
        return DecodeKeyframes(
            latents=self.latents[:, :, :, height, width],
            pixel_frame_indices=self.pixel_frame_indices,
            clip_start_frame=self.clip_start_frame,
        )

    @property
    def num_planes(self) -> int:
        return int(self.latents.shape[2])


@dataclass(frozen=True)
class KeyframeStream:
    x: torch.Tensor
    times: torch.Tensor
    valid: torch.Tensor

    def masked(self) -> KeyframeStream:
        return KeyframeStream(x=self.x * self.valid[None, :, None, None, None], times=self.times, valid=self.valid)

    def select_planes(self, keep: torch.Tensor) -> KeyframeStream:
        if keep.shape != (self.num_planes,):
            raise ValueError(f"keep must be ({self.num_planes},) bool, got {tuple(keep.shape)}")
        return KeyframeStream(x=self.x[:, keep], times=self.times[keep], valid=self.valid[keep])

    def crop_spatial(self, height: slice, width: slice) -> KeyframeStream:
        return KeyframeStream(x=self.x[:, :, height, width, :], times=self.times, valid=self.valid)

    @property
    def num_planes(self) -> int:
        return int(self.x.shape[1])


def keyframe_stage_times(pixel_frame_indices: torch.Tensor, remaining_time_stride: int) -> torch.Tensor:
    if remaining_time_stride < 1:
        raise ValueError(f"remaining_time_stride must be positive, got {remaining_time_stride}")
    frames = pixel_frame_indices.to(torch.float32)
    center_offset = (remaining_time_stride - 1) / 2
    times = (frames + center_offset) / remaining_time_stride
    return torch.where(frames == 0, torch.zeros_like(times), times)


def keyframe_clip_times(
    pixel_frame_indices: torch.Tensor,
    remaining_time_stride: int,
    clip_start_frame: int,
    extra_origin: float = 0.0,
) -> torch.Tensor:
    times = keyframe_stage_times(pixel_frame_indices, remaining_time_stride)
    origin = keyframe_stage_times(
        torch.as_tensor([clip_start_frame], dtype=torch.int64, device=pixel_frame_indices.device),
        remaining_time_stride,
    )
    return times - origin - extra_origin


def planes_for_tile(
    pixel_frame_indices: torch.Tensor,
    frame_lo: int,
    frame_hi: int,
    *,
    clip_start_frame: int = 0,
) -> torch.Tensor:
    frame_lo = frame_lo + clip_start_frame
    frame_hi = frame_hi + clip_start_frame
    indices = pixel_frame_indices.to(torch.int64)
    keep = (indices >= frame_lo) & (indices <= frame_hi)
    before = indices < frame_lo
    if bool(before.any()):
        # Latest plane strictly before the tile.
        keep[int(torch.where(before, indices, torch.full_like(indices, -1)).argmax())] = True
    after = indices > frame_hi
    if bool(after.any()):
        # Earliest plane strictly after the tile.
        sentinel = int(indices.max()) + 1
        keep[int(torch.where(after, indices, torch.full_like(indices, sentinel)).argmin())] = True
    return keep


def remaining_time_strides(upsamples: Sequence[torch.nn.Module]) -> tuple[int, ...]:
    strides = [int(up.stride[0]) for up in upsamples]
    remaining: list[int] = []
    for index in range(len(strides)):
        product = 1
        for stride in strides[index:]:
            product *= stride
        remaining.append(product)
    remaining.append(1)
    return tuple(remaining)


def upsample_keyframe_planes(upsample: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:
    planes = x.shape[1]
    flat = rearrange(x, "b p h w c -> (b p) 1 h w c")
    upsampled = upsample(flat, drop_leading_frame=True)
    if upsampled.shape[1] != 1:
        raise RuntimeError(f"isolated keyframe upsampling must preserve one temporal plane, got T={upsampled.shape[1]}")
    out = rearrange(upsampled[:, 0], "(b p) h w c -> b p h w c", p=planes)
    if out.shape[1] != planes:
        raise RuntimeError(f"keyframe plane count changed under upsample: {planes} -> {out.shape[1]}")
    return out


def _nearest_slots(
    query_times: torch.Tensor,
    candidate_times: torch.Tensor,
    candidate_valid: torch.Tensor | None,
    num_slots: int,
) -> torch.Tensor:
    distances = (query_times[:, None] - candidate_times[None, :]).abs().to(torch.float32)
    if candidate_valid is not None:
        distances = distances.masked_fill(~candidate_valid[None, :], float("inf"))
    order = torch.argsort(distances, dim=-1, stable=True)
    take = min(num_slots, candidate_times.shape[0])
    chosen = order[:, :take]
    # Drop slots that only exist because every remaining candidate was invalid.
    finite = torch.gather(distances, 1, chosen).isfinite()
    chosen = torch.where(finite, chosen, torch.full_like(chosen, -1))
    if take < num_slots:
        pad = torch.full((chosen.shape[0], num_slots - take), -1, dtype=chosen.dtype, device=chosen.device)
        chosen = torch.cat([chosen, pad], dim=1)
    return chosen


def video_keyframe_slots(
    keyframe_times: torch.Tensor,
    keyframe_valid: torch.Tensor,
    video_length: int,
    num_slots: int = KEYFRAME_CONTEXT_SLOTS,
) -> torch.Tensor:
    query = torch.arange(video_length, dtype=torch.float32, device=keyframe_times.device)
    return _nearest_slots(query, keyframe_times.to(torch.float32), keyframe_valid, num_slots)


def keyframe_video_slots(
    keyframe_times: torch.Tensor,
    keyframe_valid: torch.Tensor,
    video_length: int,
    num_slots: int = KEYFRAME_CONTEXT_SLOTS,
) -> torch.Tensor:
    candidates = torch.arange(video_length, dtype=torch.float32, device=keyframe_times.device)
    slots = _nearest_slots(keyframe_times.to(torch.float32), candidates, None, num_slots)
    return torch.where(keyframe_valid[:, None], slots, torch.full_like(slots, -1))


"""DiffVAE tiling helpers: schedule, pad/crop/size-floor, blend utilities.
Decode orchestration lives on ``DiffusionVideoDecoder``. This module owns the
geometry/schedule/mask pieces that tiling uses.
"""


ResizeAxisMode = Literal["repeat_last", "symmetric"]


_GIB: int = 1 << 30


@dataclass(frozen=True, slots=True)
class _StageFiveBudget:
    coef: float
    coef_keyframes: float
    reserve_bytes: int


_BUDGET_BY_MODE: dict[DiffVAEMode, _StageFiveBudget] = {
    DiffVAEMode.COMBINED_COMPILE: _StageFiveBudget(coef=11, coef_keyframes=15, reserve_bytes=2 * _GIB),
    DiffVAEMode.CHUNKED_COMPILE: _StageFiveBudget(coef=7, coef_keyframes=5, reserve_bytes=2 * _GIB),
    DiffVAEMode.CHUNKED_EAGER: _StageFiveBudget(coef=5, coef_keyframes=5, reserve_bytes=1 * _GIB),
    DiffVAEMode.BLACKWELL_DSL: _StageFiveBudget(coef=2.5, coef_keyframes=2.5, reserve_bytes=2 * _GIB),
}


_DEFAULT_ELEMENT_SIZE: int = 2  # bf16 features → fp16 accumulator / bf16 stage-5


_ACCUMULATOR_CHANNELS: int = 3  # RGB pixel blend buffer (decoder out_channels)


_MIN_MODEL_BYTES_FLOOR: int = 1 << 30  # never assume a free DiffVAE weight footprint


_BUDGET_SAFETY_BYTES_EAGER: int = 1 * _GIB


_BUDGET_SAFETY_BYTES_JOINT_MATERIALIZED: int = 2 * _GIB


def _falls_back_to_eager_na(mode: DiffVAEMode) -> bool:
    resolved = resolve_attention_for_host(mode.resolve())
    return resolved.attention in (NAttentionKind.TRITON, NAttentionKind.EAGER_SDPA) and not resolved.compile_blocks


def stage5_mem_coef(mode: DiffVAEMode, *, keyframes: bool = False) -> float:
    try:
        budget = _BUDGET_BY_MODE[mode]
    except KeyError as exc:
        raise ValueError(f"Unsupported DiffVAEMode for tiling budget: {mode!r}") from exc
    if keyframes:
        return budget.coef_keyframes
    if _falls_back_to_eager_na(mode):
        return _BUDGET_BY_MODE[DiffVAEMode.CHUNKED_EAGER].coef
    return budget.coef


_CONVERT_PEAK_UV_CHANNELS_X2 = 5  # uv_full (2) + pooled uv (0.5)


_PACK_PEAK_UV_CHANNELS_X2 = 4  # pooled uv (0.5) + packed float (1.5)


_PACK_PEAK_UINT8_BYTES_X2 = 3  # 1.5 B/px


def max_emitted_frames(*, num_frames: int, tile_frames: int, overlap_frames: int) -> int:
    if tile_frames >= num_frames:
        return num_frames
    stride = tile_frames - overlap_frames
    if stride <= 0:
        return tile_frames
    n_tiles = 1 + -(-(num_frames - tile_frames) // stride)
    last_group = num_frames - (n_tiles - 1) * stride
    return min(tile_frames, max(stride, last_group + 1))


def emit_convert_bytes(
    *,
    tile_frames: int,
    height: int,
    width: int,
    out_channels: int,
    element_size: int,
) -> int:
    frames = int(tile_frames)
    gemm_frames = min(frames, frames_per_yuv_gemm(height, width))
    # Write-back reuses the RGB storage, so no full YUV tensor survives into the pack.
    resident_yuv_frames = frames if gemm_frames == frames else 0
    yuv_channels_x2 = 2 * int(out_channels)

    convert_peak_x2 = element_size * (yuv_channels_x2 * gemm_frames + _CONVERT_PEAK_UV_CHANNELS_X2 * frames)
    pack_peak_x2 = (
        element_size * (yuv_channels_x2 * resident_yuv_frames + _PACK_PEAK_UV_CHANNELS_X2 * frames)
        + _PACK_PEAK_UINT8_BYTES_X2 * frames
    )

    return int(height) * int(width) * max(convert_peak_x2, pack_peak_x2) // 2


def budget_safety_bytes(
    mode: DiffVAEMode,
    *,
    keyframes: bool = False,
    joint_sdpa_materializes: bool = False,
) -> int:
    try:
        budget = _BUDGET_BY_MODE[mode]
    except KeyError as exc:
        raise ValueError(f"Unsupported DiffVAEMode for tiling budget: {mode!r}") from exc
    if keyframes and mode is not DiffVAEMode.BLACKWELL_DSL:
        return _BUDGET_SAFETY_BYTES_JOINT_MATERIALIZED if joint_sdpa_materializes else _BUDGET_SAFETY_BYTES_EAGER
    if _falls_back_to_eager_na(mode):
        return _BUDGET_SAFETY_BYTES_EAGER
    return budget.reserve_bytes


def accumulator_element_size(feature_dtype: torch.dtype) -> int:
    if feature_dtype is torch.bfloat16:
        return 2  # stored as fp16
    return int(torch.tensor([], dtype=feature_dtype).element_size())


def stage4_feature_bytes(
    *,
    height: int,
    width: int,
    num_frames: int,
    upsample_strides: Sequence[Tuple[int, int, int]],
    stage4_channels: int,
    element_size: int = _DEFAULT_ELEMENT_SIZE,
    natten_trailing_pad_latent_frames: int = 0,
) -> int:
    if stage4_channels < 1:
        raise ValueError(f"stage4_channels must be >= 1, got {stage4_channels}")
    if element_size < 1:
        raise ValueError(f"element_size must be >= 1, got {element_size}")
    if len(upsample_strides) < 3:
        raise ValueError(f"need at least 3 upsample strides, got {len(upsample_strides)}")
    if natten_trailing_pad_latent_frames < 0:
        raise ValueError(f"natten_trailing_pad_latent_frames must be >= 0, got {natten_trailing_pad_latent_frames}")

    # Local import: types ↔ tiling cycle avoidance at module import time.
    from ltx_core.types import VIDEO_SCALE_FACTORS, VideoLatentShape, VideoPixelShape  # noqa: PLC0415

    latent = VideoLatentShape.from_pixel_shape(
        VideoPixelShape(batch=1, frames=int(num_frames), height=int(height), width=int(width), fps=24.0),
        scale_factors=VIDEO_SCALE_FACTORS,
    )
    s4_t, s4_h, s4_w = stage4_thw_from_latent(
        upsample_strides[:3],
        latent.frames + int(natten_trailing_pad_latent_frames),
        latent.height,
        latent.width,
        drop_leading_frame=True,
    )
    return int(s4_t) * int(s4_h) * int(s4_w) * int(stage4_channels) * int(element_size)


def recommended_decode_tiling_config(  # noqa: PLR0913
    *,
    tile_halos: Tuple[Tuple[int, int, int], Tuple[int, int, int]],
    pixel_scale: SpatioTemporalScaleFactors,
    min_tile_size_s4: Tuple[int, int, int],
    patch_size: int,
    height: int,
    width: int,
    num_frames: int,
    mode: DiffVAEMode,
    free_bytes: int,
    stage5_channels: int,
    stage4_channels: int,
    upsample_strides: Sequence[Tuple[int, int, int]],
    model_bytes: int = 0,
    element_size: int = _DEFAULT_ELEMENT_SIZE,
    natten_trailing_pad_latent_frames: int = 0,
    out_channels: int = _ACCUMULATOR_CHANNELS,
    keyframes: bool = False,
    joint_sdpa_materializes: bool = False,
) -> TileSizeConfig:
    if height < 1 or width < 1 or num_frames < 1:
        raise ValueError(f"height/width/num_frames must be >= 1, got {height}x{width}x{num_frames}")
    if patch_size < 1:
        raise ValueError(f"patch_size must be >= 1, got {patch_size}")
    if stage5_channels < 1:
        raise ValueError(f"stage5_channels must be >= 1, got {stage5_channels}")
    if out_channels < 1:
        raise ValueError(f"out_channels must be >= 1, got {out_channels}")
    if element_size < 1:
        raise ValueError(f"element_size must be >= 1, got {element_size}")

    overlap_t, overlap_hw = recommended_pixel_overlaps(tile_halos, pixel_scale)

    ft, fh, fw = pixel_scale.time, pixel_scale.height, pixel_scale.width
    # Construction validates fixed 8/32/32; to_splitters uses pixel_scale - step both.
    step_t = math.lcm(ft, VIDEO_SCALE_FACTORS.time)
    step_h = math.lcm(fh, VIDEO_SCALE_FACTORS.height)
    step_w = math.lcm(fw, VIDEO_SCALE_FACTORS.width)
    min_t_px = _round_up(
        # ``2 * overlap`` so left+right ramps fit (else masks are not complementary and
        # decode allocates a full weights buffer ≈ another accumulator).
        max(2 * ft, 2 * overlap_t, _round_up(min_tile_size_s4[0] * ft, ft), 16),
        step_t,
    )
    min_h_px = _round_up(
        max(2 * fh, 2 * overlap_hw, _round_up(min_tile_size_s4[1] * fh, fh), 64),
        step_h,
    )
    min_w_px = _round_up(
        max(2 * fw, 2 * overlap_hw, _round_up(min_tile_size_s4[2] * fw, fw), 64),
        step_w,
    )

    model_cost = max(int(model_bytes), _MIN_MODEL_BYTES_FLOOR)
    coef = stage5_mem_coef(mode, keyframes=keyframes)
    s4_feat_bytes = stage4_feature_bytes(
        height=height,
        width=width,
        num_frames=num_frames,
        upsample_strides=upsample_strides,
        stage4_channels=stage4_channels,
        element_size=element_size,
        natten_trailing_pad_latent_frames=natten_trailing_pad_latent_frames,
    )
    reserve = budget_safety_bytes(mode, keyframes=keyframes, joint_sdpa_materializes=joint_sdpa_materializes)
    usable = max(0, int(free_bytes) - model_cost - reserve - s4_feat_bytes)
    s5_bytes_per_token = max(1.0, float(stage5_channels) * float(element_size) * coef)
    acc_bytes_per_pixel = int(out_channels) * int(element_size)

    t_cands = _axis_candidates(num_frames, overlap_t, min_t_px, step_t)
    h_cands = _axis_candidates(height, overlap_hw, min_h_px, step_h)
    w_cands = _axis_candidates(width, overlap_hw, min_w_px, step_w)

    scored: list[tuple[float, int, int, int, int, int]] = []
    # (waste, -volume, n_t*n_h*n_w, tile_t, tile_h, tile_w) - minimize waste, then launches.
    for tile_t, n_t in t_cands:
        # Current group buffer + still-live emit/stub during temporal handoff.
        acc_frames = 2 * int(tile_t)
        acc_bytes = acc_frames * int(height) * int(width) * acc_bytes_per_pixel
        # The consumer converts each yielded chunk while this decode is suspended.
        downstream_bytes = emit_convert_bytes(
            tile_frames=max_emitted_frames(num_frames=num_frames, tile_frames=tile_t, overlap_frames=overlap_t),
            height=height,
            width=width,
            out_channels=out_channels,
            element_size=element_size,
        )
        if acc_bytes + downstream_bytes >= usable:
            continue
        s5_budget_bytes = usable - acc_bytes - downstream_bytes
        max_s5_tokens = int(s5_budget_bytes // s5_bytes_per_token)
        for tile_h, n_h in h_cands:
            for tile_w, n_w in w_cands:
                if stage5_tokens_for_pixel_tile(tile_t, tile_h, tile_w, patch_size=patch_size) > max_s5_tokens:
                    continue
                waste = volumetric_overlap_waste(
                    num_frames=num_frames,
                    height=height,
                    width=width,
                    tile_frames=tile_t,
                    tile_height=tile_h,
                    tile_width=tile_w,
                    n_t=n_t,
                    n_h=n_h,
                    n_w=n_w,
                )
                scored.append((waste, -tile_t * tile_h * tile_w, n_t * n_h * n_w, tile_t, tile_h, tile_w))

    if not scored:
        raise ValueError(
            "Cannot fit a DiffVAE decode tile under the memory budget: "
            f"min tile ~{min_t_px}f x {min_h_px}x{min_w_px}px "
            f"(overlaps T={overlap_t}, HW={overlap_hw}), "
            f"mode={mode.value}, keyframes={keyframes}, coef={coef}, stage5_channels={stage5_channels}, "
            f"stage4_feature_bytes={s4_feat_bytes}, usable_bytes={usable}. "
            "Reduce resolution, reduce num_frames (stage-4 features and the per-chunk "
            "encode buffers both scale with it), or free GPU memory."
        )

    scored.sort()
    _waste, _vol, _ntiles, tile_t, tile_h, tile_w = scored[0]
    return TileSizeConfig(
        frames=DimensionSizeConfig(tile_size=tile_t, overlap=overlap_t),
        height=DimensionSizeConfig(tile_size=tile_h, overlap=overlap_hw),
        width=DimensionSizeConfig(tile_size=tile_w, overlap=overlap_hw),
    )


def prepare_tile_schedule(
    stage4_shape_bcthw: torch.Size,
    tiling_config: TilingConfig | None,
    *,
    upsample3_stride: Tuple[int, int, int],
    patch_size: int,
    min_tile_size: Tuple[int, int, int],
    tile_halos: Tuple[Tuple[int, int, int], Tuple[int, int, int]],
) -> List[Tile]:
    pixel_scale = stage4_to_pixel_scale_factors(upsample3_stride, patch_size)
    if tiling_config is None:
        return [
            Tile(
                in_coords=(slice(None), slice(None), slice(None), slice(None), slice(None)),
                out_coords=(slice(None), slice(None), slice(None), slice(None), slice(None)),
                masks_1d=(
                    untiled_mask_1d(),
                    untiled_mask_1d(),
                    untiled_mask_1d(),
                    untiled_mask_1d(),
                    untiled_mask_1d(),
                ),
            )
        ]

    overlap_t, overlap_hw = recommended_pixel_overlaps(tile_halos, pixel_scale)
    _validate_overlap(tiling_config, min_overlap_frames=overlap_t, min_overlap_pixels=overlap_hw)
    # Plain split (not split_temporal_causal): no start-1 / left_ramp+1 copycat of ConvVAE.
    t_split, h_split, w_split = tiling_config.to_splitters(
        pixel_scale, min_tile_size=min_tile_size, causal_temporal=False
    )
    st, sh, sw = upsample3_stride

    def axis_specs(
        split_op: SplitOperation,
        dim_len: int,
        stride_component: int,
        *,
        propagate_causal: bool,
        apply_patch: bool,
    ) -> list[tuple[slice, slice, torch.Tensor]]:
        if split_op is DEFAULT_SPLIT_OPERATION:
            return [(slice(None), slice(None), untiled_mask_1d())]
        intervals = split_op(dim_len).intervals
        specs = []
        for iv in intervals:
            stage5 = _propagate_interval_through_upsample_hops(iv, [stride_component], propagate_causal)
            if apply_patch:
                pixel = _propagate_interval_through_upsample_hops(stage5, [patch_size], causal=False)
            else:
                pixel = stage5
            # Symmetric ramps (left_starts_from_0=False) for partition-of-unity with
            # pixel-shuffle out_coords; ConvVAE sacrificial first-sample is not used.
            mask_pixel = compute_trapezoidal_mask_1d(
                pixel.end - pixel.start, pixel.left_ramp, pixel.right_ramp, left_starts_from_0=False
            )
            specs.append((slice(iv.start, iv.end), slice(pixel.start, pixel.end), mask_pixel))
        return specs

    # Temporal: pixel-shuffle propagate (drop-leading geometry); spatial: exact x stride.
    t_specs = axis_specs(t_split, stage4_shape_bcthw[2], st, propagate_causal=True, apply_patch=False)
    h_specs = axis_specs(h_split, stage4_shape_bcthw[3], sh, propagate_causal=False, apply_patch=True)
    w_specs = axis_specs(w_split, stage4_shape_bcthw[4], sw, propagate_causal=False, apply_patch=True)

    tiles: List[Tile] = []
    for t_spec, h_spec, w_spec in itertools.product(t_specs, h_specs, w_specs):
        t_s4, t_px, t_mask = t_spec
        h_s4, h_px, h_mask = h_spec
        w_s4, w_px, w_mask = w_spec
        tiles.append(
            Tile(
                in_coords=(slice(None), t_s4, h_s4, w_s4, slice(None)),
                out_coords=(slice(None), slice(None), t_px, h_px, w_px),
                masks_1d=(untiled_mask_1d(), untiled_mask_1d(), t_mask, h_mask, w_mask),
            )
        )
    return tiles


def slice_stage4_tile(
    feat_s4: torch.Tensor,
    tile: Tile,
    *,
    content_frames: int,
) -> tuple[torch.Tensor, bool, bool, tuple[int, int, int]]:
    is_origin = tile.in_coords[1].start in (0, None)
    _, stop, _ = tile.in_coords[1].indices(content_frames)
    pad_trailing = stop == content_frames
    _b, t_coord, h_coord, w_coord, _c = tile.in_coords
    t0, t1, _ = t_coord.indices(content_frames)
    h0, h1, _ = h_coord.indices(feat_s4.shape[2])
    w0, w1, _ = w_coord.indices(feat_s4.shape[3])
    content_thw = (t1 - t0, h1 - h0, w1 - w0)
    if pad_trailing:
        t1 = feat_s4.shape[1]
    feat_tile = feat_s4[:, t0:t1, h_coord, w_coord, :]
    return feat_tile, is_origin, pad_trailing, content_thw


@dataclass(frozen=True)
class AxisPad:
    before: int
    after: int


def resize_axis(
    x: torch.Tensor,
    dim: int,
    size: int,
    *,
    mode: ResizeAxisMode,
) -> tuple[torch.Tensor, AxisPad]:
    if size < 1:
        raise ValueError(f"resize_axis target size must be >= 1, got {size}")
    if dim < 0:
        dim += x.ndim
    if not 0 <= dim < x.ndim:
        raise ValueError(f"dim {dim} out of range for rank-{x.ndim} tensor")

    length = x.shape[dim]
    if length == size:
        return x, AxisPad(0, 0)

    if length < size:
        need = size - length
        if mode == "repeat_last":
            last = x.narrow(dim, length - 1, 1)
            expand_shape = list(x.shape)
            expand_shape[dim] = need
            pad = last.expand(expand_shape)
            return torch.cat([x, pad], dim=dim), AxisPad(0, need)

        before = need // 2
        after = need - before
        first = x.narrow(dim, 0, 1)
        last = x.narrow(dim, length - 1, 1)
        parts: list[torch.Tensor] = []
        if before:
            expand_shape = list(x.shape)
            expand_shape[dim] = before
            parts.append(first.expand(expand_shape))
        parts.append(x)
        if after:
            expand_shape = list(x.shape)
            expand_shape[dim] = after
            parts.append(last.expand(expand_shape))
        return torch.cat(parts, dim=dim), AxisPad(before, after)

    need = length - size
    if mode == "repeat_last":
        return x.narrow(dim, 0, size).contiguous(), AxisPad(0, need)

    before = need // 2
    after = need - before
    return x.narrow(dim, before, size).contiguous(), AxisPad(before, after)


def ensure_min_latent_shape(
    latent: torch.Tensor,
    min_tile_sizes: Tuple[int, int, int],
) -> tuple[torch.Tensor, tuple[AxisPad, AxisPad, AxisPad]]:
    min_t, min_h, min_w = min_tile_sizes
    t_pad = AxisPad(0, 0)
    h_pad = AxisPad(0, 0)
    w_pad = AxisPad(0, 0)
    x = latent
    if x.shape[2] < min_t:
        x, t_pad = resize_axis(x, 2, min_t, mode="repeat_last")
    if x.shape[3] < min_h:
        x, h_pad = resize_axis(x, 3, min_h, mode="symmetric")
    if x.shape[4] < min_w:
        x, w_pad = resize_axis(x, 4, min_w, mode="symmetric")
    return x, (t_pad, h_pad, w_pad)


def scale_axis_pad(pad: AxisPad, scale: int) -> AxisPad:
    return AxisPad(pad.before * scale, pad.after * scale)


def crop_pixels_to_content(
    pixels: torch.Tensor,
    frames: int,
    height: int,
    width: int,
    *,
    h_pad: AxisPad | None = None,
    w_pad: AxisPad | None = None,
    spatial_scale: Tuple[int, int] = (1, 1),
) -> torch.Tensor:
    x, _ = resize_axis(pixels, 2, frames, mode="repeat_last")
    scale_h, scale_w = spatial_scale
    if h_pad is not None:
        before = scale_axis_pad(h_pad, scale_h).before
        if before + height > x.shape[3]:
            raise ValueError(f"H crop out of range: before={before}, height={height}, got {x.shape[3]}")
        x = x.narrow(3, before, height).contiguous()
    else:
        x, _ = resize_axis(x, 3, height, mode="symmetric")
    if w_pad is not None:
        before = scale_axis_pad(w_pad, scale_w).before
        if before + width > x.shape[4]:
            raise ValueError(f"W crop out of range: before={before}, width={width}, got {x.shape[4]}")
        x = x.narrow(4, before, width).contiguous()
    else:
        x, _ = resize_axis(x, 4, width, mode="symmetric")
    return x


def stage5_pixel_shape_from_stage4(
    stage4_t: int,
    stage4_h: int,
    stage4_w: int,
    *,
    upsample_stride: Tuple[int, int, int],
    patch_size: int,
    stage5_kernel_t: int,
    drop_leading_frame: bool,
    pad_trailing: bool,
) -> tuple[int, int, int]:
    st, sh, sw = upsample_stride
    frames = stage4_t * st - 1 if drop_leading_frame and st == 2 else stage4_t * st
    if pad_trailing:
        frames = max(frames, stage5_kernel_t)
    return frames, stage4_h * sh * patch_size, stage4_w * sw * patch_size


def pad_trailing_latent_for_natten_border(latent: torch.Tensor, n_frames: int) -> torch.Tensor:
    if n_frames <= 0:
        return latent
    padded, _ = resize_axis(latent, 2, latent.shape[2] + n_frames, mode="repeat_last")
    return padded


def crop_trailing_context_natten_pad(
    context: torch.Tensor,
    *,
    n_latent_frames: int,
    time_scale: int,
    stage5_kernel_t: int,
) -> torch.Tensor:
    if n_latent_frames <= 0:
        return context
    ghost = n_latent_frames * time_scale
    content_t = max(context.shape[1] - ghost, 1)
    keep = min(context.shape[1], max(content_t, stage5_kernel_t))
    cropped, _ = resize_axis(context, 1, keep, mode="repeat_last")
    return cropped


def _weight_floor(dtype: torch.dtype) -> float:
    return max(1e-8, torch.finfo(dtype).tiny)


def stage4_thw_from_latent(
    upsample_strides: Sequence[Tuple[int, int, int]],
    latent_t: int,
    latent_h: int,
    latent_w: int,
    *,
    drop_leading_frame: bool = True,
) -> Tuple[int, int, int]:
    t, h, w = latent_t, latent_h, latent_w
    for st, sh, sw in upsample_strides[:3]:
        t, h, w = t * st, h * sh, w * sw
        if st == 2 and drop_leading_frame:
            t -= 1
    return t, h, w


def stage4_to_pixel_scale_factors(
    upsample_stride: Tuple[int, int, int],
    patch_size: int,
) -> SpatioTemporalScaleFactors:
    st, sh, sw = upsample_stride
    return SpatioTemporalScaleFactors(time=st, height=sh * patch_size, width=sw * patch_size)


def compute_tile_min_size(
    stage4_kernel: Tuple[int, int, int],
    stage5_kernel: Tuple[int, int, int],
    upsample3_stride: Tuple[int, int, int],
) -> Tuple[int, int, int]:
    return tuple(max(stage4_kernel[a], -(-stage5_kernel[a] // upsample3_stride[a])) for a in range(3))


def compute_tile_halos(
    stage4_kernel: Tuple[int, int, int],
    stage4_depth: int,
    stage5_kernel: Tuple[int, int, int],
    stage5_depth: int,
    upsample3_stride: Tuple[int, int, int],
) -> Tuple[Tuple[int, int, int], Tuple[int, int, int]]:
    halo4 = tuple(stage4_depth * (stage4_kernel[a] // 2) for a in range(3))
    halo5 = tuple(-(-(stage5_depth * (stage5_kernel[a] // 2)) // upsample3_stride[a]) for a in range(3))
    return halo4, halo5  # type: ignore[return-value]


def _cumulative_upsample_strides(
    upsamples: Sequence[Tuple[Tuple[int, int, int], int]],
) -> List[Tuple[int, int, int]]:
    cumulative = [(1, 1, 1)]
    t, h, w = 1, 1, 1
    for stride, _ in upsamples:
        t, h, w = t * stride[0], h * stride[1], w * stride[2]
        cumulative.append((t, h, w))
    return cumulative


def all_stages_min_tile_size(
    stage_kernels: Sequence[Tuple[int, int, int]],
    upsamples: Sequence[Tuple[Tuple[int, int, int], int]],
    stage5_kernel: Tuple[int, int, int],
) -> Tuple[int, int, int]:
    cumulative = _cumulative_upsample_strides(upsamples)
    mins = [1, 1, 1]
    for stage_i in range(len(upsamples)):
        strides = cumulative[stage_i]
        for axis in range(3):
            mins[axis] = max(mins[axis], -(-stage_kernels[stage_i][axis] // strides[axis]))
    strides5 = cumulative[len(upsamples)]
    for axis in range(3):
        mins[axis] = max(mins[axis], -(-stage5_kernel[axis] // strides5[axis]))
    return (mins[0], mins[1], mins[2])


def pixel_tile_shape(full_shape: tuple[int, ...], out_coords: tuple[slice, ...]) -> tuple[int, ...]:
    dims: list[int] = []
    for size, coord in zip(full_shape, out_coords, strict=True):
        start, stop, step = coord.indices(size)
        dims.append(len(range(start, stop, step)))
    return tuple(dims)


def _round_up(value: int, multiple: int) -> int:
    return -(-value // multiple) * multiple


def recommended_pixel_overlaps(
    tile_halos: Tuple[Tuple[int, int, int], Tuple[int, int, int]],
    pixel_scale: SpatioTemporalScaleFactors,
) -> Tuple[int, int]:
    def dominant(axis: int) -> int:
        return max(tile_halos[i][axis] for i in range(len(tile_halos)))

    overlap_t = _round_up(dominant(0) * pixel_scale.time, 8)
    halo_hw = max(dominant(1), dominant(2))
    overlap_hw = _round_up(halo_hw * pixel_scale.height, 32)
    return overlap_t, overlap_hw


def stage5_tokens_for_pixel_tile(
    tile_frames: int,
    tile_height: int,
    tile_width: int,
    *,
    patch_size: int,
) -> int:
    h5 = max(1, tile_height // patch_size)
    w5 = max(1, tile_width // patch_size)
    return tile_frames * h5 * w5


def _axis_candidates(length: int, overlap: int, min_size: int, multiple: int) -> list[tuple[int, int]]:
    out: list[tuple[int, int]] = []
    max_size = max(_round_up(length, multiple), min_size)
    for size in range(min_size, max_size + multiple, multiple):
        if size <= overlap:
            continue
        n = len(split_by_size(size, overlap)(length).intervals)
        out.append((size, n))
    return out


def volumetric_overlap_waste(
    *,
    num_frames: int,
    height: int,
    width: int,
    tile_frames: int,
    tile_height: int,
    tile_width: int,
    n_t: int,
    n_h: int,
    n_w: int,
) -> float:
    processed = n_t * n_h * n_w * tile_frames * tile_height * tile_width
    unique = max(1, num_frames * height * width)
    return processed / unique


def _propagate_interval_through_upsample_hops(
    interval: DimensionInterval,
    strides: Sequence[int],
    causal: bool,
) -> DimensionInterval:
    x = interval
    for stride in strides:
        if stride < 1:
            raise ValueError(f"upsample stride must be >= 1, got {stride}")
        start = x.start * stride
        end = x.end * stride
        left_ramp = x.left_ramp * stride
        right_ramp = x.right_ramp * stride
        if causal and stride == 2:
            end -= 1
            if x.start != 0:
                start -= 1
        x = DimensionInterval(start=start, end=end, left_ramp=left_ramp, right_ramp=right_ramp)
    return x


"""Shared small layers for the diffusion-VAE NA transformer stack."""


class ChannelLinear(nn.Linear):
    @property
    def in_channels(self) -> int:
        return self.in_features

    @property
    def out_channels(self) -> int:
        return self.out_features


class LinearPixelShuffleUpsample(nn.Module):
    def __init__(
        self,
        in_channels: int,
        stride: tuple[int, int, int],
        out_channels_reduction_factor: int = 1,
    ) -> None:
        super().__init__()
        self.stride = stride
        self.proj_out_channels = math.prod(stride) * in_channels // out_channels_reduction_factor
        self.out_channels = self.proj_out_channels // math.prod(stride)
        self.proj = nn.Linear(in_channels, self.proj_out_channels, bias=True)

    def forward(self, x: torch.Tensor, drop_leading_frame: bool = True) -> torch.Tensor:
        x = self.proj(x)
        x = rearrange(
            x,
            "b t h w (c p1 p2 p3) -> b (t p1) (h p2) (w p3) c",
            p1=self.stride[0],
            p2=self.stride[1],
            p3=self.stride[2],
        )
        if self.stride[0] == 2 and drop_leading_frame:
            x = x[:, 1:, :, :, :]
        return x


class AdaLNZero(nn.Module):
    NUM_CHUNKS: int = 7  # scale_msa, shift_msa, gate_msa, scale_mlp, shift_mlp, gate_mlp, gate_ctx

    def __init__(self, dim: int, t_emb_dim: int) -> None:
        super().__init__()
        self.dim = dim
        self.proj = nn.Linear(t_emb_dim, self.NUM_CHUNKS * dim, bias=True)
        nn.init.zeros_(self.proj.weight)
        nn.init.zeros_(self.proj.bias)

    def forward(self, t_emb: torch.Tensor) -> tuple[torch.Tensor, ...]:
        h = self.proj(F.silu(t_emb))
        chunks = h.chunk(self.NUM_CHUNKS, dim=-1)
        return tuple(c[:, None, None, None, :] for c in chunks)


def modulate(x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor) -> torch.Tensor:
    return x * (1.0 + scale) + shift


"""Shared absolute-RoPE math helpers (no consumer policy)."""


DEFAULT_ABS_ROPE_NUM_TILES = 4


def t_positions(t: int, device: torch.device) -> torch.Tensor:
    return torch.arange(t, dtype=torch.float32, device=device)


def h_positions(h: int, device: torch.device) -> torch.Tensor:
    return torch.arange(h, dtype=torch.float32, device=device)


def default_rope_dim_split(head_dim: int) -> tuple[int, int, int]:
    assert head_dim % 8 == 0, f"head_dim={head_dim} must be a multiple of 8 for default split"
    d_t = (head_dim // 4) // 2 * 2
    d_hw = (head_dim - d_t) // 2
    if d_hw % 2 != 0:
        d_t -= 2
        d_hw = (head_dim - d_t) // 2
    assert d_t > 0
    assert d_hw > 0
    return (d_t, d_hw, d_hw)


def rope_inv_freqs(dim: int, base: float = 10000.0) -> torch.Tensor:
    assert dim % 2 == 0, f"RoPE dim must be even, got {dim}"
    exponents = np.arange(0, dim, 2, dtype=np.float64) / dim
    inv_freqs = 1.0 / np.power(float(base), exponents)
    return torch.from_numpy(inv_freqs).to(torch.float32)


def rot_abs_axis_impl(
    xc: torch.Tensor,
    pos: torch.Tensor,
    inv: torch.Tensor,
    axis: int,
    *,
    compute_dtype: torch.dtype,
) -> torch.Tensor:
    out_dtype = xc.dtype
    pairs = xc.reshape(*xc.shape[:-1], xc.shape[-1] // 2, 2)
    xe = pairs[..., 0].to(compute_dtype)
    xo = pairs[..., 1].to(compute_dtype)
    shape = [1, 1, 1, 1, 1, inv.shape[0]]
    shape[axis] = pos.shape[0]
    ang = (pos[:, None] * inv[None, :]).reshape(shape)
    c = ang.cos().to(compute_dtype)
    s = ang.sin().to(compute_dtype)
    re = xe * c - xo * s
    ro = xe * s + xo * c
    out = torch.stack([re, ro], dim=-1).reshape(xc.shape)
    return out.to(out_dtype) if out.dtype != out_dtype else out


"""Opaque full-volume abs-RoPE for deterministic (pre-diffusion) NA.
Owns QKV + opaque ``custom_op`` packaging for det ``NA.forward``. Det stages
have differently shaped T/H/W; the opaque op keeps Dynamo from specializing
on each stage shape. Diffusion paths use ``diff_attn/`` RoPE — not this file.
No T/H/W origin/offset is threaded for tiled decode, and none is needed:
every attention call here is ``natten.na3d``, a local window with no
cross-tile tokens. Absolute-vs-local RoPE differs by a global phase that
cancels inside the attention softmax over that window, so the attention
output is unchanged. Since every tiled-decode call processes exactly one
tile in isolation, using each tile's local 0-based positions is identical
to using its true absolute origin. Absolute origin still matters for
whether a tile contains the latent's first frame (``drop_leading_frame``),
which is handled outside RoPE in the decoder stage / per-tile decode.
"""


def _apply_opaque_rope_slab(
    x: torch.Tensor,
    rope_split: tuple[int, int, int],
    inv_freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
    *,
    w_pos: torch.Tensor,
    compute_dtype: torch.dtype,
    t_pos: torch.Tensor | None = None,
) -> torch.Tensor:
    d_t, d_h, _ = rope_split
    inv_t, inv_h, inv_w = inv_freqs
    t = x.shape[1]
    h = x.shape[2]
    positions_t = t_positions(t, x.device) if t_pos is None else t_pos
    xt = rot_abs_axis_impl(x[..., :d_t], positions_t, inv_t, axis=1, compute_dtype=compute_dtype)
    xh = rot_abs_axis_impl(
        x[..., d_t : d_t + d_h],
        h_positions(h, x.device),
        inv_h,
        axis=2,
        compute_dtype=compute_dtype,
    )
    xw = rot_abs_axis_impl(x[..., d_t + d_h :], w_pos, inv_w, axis=3, compute_dtype=compute_dtype)
    return torch.cat([xt, xh, xw], dim=-1)


def _apply_opaque_tiled_rope(
    x: torch.Tensor,
    rope_split: tuple[int, int, int],
    inv_freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
    *,
    num_tiles: int,
    compute_dtype: torch.dtype,
    t_pos: torch.Tensor | None = None,
) -> torch.Tensor:
    slabs = torch.chunk(x, num_tiles, dim=3)
    w_off = 0
    parts: list[torch.Tensor] = []
    for slab in slabs:
        w_slab = slab.shape[3]
        w_pos = torch.arange(w_slab, dtype=torch.float32, device=x.device) + w_off
        parts.append(
            _apply_opaque_rope_slab(
                slab,
                rope_split,
                inv_freqs,
                w_pos=w_pos,
                compute_dtype=compute_dtype,
                t_pos=t_pos,
            )
        )
        w_off = w_off + w_slab
    return torch.cat(parts, dim=3)


@torch.library.custom_op("diffsynth_ltx25::abs_rope", mutates_args=())
def _abs_rope_op(
    x: torch.Tensor,
    inv_t: torch.Tensor,
    inv_h: torch.Tensor,
    inv_w: torch.Tensor,
    d_t: int,
    d_h: int,
    d_w: int,
    num_tiles: int,
    compute_dtype_is_bf16: bool,
) -> torch.Tensor:
    compute_dtype = torch.bfloat16 if compute_dtype_is_bf16 else torch.float32
    return _apply_opaque_tiled_rope(
        x,
        (d_t, d_h, d_w),
        (inv_t, inv_h, inv_w),
        num_tiles=num_tiles,
        compute_dtype=compute_dtype,
    )


def _apply_opaque_abs_rope(
    x: torch.Tensor,
    rope_split: tuple[int, int, int],
    inv_freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
    *,
    num_tiles: int,
    compute_dtype: torch.dtype,
) -> torch.Tensor:
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    if compute_dtype not in (torch.float32, torch.bfloat16):
        raise ValueError(f"compute_dtype must be float32 or bfloat16, got {compute_dtype}")
    d_t, d_h, d_w = rope_split
    inv_t, inv_h, inv_w = inv_freqs
    return _abs_rope_op(
        x,
        inv_t,
        inv_h,
        inv_w,
        d_t,
        d_h,
        d_w,
        num_tiles,
        compute_dtype == torch.bfloat16,
    )


@torch.library.custom_op("diffsynth_ltx25::abs_rope_at_t", mutates_args=())
def _abs_rope_at_t_op(
    x: torch.Tensor,
    t_pos: torch.Tensor,
    inv_t: torch.Tensor,
    inv_h: torch.Tensor,
    inv_w: torch.Tensor,
    d_t: int,
    d_h: int,
    d_w: int,
    num_tiles: int,
    compute_dtype_is_bf16: bool,
) -> torch.Tensor:
    compute_dtype = torch.bfloat16 if compute_dtype_is_bf16 else torch.float32
    return _apply_opaque_tiled_rope(
        x,
        (d_t, d_h, d_w),
        (inv_t, inv_h, inv_w),
        num_tiles=num_tiles,
        compute_dtype=compute_dtype,
        t_pos=t_pos,
    )


def _rope_config(attn: object, x: torch.Tensor) -> tuple[tuple[torch.Tensor, ...], int, torch.dtype]:
    inv_freqs = (
        attn.rope_inv_t.to(device=x.device),  # type: ignore[attr-defined]
        attn.rope_inv_h.to(device=x.device),  # type: ignore[attr-defined]
        attn.rope_inv_w.to(device=x.device),  # type: ignore[attr-defined]
    )
    num_tiles = getattr(attn, "rope_num_tiles", DEFAULT_ABS_ROPE_NUM_TILES)
    compute_dtype = getattr(attn, "rope_compute_dtype", torch.float32)
    return inv_freqs, num_tiles, compute_dtype


def _det_project_qkv(attn: object, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    q, k, v = attn.project_qkv(x)  # type: ignore[attr-defined]
    q = attn.q_norm(q)  # type: ignore[attr-defined]
    k = attn.k_norm(k)  # type: ignore[attr-defined]
    q = q * attn.scale  # type: ignore[attr-defined]
    return q, k, v


def det_qkv_rope_at_times(
    attn: object,
    x: torch.Tensor,
    t_pos: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    if t_pos.ndim != 1 or t_pos.shape[0] != x.shape[1]:
        raise ValueError(f"t_pos must be ({x.shape[1]},) to match the plane axis, got {tuple(t_pos.shape)}")
    q, k, v = _det_project_qkv(attn, x)

    inv_freqs, num_tiles, compute_dtype = _rope_config(attn, x)
    d_t, d_h, d_w = attn.rope_dim_split  # type: ignore[attr-defined]
    positions = t_pos.to(device=x.device, dtype=torch.float32)
    rotated = [
        _abs_rope_at_t_op(
            tensor,
            positions,
            *inv_freqs,
            d_t,
            d_h,
            d_w,
            num_tiles,
            compute_dtype == torch.bfloat16,
        )
        for tensor in (q, k)
    ]
    return rotated[0], rotated[1], v


def det_qkv_rope(attn: object, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    q, k, v = _det_project_qkv(attn, x)
    inv_freqs, num_tiles, compute_dtype = _rope_config(attn, x)
    q = _apply_opaque_abs_rope(
        q,
        attn.rope_dim_split,  # type: ignore[attr-defined]
        inv_freqs,
        num_tiles=num_tiles,
        compute_dtype=compute_dtype,
    )
    k = _apply_opaque_abs_rope(
        k,
        attn.rope_dim_split,  # type: ignore[attr-defined]
        inv_freqs,
        num_tiles=num_tiles,
        compute_dtype=compute_dtype,
    )
    return q, k, v


def vram_ready_linear(module: nn.Module) -> tuple[torch.Tensor, torch.Tensor | None]:
    # This decoder calls several projections functionally, bypassing the VRAM wrappers'
    # forward, so ask the wrapper for computation-ready weights instead of reading them raw.
    computation = getattr(module, "computation", None)
    if computation is not None:
        return computation()
    return module.weight, module.bias


class QKVProjections(nn.Module):
    def __init__(self, dim: int) -> None:
        super().__init__()
        linear = nn.Linear(dim, dim * 3, bias=True)
        self.weight = linear.weight
        self.bias = linear.bias

    def forward(self, x):
        weight = self.weight.to(device=x.device, dtype=x.dtype)
        bias = self.bias.to(device=x.device, dtype=x.dtype)
        weights = weight.chunk(3, dim=0)
        biases = bias.chunk(3, dim=0)
        return tuple(F.linear(x, weight, bias) for weight, bias in zip(weights, biases, strict=True))


DEFAULT_SWIGLU_TILE_SIZE: Final[int] = 16_384
DEFAULT_SWIGLU_TILES: Final[int] = 4


@dataclass(frozen=True)
class SwiGLUTileSpec:
    num_tiles: int | None = None
    tile_size: int | None = None

    def __post_init__(self) -> None:
        if (self.num_tiles is None) == (self.tile_size is None):
            raise ValueError("Provide exactly one of num_tiles or tile_size")
        if self.num_tiles is not None and self.num_tiles < 1:
            raise ValueError("num_tiles must be >= 1")
        if self.tile_size is not None and self.tile_size < 1:
            raise ValueError("tile_size must be >= 1")

DEFAULT_SWIGLU_TILE_SPEC = SwiGLUTileSpec(tile_size=DEFAULT_SWIGLU_TILE_SIZE)


def _swiglu_chunk(x, w_gate, w_up, w_down):
    return F.linear(F.silu(F.linear(x, w_gate)) * F.linear(x, w_up), w_down)


def swiglu_tiled(x, w_gate, w_up, w_down, tile, *, use_triton=None):
    del use_triton
    if x.numel() == 0:
        return x
    leading = x.shape[:-1]
    flat = x.reshape(-1, x.shape[-1])
    if tile.tile_size is not None:
        chunk_size = tile.tile_size
    else:
        chunk_size = max(1, math.ceil(flat.shape[0] / tile.num_tiles))
    output = torch.cat([_swiglu_chunk(chunk, w_gate, w_up, w_down) for chunk in flat.split(chunk_size)], dim=0)
    return output.reshape(*leading, output.shape[-1])


def swiglu_weights(mlp) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    return (
        vram_ready_linear(mlp.w_gate)[0],
        vram_ready_linear(mlp.w_up)[0],
        vram_ready_linear(mlp.w_down)[0],
    )


def plain_mlp(x, mlp, norm, tile):
    y = norm(x)
    if y.numel() == 0:
        return x
    return x + swiglu_tiled(y, *swiglu_weights(mlp), tile)


class SwiGLU(nn.Module):
    def __init__(self, dim: int, hidden_dim: int, tile: SwiGLUTileSpec = DEFAULT_SWIGLU_TILE_SPEC) -> None:
        super().__init__()
        self.w_up = nn.Linear(dim, hidden_dim, bias=False)
        self.w_gate = nn.Linear(dim, hidden_dim, bias=False)
        self.w_down = nn.Linear(hidden_dim, dim, bias=False)
        self.tile = tile

    def forward(self, x):
        return swiglu_tiled(x, *swiglu_weights(self), self.tile)


"""Limited-workspace 3D neighborhood attention (NATTEN ``na3d`` semantics) in pure torch.
Vendored from comfy-kitchen ``backends/eager/na.py`` (Apache-2.0) for DiffVAE hosts
without natten or Triton. Queries are tiled; tiles that share window geometry stack
into batched ``scaled_dot_product_attention`` calls with one additive mask per group.
"""


NA_SCORE_BUDGET = 2**25


NA_KV_STACK_BUDGET = 2**28


def _window_bounds(length: int, kernel: int, causal: bool) -> tuple[list[int], list[int]]:
    starts: list[int] = []
    ends: list[int] = []
    if causal:
        for i in range(length):
            starts.append(max(0, i - kernel + 1))
            ends.append(i + 1)
    else:
        kernel = min(kernel, length)
        lo = length - kernel
        half = kernel // 2
        for i in range(length):
            start = min(max(i - half, 0), lo)
            starts.append(start)
            ends.append(start + kernel)
    return starts, ends


def _pick_tiles(dims: tuple[int, int, int], kernels: list[int]) -> list[int]:
    tiles = list(dims)

    def cost(ts: list[int]) -> int:
        nq = math.prod(ts)
        nk = math.prod(min(d, t + k - 1) for t, k, d in zip(ts, kernels, dims, strict=True))
        return nq * nk

    while cost(tiles) > NA_SCORE_BUDGET and max(tiles) > 1:
        i = max(range(3), key=lambda a: tiles[a] / kernels[a])
        if tiles[i] <= 1:
            break
        tiles[i] = max(1, (tiles[i] + 1) // 2)
    return tiles


def _group_mask(
    rel_bounds: tuple[tuple[tuple[int, ...], tuple[int, ...]], ...],
    dtype: torch.dtype,
    device: torch.device,
) -> torch.Tensor:
    bools = []
    for starts, ends in rel_bounds:
        st = torch.tensor(starts, device=device)
        en = torch.tensor(ends, device=device)
        kj = torch.arange(int(en.max()), device=device)
        bools.append((kj[None, :] >= st[:, None]) & (kj[None, :] < en[:, None]))
    visible = (
        bools[0][:, None, None, :, None, None]
        & bools[1][None, :, None, None, :, None]
        & bools[2][None, None, :, None, None, :]
    )
    nq = visible.shape[0] * visible.shape[1] * visible.shape[2]
    nk = visible.shape[3] * visible.shape[4] * visible.shape[5]
    mask = torch.zeros((nq, nk), dtype=dtype, device=device)
    mask.masked_fill_(~visible.reshape(nq, nk), torch.finfo(dtype).min)
    return mask.reshape(1, 1, nq, nk)


def na3d(
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    kernel_size: list[int] | tuple[int, ...],
    is_causal: list[bool] | None = None,
    scale: float | None = None,
) -> torch.Tensor:
    batch, t, h, w, nh, hd = q.shape
    dims = (t, h, w)
    causal = [False, False, False] if is_causal is None else list(is_causal)
    kernels = [k_ if c else min(k_, d) for k_, c, d in zip(kernel_size, causal, dims, strict=True)]
    if scale is None:
        scale = hd**-0.5
    device = q.device
    if scale != 1.0:
        q = q * scale

    bounds = [_window_bounds(d, k_, c) for d, k_, c in zip(dims, kernels, causal, strict=True)]
    tile_t, tile_h, tile_w = _pick_tiles(dims, [min(k_, d) for k_, d in zip(kernels, dims, strict=True)])

    groups: dict[
        tuple[
            tuple[tuple[int, ...], tuple[int, ...]],
            tuple[tuple[int, ...], tuple[int, ...]],
            tuple[tuple[int, ...], tuple[int, ...]],
        ],
        list[tuple[tuple[slice, slice, slice], tuple[slice, slice, slice]]],
    ] = {}
    for t0 in range(0, t, tile_t):
        t1 = min(t0 + tile_t, t)
        rt0, rt1 = bounds[0][0][t0], bounds[0][1][t1 - 1]
        rel_t = (
            tuple(s - rt0 for s in bounds[0][0][t0:t1]),
            tuple(e - rt0 for e in bounds[0][1][t0:t1]),
        )
        for h0 in range(0, h, tile_h):
            h1 = min(h0 + tile_h, h)
            rh0, rh1 = bounds[1][0][h0], bounds[1][1][h1 - 1]
            rel_h = (
                tuple(s - rh0 for s in bounds[1][0][h0:h1]),
                tuple(e - rh0 for e in bounds[1][1][h0:h1]),
            )
            for w0 in range(0, w, tile_w):
                w1 = min(w0 + tile_w, w)
                rw0, rw1 = bounds[2][0][w0], bounds[2][1][w1 - 1]
                rel_w = (
                    tuple(s - rw0 for s in bounds[2][0][w0:w1]),
                    tuple(e - rw0 for e in bounds[2][1][w0:w1]),
                )
                groups.setdefault((rel_t, rel_h, rel_w), []).append(
                    (
                        (slice(t0, t1), slice(h0, h1), slice(w0, w1)),
                        (slice(rt0, rt1), slice(rh0, rh1), slice(rw0, rw1)),
                    )
                )

    out = torch.empty((batch, t, h, w, nh, hd), device=device, dtype=v.dtype)
    for rel, tiles in groups.items():
        mask = _group_mask(rel, q.dtype, device)
        nq, nk = mask.shape[2], mask.shape[3]
        g_max = max(1, NA_KV_STACK_BUDGET // max(1, batch * nh * nk * hd * 2)) if device.type == "cuda" else 1
        qs0, _ = tiles[0]
        tq = qs0[0].stop - qs0[0].start
        th = qs0[1].stop - qs0[1].start
        tw = qs0[2].stop - qs0[2].start
        for c0 in range(0, len(tiles), g_max):
            chunk = tiles[c0 : c0 + g_max]
            g = len(chunk)
            q_s = torch.stack([q[:, qs[0], qs[1], qs[2]] for qs, _ in chunk])
            k_s = torch.stack([k[:, rs[0], rs[1], rs[2]] for _, rs in chunk])
            v_s = torch.stack([v[:, rs[0], rs[1], rs[2]] for _, rs in chunk])
            q_s = q_s.permute(0, 1, 5, 2, 3, 4, 6).reshape(g * batch, nh, nq, hd)
            k_s = k_s.permute(0, 1, 5, 2, 3, 4, 6).reshape(g * batch, nh, nk, hd)
            v_s = v_s.permute(0, 1, 5, 2, 3, 4, 6).reshape(g * batch, nh, nk, hd)
            o = functional.scaled_dot_product_attention(q_s, k_s, v_s, attn_mask=mask, scale=1.0)
            o = o.view(g, batch, nh, tq, th, tw, hd).permute(0, 1, 3, 4, 5, 2, 6)
            for i, (qs, _) in enumerate(chunk):
                out[:, qs[0], qs[1], qs[2]] = o[i]

    return out


"""Pure-torch joint (video + keyframe) 3D neighborhood attention.
Computes, for one softmax per query:
* **video query** at ``(t, h, w)``: its local ``Kt x Kh x Kw`` video window, clamped to the
  volume and masked where it hangs over the edge, **plus** the whole ``Kh x Kw`` window at the
  same ``(h, w)`` on each of the ``num_slots`` nearest keyframe planes. Keyframe visibility does
  not depend on ``Kt`` -- a plane far outside the temporal radius is still visible.
* **keyframe query** on plane ``i``: the ``Kh x Kw`` window on its own plane (there is no
  plane-to-plane attention), plus the same window on each of the nearest video frames.
Which planes and frames are "nearest" comes from :func:`video_keyframe_slots` /
:func:`keyframe_video_slots`, so every backend agrees on visibility.
No Triton, no natten, no ``torch.compile``: this is the backend that always exists -- CPU,
macOS/MPS, Windows without a built extra.
Structure
---------
Everything is arranged so the arithmetic happens inside ``F.scaled_dot_product_attention``:
**Query bricks.** Queries are grouped into ``(bt, bh, bw)`` bricks and many bricks ride one SDPA
call as its batch dimension, so a frame costs a handful of launches rather than thousands. All
queries in a brick share one gathered key slab, of extent ``(bt + Kt - 1, bh + Kh - 1, bw + Kw - 1)``.
**Brick shape.** Wasted work is ``Nk / keys_actually_visible`` and *grows* with the brick, so the
spatial face stays small and square (square minimizes the slab at a fixed query count). Depth is
the exception: the gather is the larger cost and it scales with ``Nk / Nq``, which *falls* with
depth, so :data:`DEFAULT_BRICK_DEPTH` frames deep beats one frame deep despite doing more
arithmetic. Both defaults sit on measured plateaus at the production stage-5 shape.
**One shared, 2D-broadcast mask.** The visible-key pattern is a property of the brick geometry,
identical for every brick, so it is built once and passed as ``(1, 1, Nq, Nk)``. That shape is
load-bearing: torch keeps the memory-efficient backend and expands neither the mask nor the
scores, whereas a pre-expanded ``(G, NH, Nq, Nk)`` bias halves throughput and costs gigabytes.
**Per-key validity rides in the keys.** Out-of-volume positions, empty slots (``-1``) and invalid
planes are data-dependent, so folding them into the mask would make it per-brick. Instead ``K``
carries one extra channel holding ``0`` for a live key and :data:`_DEAD` for a dead one, against a
constant ``1`` channel on ``Q``. Q arrives pre-scaled, so with ``scale=1.0`` that adds exactly the
bias to the score. The channel count is then rounded up to :data:`_HEAD_DIM_ALIGN`.
**Head-major staging.** Key slabs are gathered from a ``(B, NH, A, Hp, Wp, C)`` copy rather than
from the caller's channels-last layout, so the gather's innermost contiguous run is ``ew * C``
instead of ``C``.
**Runs of constant slot row.** A brick spanning several frames shares one keyframe key slab, so it
must not straddle a change of visible planes. ``T`` is cut into maximal runs of identical slot rows
and bricks are tiled inside a run -- which also means one plane gather per run, entering the slab
view with a **zero** group stride.
Both loops are budgeted by :data:`DEFAULT_WORKSPACE_BYTES`: frames per staging pass, then
``(bricks, brick rows)`` per SDPA call. Peak transient memory is therefore bounded by that budget
and not by the volume, which is what lets this sit next to a decoder that has its own memory plan.
The gather is the floor: SDPA needs materialized ``(G, NH, Nk, HD)`` keys, so every key is copied
``Nk / Nq`` times. Only a fused neighborhood kernel avoids that.
"""


_DEAD = -1.0e4


_HEAD_DIM_ALIGN = 8


DEFAULT_BRICK_QUERIES = 64


DEFAULT_BRICK_DEPTH = 4


DEFAULT_WORKSPACE_BYTES = 256 * 1024**2


_STAGING_FACTOR_FUSED = 4.75


_STAGING_FACTOR_MATERIALIZED = 22.1


def sdpa_materializes_scores(device: torch.device) -> bool:
    return device.type != "cuda"


def staging_factor(device: torch.device) -> float:
    return _STAGING_FACTOR_MATERIALIZED if sdpa_materializes_scores(device) else _STAGING_FACTOR_FUSED


def _key_channels(head_dim: int) -> int:
    return -(-(head_dim + 1) // _HEAD_DIM_ALIGN) * _HEAD_DIM_ALIGN


def _window(kernel: int) -> tuple[int, int]:
    lo = kernel // 2
    return lo, kernel - lo - 1


def pick_brick(
    time: int,
    height: int,
    width: int,
    target: int = DEFAULT_BRICK_QUERIES,
    depth: int = DEFAULT_BRICK_DEPTH,
) -> tuple[int, int, int]:
    side = max(1, round(math.sqrt(target)))
    return min(depth, time), min(side, height), min(side, width)


class _Geometry:
    def __init__(
        self,
        height: int,
        width: int,
        kernel: tuple[int, int, int],
        brick: tuple[int, int, int],
    ) -> None:
        kernel_t, kernel_h, kernel_w = kernel
        lo_h, hi_h = _window(kernel_h)
        lo_w, hi_w = _window(kernel_w)
        self.height, self.width = height, width
        self.brick = brick
        self.kernel = kernel
        self.grid = (-(-height // brick[1]), -(-width // brick[2]))
        # Slab extents: T grows with the brick depth, H/W with the spatial face.
        self.span_t = brick[0] + kernel_t - 1
        self.span = (brick[1] + kernel_h - 1, brick[2] + kernel_w - 1)
        # Halo, plus enough to cover the last (partial) brick's slab.
        self.pad_h = (lo_h, hi_h + self.grid[0] * brick[1] - height)
        self.pad_w = (lo_w, hi_w + self.grid[1] * brick[2] - width)
        self.pad_t = _window(kernel_t)
        self.queries = brick[0] * brick[1] * brick[2]
        self.footprint = self.span[0] * self.span[1]
        self.padded_height = height + sum(self.pad_h)
        self.padded_width = width + sum(self.pad_w)

    def row_extent(self, rows: int) -> int:
        return (rows - 1) * self.brick[1] + self.span[0]


class _Schedule:
    def __init__(
        self,
        geometry: _Geometry,
        blocks: int,
        heads: int,
        head_dim: int,
        axis_bricks: int,
        element_size: int,
        workspace_bytes: int,
        factor: float,
    ) -> None:
        channels = _key_channels(head_dim)
        # One (brick along the axis, brick row) pair's worth of gathered keys and values, plus its
        # score block on backends that materialize one.
        keys = blocks * geometry.footprint
        pair_bytes = geometry.grid[1] * heads * keys * (channels + head_dim) * element_size
        # ``factor`` folds in whatever the selected SDPA kernel allocates on top of the staging,
        # chiefly a materialized score block. See :data:`_STAGING_FACTOR_FUSED`.
        pairs = max(1, int(workspace_bytes / max(pair_bytes * factor, 1.0)))
        if pairs >= geometry.grid[0]:
            self.group_axis = min(axis_bricks, max(1, pairs // geometry.grid[0]))
            self.group_rows = geometry.grid[0]
        else:
            self.group_axis = 1
            self.group_rows = pairs
        staged = geometry.padded_height * geometry.padded_width * heads * (channels + head_dim) * element_size
        per_axis_brick = staged * geometry.brick[0]
        self.stage_axis = min(axis_bricks, max(self.group_axis, workspace_bytes // max(per_axis_brick, 1)))


def _banded(queries: int, span: int, kernel: int, device: torch.device) -> torch.Tensor:
    key = torch.arange(span, device=device)[None, :]
    query = torch.arange(queries, device=device)[:, None]
    return (key >= query) & (key < query + kernel)


def _joint_mask(geometry: _Geometry, num_slots: int, device: torch.device) -> torch.Tensor:
    brick_t, brick_h, brick_w = geometry.brick
    kernel_t, kernel_h, kernel_w = geometry.kernel
    spatial = (
        _banded(brick_h, geometry.span[0], kernel_h, device)[:, None, :, None]
        & _banded(brick_w, geometry.span[1], kernel_w, device)[None, :, None, :]
    ).reshape(brick_h * brick_w, geometry.footprint)
    temporal = _banded(brick_t, geometry.span_t, kernel_t, device)
    video = (temporal[:, None, :, None] & spatial[None, :, None, :]).reshape(
        geometry.queries, geometry.span_t * geometry.footprint
    )
    planes = (
        spatial[None, :, None, :]
        .expand(brick_t, brick_h * brick_w, num_slots, geometry.footprint)
        .reshape(geometry.queries, num_slots * geometry.footprint)
    )
    return torch.cat([video, planes], dim=1)[None, None].contiguous()


def _stage(
    x: torch.Tensor,
    geometry: _Geometry,
    pad_t: tuple[int, int],
    *,
    with_bias_channel: bool,
) -> torch.Tensor:
    batch, axis, height, width, heads, head_dim = x.shape
    channels = _key_channels(head_dim) if with_bias_channel else head_dim
    out = x.new_zeros((batch, heads, axis + sum(pad_t), geometry.padded_height, geometry.padded_width, channels))
    if with_bias_channel:
        out[..., head_dim] = _DEAD
    live = out[
        :,
        :,
        pad_t[0] : pad_t[0] + axis,
        geometry.pad_h[0] : geometry.pad_h[0] + height,
        geometry.pad_w[0] : geometry.pad_w[0] + width,
    ]
    live[..., :head_dim] = x.permute(0, 4, 1, 2, 3, 5)
    if with_bias_channel:
        live[..., head_dim] = 0.0
    return out


def _slabs(
    staged: torch.Tensor,
    geometry: _Geometry,
    bricks: int,
    rows: int,
    blocks: int,
    *,
    group_stride: int,
) -> torch.Tensor:
    batch, heads = staged.shape[0], staged.shape[1]
    stride_b, stride_nh, stride_a, stride_h, stride_w, _ = staged.stride()
    return staged.as_strided(
        (batch, bricks, rows, geometry.grid[1], heads, blocks, *geometry.span, staged.shape[-1]),
        (
            stride_b,
            group_stride * stride_a,
            geometry.brick[1] * stride_h,
            geometry.brick[2] * stride_w,
            stride_nh,
            stride_a,
            stride_h,
            stride_w,
            1,
        ),
    )


def _query_bricks(x: torch.Tensor, geometry: _Geometry, bricks: int, rows: int) -> torch.Tensor:
    batch, axis, height, width, heads, head_dim = x.shape
    brick_t, brick_h, brick_w = geometry.brick
    pad_t, pad_h, pad_w = bricks * brick_t - axis, rows * brick_h - height, geometry.grid[1] * brick_w - width
    if pad_t or pad_h or pad_w:
        x = F.pad(x, (0, 0, 0, 0, 0, pad_w, 0, pad_h, 0, pad_t))
    bricked = (
        x.reshape(batch, bricks, brick_t, rows, brick_h, geometry.grid[1], brick_w, heads, head_dim)
        .permute(0, 1, 3, 5, 7, 2, 4, 6, 8)
        .reshape(batch * bricks * rows * geometry.grid[1], heads, geometry.queries, head_dim)
    )
    out = bricked.new_zeros((*bricked.shape[:-1], _key_channels(head_dim)))
    out[..., :head_dim] = bricked
    out[..., head_dim] = 1.0
    return out


def _unbrick(
    attended: torch.Tensor,
    geometry: _Geometry,
    batch: int,
    bricks: int,
    rows: int,
    extent: tuple[int, int],
) -> torch.Tensor:
    brick_t, brick_h, brick_w = geometry.brick
    heads, head_dim = attended.shape[1], attended.shape[3]
    plane = (
        attended.reshape(batch, bricks, rows, geometry.grid[1], heads, brick_t, brick_h, brick_w, head_dim)
        .permute(0, 1, 5, 2, 6, 3, 7, 4, 8)
        .reshape(batch, bricks * brick_t, rows * brick_h, geometry.grid[1] * brick_w, heads, head_dim)
    )
    return plane[:, : extent[0], : extent[1], : geometry.width]


def _with_null(slots: torch.Tensor, null_index: int) -> torch.Tensor:
    return torch.where(slots < 0, torch.full_like(slots, null_index), slots)


def _append_null(keys: torch.Tensor, values: torch.Tensor, head_dim: int) -> tuple[torch.Tensor, torch.Tensor]:
    shape = (keys.shape[0], keys.shape[1], 1, *keys.shape[3:])
    null_key = keys.new_zeros(shape)
    null_key[..., head_dim] = _DEAD
    null_value = values.new_zeros((*shape[:-1], values.shape[-1]))
    return torch.cat([keys, null_key], dim=2), torch.cat([values, null_value], dim=2)


def _slot_runs(slots: torch.Tensor) -> list[tuple[int, int]]:
    rows = slots.tolist()
    runs: list[tuple[int, int]] = []
    start = 0
    for index in range(1, len(rows)):
        if rows[index] != rows[start]:
            runs.append((start, index))
            start = index
    runs.append((start, len(rows)))
    return runs


def _attend_group(
    query_slice: torch.Tensor,
    key_views: tuple[torch.Tensor, ...],
    value_views: tuple[torch.Tensor, ...],
    geometry: _Geometry,
    shape: tuple[int, int],
    mask: torch.Tensor,
) -> torch.Tensor:
    bricks, rows = shape
    batch = query_slice.shape[0]
    heads, head_dim = query_slice.shape[4], query_slice.shape[5]
    blocks = sum(view.shape[5] for view in key_views)
    channels = _key_channels(head_dim)
    keys = query_slice.new_empty((batch, bricks, rows, geometry.grid[1], heads, blocks, *geometry.span, channels))
    values = query_slice.new_empty((batch, bricks, rows, geometry.grid[1], heads, blocks, *geometry.span, head_dim))
    start = 0
    for key_view, value_view in zip(key_views, value_views, strict=True):
        stop = start + key_view.shape[5]
        keys[:, :, :, :, :, start:stop].copy_(key_view)
        values[:, :, :, :, :, start:stop].copy_(value_view)
        start = stop
    count = batch * bricks * rows * geometry.grid[1]
    attended = F.scaled_dot_product_attention(
        _query_bricks(query_slice, geometry, bricks, rows),
        keys.view(count, heads, blocks * geometry.footprint, channels),
        values.view(count, heads, blocks * geometry.footprint, head_dim),
        attn_mask=mask,
        scale=1.0,
    )
    return _unbrick(attended, geometry, batch, bricks, rows, (query_slice.shape[1], query_slice.shape[2]))


def _row_groups(geometry: _Geometry, schedule: _Schedule) -> list[tuple[int, int, slice, slice]]:
    brick_h = geometry.brick[1]
    groups = []
    for row in range(0, geometry.grid[0], schedule.group_rows):
        rows = min(schedule.group_rows, geometry.grid[0] - row)
        groups.append(
            (
                row,
                rows,
                slice(row * brick_h, row * brick_h + geometry.row_extent(rows)),
                slice(row * brick_h, min((row + rows) * brick_h, geometry.height)),
            )
        )
    return groups


def _video_query_pass(
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    keyframe_k: torch.Tensor,
    keyframe_v: torch.Tensor,
    slots: torch.Tensor,
    geometry: _Geometry,
    workspace_bytes: int,
    factor: float,
) -> torch.Tensor:
    time, heads, head_dim = q.shape[1], q.shape[4], q.shape[5]
    brick_t = geometry.brick[0]
    lo_t, hi_t = geometry.pad_t
    num_slots = slots.shape[1]
    blocks = geometry.span_t + num_slots

    plane_keys, plane_values = _append_null(
        _stage(keyframe_k, geometry, (0, 0), with_bias_channel=True),
        _stage(keyframe_v, geometry, (0, 0), with_bias_channel=False),
        head_dim,
    )
    slot_table = _with_null(slots, keyframe_k.shape[1])
    mask = _joint_mask(geometry, num_slots, q.device)
    schedule = _Schedule(
        geometry,
        blocks,
        heads,
        head_dim,
        -(-time // brick_t),
        q.element_size(),
        workspace_bytes,
        factor,
    )
    rows_groups = _row_groups(geometry, schedule)

    out = torch.empty_like(q)
    for run_start, run_stop in _slot_runs(slot_table):
        # One plane gather per run: every brick inside it sees the same slots.
        planes = plane_keys.index_select(2, slot_table[run_start])
        plane_vals = plane_values.index_select(2, slot_table[run_start])
        run_bricks = -(-(run_stop - run_start) // brick_t)
        for staged_brick in range(0, run_bricks, schedule.stage_axis):
            staged_bricks = min(schedule.stage_axis, run_bricks - staged_brick)
            first = run_start + staged_brick * brick_t
            last = first + staged_bricks * brick_t  # exclusive; may reach past the run or T
            source = slice(max(0, first - lo_t), min(time, last + hi_t))
            pad_t = (max(0, lo_t - first), max(0, last + hi_t - time))
            window_keys = _stage(k[:, source], geometry, pad_t, with_bias_channel=True)
            window_values = _stage(v[:, source], geometry, pad_t, with_bias_channel=False)

            for brick in range(staged_brick, staged_brick + staged_bricks, schedule.group_axis):
                count = min(schedule.group_axis, staged_brick + staged_bricks - brick)
                start = run_start + brick * brick_t
                stop = min(start + count * brick_t, run_stop)
                offset = (brick - staged_brick) * brick_t
                for _, rows, key_rows, out_rows in rows_groups:
                    tile = _attend_group(
                        q[:, start:stop, out_rows],
                        (
                            _slabs(
                                window_keys[:, :, offset:, key_rows],
                                geometry,
                                count,
                                rows,
                                geometry.span_t,
                                group_stride=brick_t,
                            ),
                            _slabs(planes[:, :, :, key_rows], geometry, count, rows, num_slots, group_stride=0),
                        ),
                        (
                            _slabs(
                                window_values[:, :, offset:, key_rows],
                                geometry,
                                count,
                                rows,
                                geometry.span_t,
                                group_stride=brick_t,
                            ),
                            _slabs(plane_vals[:, :, :, key_rows], geometry, count, rows, num_slots, group_stride=0),
                        ),
                        geometry,
                        (count, rows),
                        mask,
                    )
                    out[:, start:stop, out_rows] = tile
    return out


def _keyframe_query_pass(
    keyframe_q: torch.Tensor,
    keyframe_k: torch.Tensor,
    keyframe_v: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    slots: torch.Tensor,
    keyframe_valid: torch.Tensor,
    geometry: _Geometry,
    workspace_bytes: int,
    factor: float,
) -> torch.Tensor:
    planes_total, heads, head_dim = keyframe_q.shape[1], keyframe_q.shape[4], keyframe_q.shape[5]
    num_slots = slots.shape[1]
    blocks = 1 + num_slots
    time = k.shape[1]
    flat = _Geometry(geometry.height, geometry.width, (1, *geometry.kernel[1:]), (1, *geometry.brick[1:]))

    # Only the frames some plane actually points at get staged -- at most ``P * num_slots`` of them,
    # against the whole volume if this staged ``k`` wholesale. ``unique`` doubles as the remap: slot
    # rows are rewritten to index the compacted stack.
    wanted, inverse = torch.unique(_with_null(slots, time).reshape(-1), return_inverse=True)
    frame_keys = _stage(k.index_select(1, wanted.clamp(max=time - 1)), flat, (0, 0), with_bias_channel=True)
    frame_values = _stage(v.index_select(1, wanted.clamp(max=time - 1)), flat, (0, 0), with_bias_channel=False)
    # An empty slot clamped onto a real frame above; kill it here instead of appending a null row.
    frame_keys[:, :, wanted == time, ..., head_dim] = _DEAD
    own_keys = _stage(keyframe_k, flat, (0, 0), with_bias_channel=True)
    own_values = _stage(keyframe_v, flat, (0, 0), with_bias_channel=False)
    own_keys[:, :, ~keyframe_valid, ..., head_dim] = _DEAD
    slot_table = inverse.reshape(planes_total, num_slots)
    mask = _joint_mask(flat, num_slots, keyframe_q.device)
    schedule = _Schedule(
        flat,
        blocks,
        heads,
        head_dim,
        planes_total,
        keyframe_q.element_size(),
        workspace_bytes,
        factor,
    )
    rows_groups = _row_groups(flat, schedule)

    out = torch.empty_like(keyframe_q)
    for start in range(0, planes_total, schedule.group_axis):
        stop = min(start + schedule.group_axis, planes_total)
        count = stop - start
        picked = slot_table[start:stop].reshape(-1)
        frames = frame_keys.index_select(2, picked)
        frame_vals = frame_values.index_select(2, picked)
        for _, rows, key_rows, out_rows in rows_groups:
            tile = _attend_group(
                keyframe_q[:, start:stop, out_rows],
                (
                    _slabs(own_keys[:, :, start:, key_rows], flat, count, rows, 1, group_stride=1),
                    _slabs(frames[:, :, :, key_rows], flat, count, rows, num_slots, group_stride=num_slots),
                ),
                (
                    _slabs(own_values[:, :, start:, key_rows], flat, count, rows, 1, group_stride=1),
                    _slabs(frame_vals[:, :, :, key_rows], flat, count, rows, num_slots, group_stride=num_slots),
                ),
                flat,
                (count, rows),
                mask,
            )
            out[:, start:stop, out_rows] = tile
    # An invalid plane sees nothing; zero it rather than shipping the uniform mean.
    return out * keyframe_valid[None, :, None, None, None, None]


def joint_na3d(  # noqa: PLR0913
    q: torch.Tensor,
    k: torch.Tensor,
    v: torch.Tensor,
    keyframe_q: torch.Tensor,
    keyframe_k: torch.Tensor,
    keyframe_v: torch.Tensor,
    keyframe_times: torch.Tensor,
    keyframe_valid: torch.Tensor,
    kernel_size: tuple[int, int, int],
    num_slots: int = KEYFRAME_CONTEXT_SLOTS,
    brick: tuple[int, int, int] | None = None,
    workspace_bytes: int = DEFAULT_WORKSPACE_BYTES,
    factor: float | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    time, height, width = q.shape[1], q.shape[2], q.shape[3]
    video_slots = video_keyframe_slots(keyframe_times, keyframe_valid, time, num_slots)
    keyframe_slots = keyframe_video_slots(keyframe_times, keyframe_valid, time, num_slots)
    geometry = _Geometry(height, width, kernel_size, brick if brick is not None else pick_brick(time, height, width))
    if factor is None:
        factor = staging_factor(q.device)
    return (
        _video_query_pass(q, k, v, keyframe_k, keyframe_v, video_slots, geometry, workspace_bytes, factor),
        _keyframe_query_pass(
            keyframe_q,
            keyframe_k,
            keyframe_v,
            k,
            v,
            keyframe_slots,
            keyframe_valid,
            geometry,
            workspace_bytes,
            factor,
        ),
    )


class EagerNAAttention:
    def __call__(self, attn, q, k, v):
        return na3d(q, k, v, kernel_size=attn.kernel_size, scale=1.0)


class EagerJointNAAttention:
    def __call__(self, attn, q, k, v, keyframe_q, keyframe_k, keyframe_v, keyframe_times, keyframe_valid):
        return joint_na3d(
            q, k, v, keyframe_q, keyframe_k, keyframe_v,
            keyframe_times, keyframe_valid, kernel_size=attn.kernel_size,
        )


"""3D Neighborhood Attention via NATTEN + absolute RoPE prelude.
Parameter shell shared by det ``NABlock`` and both diff-attn roles.
Diffusion AdaLN residuals live in pathway packages (each owns its RoPE);
det stages use ``det_attn_rope`` from :meth:`NeighborhoodAttention3D.forward`.
``attention_function`` selects the NA backend (NATTEN, Triton/eager fallback, or CuTe DSL).
"""


if TYPE_CHECKING:
    from ltx_core.model.video_vae.keyframes import KeyframeStream


try:
    import natten

    _NATTEN_AVAILABLE = True
except ImportError:  # pragma: no cover
    natten = None  # type: ignore[assignment]
    _NATTEN_AVAILABLE = False


class NAAttentionCallable(Protocol):
    def __call__(
        self,
        attn: NeighborhoodAttention3D,
        q: torch.Tensor,
        k: torch.Tensor,
        v: torch.Tensor,
    ) -> torch.Tensor: ...


class JointNAAttentionCallable(Protocol):
    def __call__(
        self,
        attn: NeighborhoodAttention3D,
        q: torch.Tensor,
        k: torch.Tensor,
        v: torch.Tensor,
        keyframe_q: torch.Tensor,
        keyframe_k: torch.Tensor,
        keyframe_v: torch.Tensor,
        keyframe_times: torch.Tensor,
        keyframe_valid: torch.Tensor,
    ) -> tuple[torch.Tensor, torch.Tensor]: ...


class NeighborhoodAttention3D(nn.Module):
    def __init__(
        self,
        dim: int,
        kernel_size: tuple[int, int, int],
        head_dim: int = 64,
        rope_dim_split: tuple[int, int, int] | None = None,
        rope_base: float = 10000.0,
    ) -> None:
        super().__init__()
        assert dim % head_dim == 0, f"dim={dim} not divisible by head_dim={head_dim}"
        self.dim = dim
        self.num_heads = dim // head_dim
        self.head_dim = head_dim
        self.kernel_size = tuple(kernel_size)
        self.scale = head_dim**-0.5

        if rope_dim_split is None:
            rope_dim_split = default_rope_dim_split(head_dim)
        assert sum(rope_dim_split) == head_dim, f"rope_dim_split={rope_dim_split} must sum to head_dim={head_dim}"
        self.rope_dim_split = rope_dim_split
        self.rope_base = rope_base
        self.rope_num_tiles = DEFAULT_ABS_ROPE_NUM_TILES
        self.rope_compute_dtype = torch.float32
        # Kept for the chunked opaque residual (string arg); callable is the swap surface.
        self.natten_backend: str | None = None
        self.attention_function: NAAttentionCallable = EagerNAAttention()
        # Separate slot, installed for every mode; never NATTEN/DSL. Only the keyframe
        # decode path reads it, so keyframe-less decode keeps NATTEN when it is installed.
        self.joint_attention_function: JointNAAttentionCallable | None = EagerJointNAAttention()

        self.register_buffer("rope_inv_t", rope_inv_freqs(rope_dim_split[0], rope_base), persistent=False)
        self.register_buffer("rope_inv_h", rope_inv_freqs(rope_dim_split[1], rope_base), persistent=False)
        self.register_buffer("rope_inv_w", rope_inv_freqs(rope_dim_split[2], rope_base), persistent=False)

        self.qkv = QKVProjections(dim)
        self.proj = nn.Linear(dim, dim, bias=True)
        self.q_norm = nn.RMSNorm(head_dim, eps=1e-6)
        self.k_norm = nn.RMSNorm(head_dim, eps=1e-6)

        # W-chunking configuration (consumed by ``chunked.attn``).
        self.w_chunks = 1  # 1 = no chunking

    def project_qkv(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        batch, t, h, w, _ = x.shape
        q, k, v = self.qkv(x)
        shape = (batch, t, h, w, self.num_heads, self.head_dim)
        return q.view(shape), k.view(shape), v.view(shape)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        batch, t, h, w, _ = x.shape
        kt, kh, kw = self.kernel_size
        if t < kt or h < kh or w < kw:
            raise ValueError(
                f"3D neighborhood attention requires spatial dims >= kernel_size; "
                f"got (T,H,W)=({t},{h},{w}) vs kernel={self.kernel_size}"
            )

        q, k, v = det_qkv_rope(self, x)
        # natten's CUTLASS kernel silently produces wrong output if inputs are non-contiguous.
        q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
        out = self.attention_function(self, q, k, v)
        out = out.reshape(batch, t, h, w, self.dim)
        return self.proj(out)

    def forward_with_keyframes(
        self,
        x: torch.Tensor,
        keyframes: KeyframeStream,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        if self.joint_attention_function is None:
            raise RuntimeError(
                "keyframe decode needs joint_attention_function installed; build the decoder "
                "through apply_diffvae_config / apply_diffvae_mode"
            )
        batch, t, h, w, _ = x.shape
        planes = keyframes.x.shape[1]

        q, k, v = det_qkv_rope(self, x)
        keyframe_q, keyframe_k, keyframe_v = det_qkv_rope_at_times(self, keyframes.x, keyframes.times)
        q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
        keyframe_q = keyframe_q.contiguous()
        keyframe_k = keyframe_k.contiguous()
        keyframe_v = keyframe_v.contiguous()

        out, keyframe_out = self.joint_attention_function(
            self,
            q,
            k,
            v,
            keyframe_q,
            keyframe_k,
            keyframe_v,
            keyframes.times,
            keyframes.valid,
        )
        out = self.proj(out.reshape(batch, t, h, w, self.dim))
        keyframe_out = self.proj(keyframe_out.reshape(batch, planes, h, w, self.dim))
        return out, dataclasses.replace(keyframes, x=keyframe_out)


"""NABlock and DiffusionNABlock parameter shells for DiffVAE.
Pathway subclasses live in ``chunked/`` and ``combined/``; ``apply`` installs
them via ``__class__`` swap (same pattern as ``Fp8CastLinear``). The shell owns
weights + shared AdaLN helpers only — no pathway forward.
"""


if TYPE_CHECKING:
    from ltx_core.model.video_vae.keyframes import KeyframeStream


__all__ = [
    "DiffusionNABlock",
    "NABlock",
]


class NABlock(nn.Module):
    def __init__(
        self,
        dim: int,
        kernel_size: tuple[int, int, int],
        head_dim: int = 64,
        mlp_ratio: float = 4.0,
        rope_dim_split: tuple[int, int, int] | None = None,
    ) -> None:
        super().__init__()
        self.norm1 = nn.RMSNorm(dim, eps=1e-6)
        self.attn = NeighborhoodAttention3D(dim, kernel_size, head_dim=head_dim, rope_dim_split=rope_dim_split)
        self.norm2 = nn.RMSNorm(dim, eps=1e-6)
        hidden = (int(dim * mlp_ratio) + 15) // 16 * 16
        self.mlp = SwiGLU(dim, hidden)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = x + self.attn(self.norm1(x))
        x = plain_mlp(x, self.mlp, self.norm2, self.mlp.tile)
        return x

    def forward_with_keyframes(
        self,
        x: torch.Tensor,
        keyframes: KeyframeStream,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        attn_out, keyframe_attn = self.attn.forward_with_keyframes(
            self.norm1(x),
            dataclasses.replace(keyframes, x=self.norm1(keyframes.x)),
        )
        x = x + attn_out
        keyframe_x = keyframes.x + keyframe_attn.x
        x = plain_mlp(x, self.mlp, self.norm2, self.mlp.tile)
        keyframe_x = plain_mlp(keyframe_x, self.mlp, self.norm2, self.mlp.tile)
        return x, dataclasses.replace(keyframes, x=keyframe_x)


class DiffusionNABlock(nn.Module):
    def __init__(
        self,
        dim: int,
        kernel_size: tuple[int, int, int],
        context_channels: int,
        head_dim: int = 64,
        mlp_ratio: float = 4.0,
        rope_dim_split: tuple[int, int, int] | None = None,
    ) -> None:
        super().__init__()
        self.context_channels = context_channels
        self.context_proj = nn.Linear(context_channels, dim, bias=True)
        self.scale_shift_table = nn.Parameter(torch.zeros(AdaLNZero.NUM_CHUNKS, dim))

        self.norm1 = nn.RMSNorm(dim, eps=1e-6)
        self.attn = NeighborhoodAttention3D(dim, kernel_size, head_dim=head_dim, rope_dim_split=rope_dim_split)
        self.norm2 = nn.RMSNorm(dim, eps=1e-6)
        hidden = (int(dim * mlp_ratio) + 15) // 16 * 16
        self.mlp = SwiGLU(dim, hidden)
        self.attn.proj.reset_parameters()

    def _modulation(
        self, modulation: tuple[torch.Tensor, ...]
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
        scale_shift_table = self.scale_shift_table.to(dtype=modulation[0].dtype, device=modulation[0].device)
        scale_msa, shift_msa, _, scale_mlp, shift_mlp, _, _ = [
            modulation[i] + scale_shift_table[i].view(1, 1, 1, 1, -1) for i in range(AdaLNZero.NUM_CHUNKS)
        ]
        return scale_msa, shift_msa, scale_mlp, shift_mlp


"""Combined context residual: project context half of ``context_and_x`` into ``x``."""


def combined(
    context_and_x: torch.Tensor,
    w_proj: torch.Tensor,
    b_proj: torch.Tensor | None,
) -> torch.Tensor:
    context_channels = w_proj.shape[1]
    latent_context = context_and_x[..., :context_channels]
    x = context_and_x[..., context_channels:]
    return x + F.linear(latent_context, w_proj, b_proj)


inject_context = combined


"""Combined* diffusion AdaLN residual attention (full-volume NA + nested RoPE).
Owns nested full-volume abs-RoPE for the Combined / ``w_chunks==1`` path.
Does not share a residual body with ``chunked`` — only the NA module weights.
"""


_rot_abs_axis = torch.compiler.nested_compile_region(rot_abs_axis_impl)


def _apply_nested_abs_rope_slab(
    x: torch.Tensor,
    rope_split: tuple[int, int, int],
    inv_freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
    *,
    w_pos: torch.Tensor,
    compute_dtype: torch.dtype,
    t_pos: torch.Tensor | None = None,
) -> torch.Tensor:
    d_t, d_h, _ = rope_split
    inv_t, inv_h, inv_w = inv_freqs
    t = x.shape[1]
    h = x.shape[2]
    positions_t = t_positions(t, x.device) if t_pos is None else t_pos
    xt = _rot_abs_axis(x[..., :d_t], positions_t, inv_t, axis=1, compute_dtype=compute_dtype)
    xh = _rot_abs_axis(
        x[..., d_t : d_t + d_h],
        h_positions(h, x.device),
        inv_h,
        axis=2,
        compute_dtype=compute_dtype,
    )
    xw = _rot_abs_axis(x[..., d_t + d_h :], w_pos, inv_w, axis=3, compute_dtype=compute_dtype)
    return torch.cat([xt, xh, xw], dim=-1)


def _apply_nested_full_volume_rope(
    x: torch.Tensor,
    rope_split: tuple[int, int, int],
    inv_freqs: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
    *,
    num_tiles: int,
    compute_dtype: torch.dtype,
    t_pos: torch.Tensor | None = None,
) -> torch.Tensor:
    slabs = torch.chunk(x, num_tiles, dim=3)
    w_off = 0
    parts: list[torch.Tensor] = []
    for slab in slabs:
        w_slab = slab.shape[3]
        w_pos = torch.arange(w_slab, dtype=torch.float32, device=x.device) + w_off
        parts.append(
            _apply_nested_abs_rope_slab(
                slab,
                rope_split,
                inv_freqs,
                w_pos=w_pos,
                compute_dtype=compute_dtype,
                t_pos=t_pos,
            )
        )
        w_off = w_off + w_slab
    return torch.cat(parts, dim=3)


def _qkv_nested_rope(
    attn: NeighborhoodAttention3D,
    x: torch.Tensor,
    t_pos: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    q, k, v = attn.project_qkv(x)
    q = attn.q_norm(q) * attn.scale
    k = attn.k_norm(k)
    inv_freqs = (
        attn.rope_inv_t.to(device=x.device),
        attn.rope_inv_h.to(device=x.device),
        attn.rope_inv_w.to(device=x.device),
    )
    positions = None if t_pos is None else t_pos.to(device=x.device, dtype=torch.float32)
    q = _apply_nested_full_volume_rope(
        q,
        attn.rope_dim_split,
        inv_freqs,
        num_tiles=attn.rope_num_tiles,
        compute_dtype=attn.rope_compute_dtype,
        t_pos=positions,
    )
    k = _apply_nested_full_volume_rope(
        k,
        attn.rope_dim_split,
        inv_freqs,
        num_tiles=attn.rope_num_tiles,
        compute_dtype=attn.rope_compute_dtype,
        t_pos=positions,
    )
    return q, k, v


def full_with_keyframes(
    x: torch.Tensor,
    keyframe_x: torch.Tensor,
    attn: NeighborhoodAttention3D,
    norm: nn.RMSNorm,
    scale: torch.Tensor,
    shift: torch.Tensor,
    keyframe_times: torch.Tensor,
    keyframe_valid: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    if attn.joint_attention_function is None:
        raise RuntimeError(
            "keyframe decode needs joint_attention_function installed; build the decoder "
            "through apply_diffvae_config / apply_diffvae_mode"
        )
    batch, t, h, w, _ = x.shape
    planes = keyframe_x.shape[1]

    y = norm(x) * (1.0 + scale) + shift
    keyframe_y = norm(keyframe_x) * (1.0 + scale) + shift

    q, k, v = _qkv_nested_rope(attn, y)
    keyframe_q, keyframe_k, keyframe_v = _qkv_nested_rope(attn, keyframe_y, keyframe_times)
    q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
    keyframe_q = keyframe_q.contiguous()
    keyframe_k = keyframe_k.contiguous()
    keyframe_v = keyframe_v.contiguous()

    out, keyframe_out = attn.joint_attention_function(
        attn,
        q,
        k,
        v,
        keyframe_q,
        keyframe_k,
        keyframe_v,
        keyframe_times,
        keyframe_valid,
    )
    x = x + attn.proj(out.reshape(batch, t, h, w, attn.dim))
    keyframe_x = keyframe_x + attn.proj(keyframe_out.reshape(batch, planes, h, w, attn.dim))
    return x, keyframe_x


def full(
    x: torch.Tensor,
    attn: NeighborhoodAttention3D,
    norm: nn.RMSNorm,
    scale: torch.Tensor,
    shift: torch.Tensor,
) -> torch.Tensor:
    y = norm(x) * (1.0 + scale) + shift
    batch, t, h, w, _ = y.shape
    kt, kh, kw = attn.kernel_size
    if t < kt or h < kh or w < kw:
        raise ValueError(
            f"3D neighborhood attention requires spatial dims >= kernel_size; "
            f"got (T,H,W)=({t},{h},{w}) vs kernel={attn.kernel_size}"
        )

    q, k, v = _qkv_nested_rope(attn, y)
    q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
    out = attn.attention_function(attn, q, k, v)
    out = out.reshape(batch, t, h, w, attn.dim)
    return x + attn.proj(out)


residual_attn = full
residual_attn_with_keyframes = full_with_keyframes


"""Combined pathway MLP: out-of-place AdaLN SwiGLU residual."""


def residual_mlp(
    x: torch.Tensor,
    mlp: nn.Module,
    norm: nn.RMSNorm,
    scale: torch.Tensor,
    shift: torch.Tensor,
    tile: SwiGLUTileSpec,
) -> torch.Tensor:
    y = modulate(norm(x), scale, shift)
    if y.numel() == 0:
        return x
    return x + swiglu_tiled(y, *swiglu_weights(mlp), tile)


"""CombinedDiffusionNABlock: context_and_x inject + full-volume attn + residual MLP."""


class CombinedDiffusionNABlock(DiffusionNABlock):
    def forward_combined_with_keyframes(
        self,
        context_and_x: torch.Tensor,
        keyframe_context_and_x: torch.Tensor,
        modulation: tuple[torch.Tensor, ...],
        keyframe_times: torch.Tensor,
        keyframe_valid: torch.Tensor,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        scale_msa, shift_msa, scale_mlp, shift_mlp = self._modulation(modulation)
        w_proj, b_proj = vram_ready_linear(self.context_proj)
        x = inject_context(context_and_x, w_proj, b_proj)
        keyframe_x = inject_context(keyframe_context_and_x, w_proj, b_proj)
        x, keyframe_x = residual_attn_with_keyframes(
            x,
            keyframe_x,
            self.attn,
            self.norm1,
            scale_msa,
            shift_msa,
            keyframe_times,
            keyframe_valid,
        )
        x = residual_mlp(x, self.mlp, self.norm2, scale_mlp, shift_mlp, self.mlp.tile)
        keyframe_x = residual_mlp(keyframe_x, self.mlp, self.norm2, scale_mlp, shift_mlp, self.mlp.tile)
        return x, keyframe_x * keyframe_valid[None, :, None, None, None]

    def forward_combined(
        self,
        context_and_x: torch.Tensor,
        modulation: tuple[torch.Tensor, ...],
    ) -> torch.Tensor:
        scale_msa, shift_msa, scale_mlp, shift_mlp = self._modulation(modulation)
        w_proj, b_proj = vram_ready_linear(self.context_proj)
        x = inject_context(context_and_x, w_proj, b_proj)
        x = residual_attn(x, self.attn, self.norm1, scale_msa, shift_msa)
        x = residual_mlp(x, self.mlp, self.norm2, scale_mlp, shift_mlp, self.mlp.tile)
        return x

    def forward(
        self,
        context_and_x: torch.Tensor,
        modulation: tuple[torch.Tensor, ...],
    ) -> torch.Tensor:
        return self.forward_combined(context_and_x, modulation)


"""Diffusion (NATTEN) video VAE decoder."""


logger: logging.Logger = logging.getLogger(__name__)


_L_STAGE_CHANNELS: Tuple[int, ...] = (1024, 512, 256, 256, 128)


_L_STAGE_DEPTHS: Tuple[int, ...] = (4, 6, 4, 2, 2)


_L_UPSAMPLES: Tuple[Tuple[Tuple[int, int, int], int], ...] = (
    ((1, 2, 2), 2),  # compress_space x2
    ((2, 1, 1), 2),  # compress_time x2
    ((2, 2, 2), 1),  # compress_all x1 (channel-preserving)
    ((2, 2, 2), 2),  # compress_all x2
)


_L_STAGE_KERNELS: Tuple[Tuple[int, int, int], ...] = (
    (3, 7, 7),
    (3, 7, 7),
    (3, 5, 5),
    (3, 5, 5),
    (3, 3, 3),
)


_DIFF_STAGE5_KERNEL_DEFAULT: Tuple[int, int, int] = (3, 7, 7)


_DIFF_STAGE5_DEPTH_DEFAULT: int = 8


_DIFF_STAGE_DEPTHS_DEFAULT: Tuple[int, ...] = (*_L_STAGE_DEPTHS[:-1], _DIFF_STAGE5_DEPTH_DEFAULT)


class DiffusionVideoDecoder(nn.Module, Disposable, VideoDecoder):
    def __init__(  # noqa: PLR0913
        self,
        in_channels: int = 128,
        out_channels: int = 3,
        patch_size: int = 4,
        head_dim: int = 64,
        rope_dim_split: Tuple[int, int, int] | None = None,
        stage_channels: Tuple[int, ...] = _L_STAGE_CHANNELS,
        stage_depths: Tuple[int, ...] = _DIFF_STAGE_DEPTHS_DEFAULT,
        stage_kernels: Tuple[Tuple[int, int, int], ...] = _L_STAGE_KERNELS,
        upsamples: Tuple[Tuple[Tuple[int, int, int], int], ...] = _L_UPSAMPLES,
        stage5_kernel: Tuple[int, int, int] = _DIFF_STAGE5_KERNEL_DEFAULT,
        stage5_channels: int | None = None,
        t_emb_dim: int = 384,
        default_num_inference_steps: int = 2,
        timestep_scale_multiplier: float = 1.0,
        model_output_type: Literal["v", "x0"] = "v",
    ) -> None:
        super().__init__()
        assert len(stage_channels) == len(stage_depths) == len(stage_kernels)
        assert len(upsamples) == len(stage_channels) - 1
        for c in stage_channels:
            assert c % head_dim == 0, f"stage_channels {stage_channels} must each be a multiple of head_dim={head_dim}"

        self.patch_size = patch_size
        self.register_buffer(
            "default_inference_timesteps",
            torch.linspace(1.0, 1.0 / default_num_inference_steps, default_num_inference_steps, device="cpu"),
            persistent=False,
        )
        self.out_channels = out_channels
        self.stage_channels = stage_channels
        self.stage_depths = stage_depths
        self.base_channels = stage_channels[-1]
        self.causal = False
        self.timestep_conditioning = True
        self.video_downscale_factors = SpatioTemporalScaleFactors.default()
        self.stage5_kernel: Tuple[int, int, int] = tuple(stage5_kernel)  # type: ignore[assignment]
        # NATTEN last-frame border workaround: replicate last latent frame
        # ``(K_t // 2) * 2`` times through stages 1-4, then crop the appendix
        # off context before stage 5 down to at least ``stage5_kernel[0]``.
        self._natten_trailing_pad_latent_frames = (stage_kernels[0][0] // 2) * 2

        # Encoder output is per-channel normalized; undo before conv_in (same as ConvVideoDecoder).
        self.per_channel_statistics = PerChannelStatistics(latent_channels=in_channels)

        self.conv_in = ChannelLinear(in_channels, stage_channels[0], bias=True)
        # Keyframe-stream tag, added to un-normalized keyframe latents before the shared
        # ``conv_in`` and nowhere else. It is the only keyframe-specific weight in the
        # whole feature. Checkpoints predating the keyframe training have no such key, so
        # ``video_decoder_sd_ops_for_checkpoint`` synthesizes zeros -- a missing key would
        # otherwise leave the parameter on the meta device under ``strict=False`` load.
        self.type_emb = nn.Parameter(torch.zeros(in_channels))

        self.det_stages = nn.ModuleList()
        self.upsamples = nn.ModuleList()
        n_det_stages = len(stage_channels) - 1
        for stage_i in range(n_det_stages):
            c = stage_channels[stage_i]
            depth = stage_depths[stage_i]
            kernel = stage_kernels[stage_i]
            self.det_stages.append(
                nn.ModuleList(
                    [
                        NABlock(dim=c, kernel_size=kernel, head_dim=head_dim, rope_dim_split=rope_dim_split)
                        for _ in range(depth)
                    ]
                )
            )
            stride, reduction = upsamples[stage_i]
            self.upsamples.append(
                LinearPixelShuffleUpsample(in_channels=c, stride=stride, out_channels_reduction_factor=reduction)
            )

        self.t_embedder = PixArtAlphaCombinedTimestepSizeEmbeddings(embedding_dim=t_emb_dim, size_emb_dim=0)

        c_ctx = stage_channels[-1]
        self.context_channels = c_ctx
        c5 = stage5_channels if stage5_channels is not None else c_ctx
        d5 = stage_depths[-1]
        assert c5 % head_dim == 0, f"stage5_channels {c5} must be a multiple of head_dim={head_dim}"
        noised_pixel_channels = out_channels * (patch_size**2)

        # Latent-grid floor so stages 1-3 (full volume) never undershoot NA.
        self.stage_min_tile_sizes: Tuple[int, int, int] = all_stages_min_tile_size(
            stage_kernels, upsamples, stage5_kernel
        )
        # Stage-4-input tile floor / overlap halos (only stages 4-5 are tiled).
        up3_stride = upsamples[3][0]
        self.tile_min_sizes: Tuple[int, int, int] = compute_tile_min_size(
            stage_kernels[3], stage5_kernel, up3_stride
        )
        self.tile_halos: Tuple[Tuple[int, int, int], Tuple[int, int, int]] = compute_tile_halos(
            stage_kernels[3],
            stage_depths[3],
            stage5_kernel,
            stage_depths[-1],
            up3_stride,
        )
        self.conv_in_x_t = ChannelLinear(noised_pixel_channels, c5, bias=True)

        # Shared AdaLN-Zero (7-chunk for shape compat; gate slots unused in block).
        self.shared_adaln = AdaLNZero(dim=c5, t_emb_dim=t_emb_dim)

        self.diff_blocks = nn.ModuleList(
            [
                CombinedDiffusionNABlock(
                    dim=c5,
                    kernel_size=stage5_kernel,
                    context_channels=c_ctx,
                    head_dim=head_dim,
                    rope_dim_split=rope_dim_split,
                )
                for _ in range(d5)
            ]
        )

        self.norm_out = nn.RMSNorm(c5, eps=1e-6)
        self.conv_out = ChannelLinear(c5, noised_pixel_channels, bias=True)

        self.timestep_scale_multiplier = timestep_scale_multiplier
        self.model_output_type = model_output_type
        # Set True by ``compile_diffusion_decoder`` so decode marks T/H/W dynamic.
        self.mark_dynamic_shapes = False
        # When True, skip stage-4 upsample and inject via deferred sequential upsample+proj.
        # Default False = combined pathway (``CombinedDiffusionNABlock``). Chunked DiffVAE
        # modes flip this via ``apply_diffvae_config``.
        self.deferred_stage4_upsample = False
        # Remaining temporal upsampling per stage input, plus 1 for stage 5: the divisor in
        # ``keyframe_stage_times``. (8, 8, 4, 2, 1) for the production ladder.
        self._keyframe_time_strides: Tuple[int, ...] = remaining_time_strides(self.upsamples)

    def _run_det_stage(self, x: torch.Tensor, stage_i: int, drop_leading_frame: bool) -> torch.Tensor:
        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(x, dim)
        for block in self.det_stages[stage_i]:
            x = block(x)
        return self.upsamples[stage_i](x, drop_leading_frame=drop_leading_frame)

    def forward_stages_1_to_3(
        self,
        z_noisy: torch.Tensor,
        drop_leading_frame: bool = True,
    ) -> torch.Tensor:
        z_noisy = self.per_channel_statistics.un_normalize(z_noisy)
        x = z_noisy.permute(0, 2, 3, 4, 1)
        x = self.conv_in(x)
        for stage_i in range(3):
            x = self._run_det_stage(x, stage_i, drop_leading_frame)
        return x

    def _keyframe_stream_from_latents(
        self,
        keyframes: DecodeKeyframes,
        *,
        valid: torch.Tensor | None = None,
    ) -> KeyframeStream:
        latents = self.per_channel_statistics.un_normalize(keyframes.latents)
        x = latents.permute(0, 2, 3, 4, 1)
        x = x + self.type_emb.to(dtype=x.dtype, device=x.device).view(1, 1, 1, 1, -1)
        x = self.conv_in(x)
        planes = x.shape[1]
        if valid is None:
            valid = torch.ones(planes, dtype=torch.bool, device=x.device)
        times = keyframe_clip_times(
            keyframes.pixel_frame_indices,
            self._keyframe_time_strides[0],
            keyframes.clip_start_frame,
        )
        return KeyframeStream(x=x, times=times.to(device=x.device), valid=valid.to(device=x.device)).masked()

    def _run_det_stage_with_keyframes(
        self,
        x: torch.Tensor,
        keyframes: KeyframeStream,
        stage_i: int,
        drop_leading_frame: bool,
        pixel_frame_indices: torch.Tensor,
        next_time_origin: float,
        clip_start_frame: int = 0,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(x, dim)
        for block in self.det_stages[stage_i]:
            x, keyframes = block.forward_with_keyframes(x, keyframes)
        x = self.upsamples[stage_i](x, drop_leading_frame=drop_leading_frame)
        keyframe_x = upsample_keyframe_planes(self.upsamples[stage_i], keyframes.x)
        next_times = keyframe_clip_times(
            pixel_frame_indices,
            self._keyframe_time_strides[stage_i + 1],
            clip_start_frame,
            extra_origin=next_time_origin,
        )
        return x, KeyframeStream(
            x=keyframe_x,
            times=next_times.to(device=keyframe_x.device),
            valid=keyframes.valid,
        ).masked()

    def forward_stages_1_to_3_with_keyframes(
        self,
        z_noisy: torch.Tensor,
        keyframes: DecodeKeyframes,
        drop_leading_frame: bool = True,
        *,
        keyframe_valid: torch.Tensor | None = None,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        keyframes.validate()
        if z_noisy.shape[-2:] != keyframes.latents.shape[-2:]:
            raise ValueError(
                f"keyframe latents must share the video latent's H/W (identical padding), got "
                f"{tuple(keyframes.latents.shape[-2:])} vs {tuple(z_noisy.shape[-2:])}"
            )
        stream = self._keyframe_stream_from_latents(keyframes, valid=keyframe_valid)
        x = self.per_channel_statistics.un_normalize(z_noisy).permute(0, 2, 3, 4, 1)
        x = self.conv_in(x)
        for stage_i in range(3):
            x, stream = self._run_det_stage_with_keyframes(
                x,
                stream,
                stage_i,
                drop_leading_frame,
                keyframes.pixel_frame_indices,
                0.0,
                clip_start_frame=keyframes.clip_start_frame,
            )
        return x, stream

    def forward_stage_4_with_keyframes(
        self,
        x: torch.Tensor,
        keyframes: KeyframeStream,
        pixel_frame_indices: torch.Tensor,
        drop_leading_frame: bool = True,
        pad_trailing: bool = True,
        *,
        stage4_time_origin: float = 0.0,
        pixel_time_origin: float = 0.0,
        clip_start_frame: int = 0,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        # Rebuild from global indices rather than trusting the caller's stream: stages 1-3 of a
        # full-clip decode are global, and Dist has already folded clip_start into clip times.
        keyframes = dataclasses.replace(
            keyframes,
            times=keyframe_clip_times(
                pixel_frame_indices,
                self._keyframe_time_strides[3],
                clip_start_frame,
                extra_origin=stage4_time_origin,
            ).to(device=keyframes.x.device),
        )
        if self.deferred_stage4_upsample:
            return self._forward_stage_4_deferred_with_keyframes(
                x, keyframes, pixel_frame_indices, pad_trailing, pixel_time_origin, clip_start_frame
            )
        x, keyframes = self._run_det_stage_with_keyframes(
            x,
            keyframes,
            3,
            drop_leading_frame,
            pixel_frame_indices,
            pixel_time_origin,
            clip_start_frame=clip_start_frame,
        )
        if pad_trailing:
            x = crop_trailing_context_natten_pad(
                x,
                n_latent_frames=self._natten_trailing_pad_latent_frames,
                time_scale=self.video_downscale_factors.time,
                stage5_kernel_t=self.stage5_kernel[0],
            )
        return x, keyframes

    def _forward_stage_4_deferred_with_keyframes(
        self,
        x: torch.Tensor,
        keyframes: KeyframeStream,
        pixel_frame_indices: torch.Tensor,
        pad_trailing: bool,
        pixel_time_origin: float,
        clip_start_frame: int = 0,
    ) -> tuple[torch.Tensor, KeyframeStream]:
        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(x, dim)
        for block in self.det_stages[3]:
            x, keyframes = block.forward_with_keyframes(x, keyframes)
        if pad_trailing:
            up_t = int(self.upsamples[3].stride[0])
            x = crop_trailing_context_natten_pad(
                x,
                n_latent_frames=self._natten_trailing_pad_latent_frames,
                time_scale=self.video_downscale_factors.time // up_t,
                stage5_kernel_t=max(1, -(-self.stage5_kernel[0] // up_t)),
            )
        stage5_times = keyframe_clip_times(
            pixel_frame_indices,
            self._keyframe_time_strides[4],
            clip_start_frame,
            extra_origin=pixel_time_origin,
        )
        return x, KeyframeStream(
            x=keyframes.x,
            times=stage5_times.to(device=keyframes.x.device),
            valid=keyframes.valid,
        ).masked()

    def forward_stage_4(
        self,
        x: torch.Tensor,
        drop_leading_frame: bool = True,
        pad_trailing: bool = True,
    ) -> torch.Tensor:
        if self.deferred_stage4_upsample:
            if self.mark_dynamic_shapes:
                for dim in (1, 2, 3):
                    torch._dynamo.mark_dynamic(x, dim)
            for block in self.det_stages[3]:
                x = block(x)
            if pad_trailing:
                up_t = int(self.upsamples[3].stride[0])
                x = crop_trailing_context_natten_pad(
                    x,
                    n_latent_frames=self._natten_trailing_pad_latent_frames,
                    time_scale=self.video_downscale_factors.time // up_t,
                    stage5_kernel_t=max(1, -(-self.stage5_kernel[0] // up_t)),
                )
            return x

        x = self._run_det_stage(x, 3, drop_leading_frame)
        if pad_trailing:
            x = crop_trailing_context_natten_pad(
                x,
                n_latent_frames=self._natten_trailing_pad_latent_frames,
                time_scale=self.video_downscale_factors.time,
                stage5_kernel_t=self.stage5_kernel[0],
            )
        return x

    def _context_and_x_for_diff_step(self, context: torch.Tensor, x_t: torch.Tensor) -> torch.Tensor:
        noised_pixels_patched = patchify(x_t, patch_size_hw=self.patch_size, patch_size_t=1)
        x = self.conv_in_x_t(noised_pixels_patched.permute(0, 2, 3, 4, 1))
        return torch.cat([context, x], dim=-1)

    def _keyframe_context_and_x_for_diff_step(
        self,
        keyframe_context: torch.Tensor,
        keyframe_x_t: torch.Tensor,
        keyframe_valid: torch.Tensor,
    ) -> torch.Tensor:
        patched = patchify(keyframe_x_t, patch_size_hw=self.patch_size, patch_size_t=1)
        x = self.conv_in_x_t(patched.permute(0, 2, 3, 4, 1))
        x = x * keyframe_valid[None, :, None, None, None]
        return torch.cat([keyframe_context, x], dim=-1)

    def _x_for_diff_step(self, x_t: torch.Tensor) -> torch.Tensor:
        noised_pixels_patched = patchify(x_t, patch_size_hw=self.patch_size, patch_size_t=1)
        return self.conv_in_x_t(noised_pixels_patched.permute(0, 2, 3, 4, 1))

    def _keyframe_x_for_diff_step(self, keyframe_x_t: torch.Tensor, keyframe_valid: torch.Tensor) -> torch.Tensor:
        patched = patchify(keyframe_x_t, patch_size_hw=self.patch_size, patch_size_t=1)
        x = self.conv_in_x_t(patched.permute(0, 2, 3, 4, 1))
        return (x * keyframe_valid[None, :, None, None, None]).contiguous()

    def forward_diff_step(
        self,
        context_and_x: torch.Tensor,
        t: torch.Tensor,
    ) -> torch.Tensor:
        x_half = context_and_x[..., self.context_channels :]
        t_emb = self.t_embedder(self.timestep_scale_multiplier * t, hidden_dtype=x_half.dtype)
        modulation = self.shared_adaln(t_emb)

        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(context_and_x, dim)

        for block in self.diff_blocks:
            x_half.copy_(block.forward_combined(context_and_x, modulation))
        return self._pixels_from_stage5(x_half)

    def forward_diff_step_with_keyframes(
        self,
        context_and_x: torch.Tensor,
        keyframe_context_and_x: torch.Tensor,
        t: torch.Tensor,
        keyframe_times: torch.Tensor,
        keyframe_valid: torch.Tensor,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        x_half = context_and_x[..., self.context_channels :]
        keyframe_half = keyframe_context_and_x[..., self.context_channels :]
        t_emb = self.t_embedder(self.timestep_scale_multiplier * t, hidden_dtype=x_half.dtype)
        modulation = self.shared_adaln(t_emb)

        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(context_and_x, dim)
            # Keyframe dim 1 is the plane count, not T, and keyframe_times / keyframe_valid pin it.
            # Marking it dynamic and then specializing to P raises ConstraintViolationError.
            for dim in (2, 3):
                torch._dynamo.mark_dynamic(keyframe_context_and_x, dim)

        for block in self.diff_blocks:
            x_out, keyframe_out = block.forward_combined_with_keyframes(
                context_and_x,
                keyframe_context_and_x,
                modulation,
                keyframe_times,
                keyframe_valid,
            )
            x_half.copy_(x_out)
            keyframe_half.copy_(keyframe_out)

        return self._pixels_from_stage5(x_half), self._pixels_from_stage5(keyframe_half)

    def _pixels_from_stage5(self, x: torch.Tensor) -> torch.Tensor:
        x = self.norm_out(x)
        x = self.conv_out(x)
        x = x.permute(0, 4, 1, 2, 3).contiguous()
        return unpatchify(x, patch_size_hw=self.patch_size, patch_size_t=1)

    def forward_diff_step_deferred(
        self,
        x: torch.Tensor,
        stage4_feat: torch.Tensor,
        t: torch.Tensor,
        *,
        drop_leading_frame: bool = True,
    ) -> torch.Tensor:
        t_emb = self.t_embedder(self.timestep_scale_multiplier * t, hidden_dtype=x.dtype)
        modulation = self.shared_adaln(t_emb)

        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(x, dim)
                torch._dynamo.mark_dynamic(stage4_feat, dim)

        from ltx_core.model.video_vae.transformer.dsl_kernels import DSLDiffusionBlockChain  # noqa: PLC0415

        if isinstance(self.diff_blocks, DSLDiffusionBlockChain):
            # Ping-pong fused launches; same deferred (x, stage4_feat) contract.
            x = self.diff_blocks(x, stage4_feat, modulation, drop_leading_frame=drop_leading_frame)
        else:
            for block in self.diff_blocks:
                x = block.forward_x_ctx(x, stage4_feat, modulation, drop_leading_frame=drop_leading_frame)

        x = self.norm_out(x)
        x = self.conv_out(x)
        x = x.permute(0, 4, 1, 2, 3).contiguous()
        return unpatchify(x, patch_size_hw=self.patch_size, patch_size_t=1)

    def forward_diff_step_deferred_with_keyframes(
        self,
        x: torch.Tensor,
        stage4_feat: torch.Tensor,
        keyframe_x: torch.Tensor,
        keyframe_stage4_feat: torch.Tensor,
        t: torch.Tensor,
        keyframe_times: torch.Tensor,
        keyframe_valid: torch.Tensor,
        *,
        drop_leading_frame: bool = True,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        t_emb = self.t_embedder(self.timestep_scale_multiplier * t, hidden_dtype=x.dtype)
        modulation = self.shared_adaln(t_emb)

        if self.mark_dynamic_shapes:
            for dim in (1, 2, 3):
                torch._dynamo.mark_dynamic(x, dim)
                torch._dynamo.mark_dynamic(stage4_feat, dim)
            # Plane count, not T -- see the note in the combined path above.
            for dim in (2, 3):
                torch._dynamo.mark_dynamic(keyframe_x, dim)
                torch._dynamo.mark_dynamic(keyframe_stage4_feat, dim)

        # The DSL chain drives itself, so both streams recycle output buffers instead of
        # allocating a volume per block per stream; the chunked blocks expose the same
        # ``forward_x_ctx_with_keyframes``, so the fallback loop drives those.
        from ltx_core.model.video_vae.transformer.dsl_kernels import DSLDiffusionBlockChain  # noqa: PLC0415

        if isinstance(self.diff_blocks, DSLDiffusionBlockChain):
            x, keyframe_x = self.diff_blocks.forward_x_ctx_with_keyframes(
                x,
                stage4_feat,
                keyframe_x,
                keyframe_stage4_feat,
                modulation,
                keyframe_times,
                keyframe_valid,
                drop_leading_frame=drop_leading_frame,
            )
        else:
            for block in self.diff_blocks:
                x, keyframe_x = block.forward_x_ctx_with_keyframes(
                    x,
                    stage4_feat,
                    keyframe_x,
                    keyframe_stage4_feat,
                    modulation,
                    keyframe_times,
                    keyframe_valid,
                    drop_leading_frame=drop_leading_frame,
                )

        return self._pixels_from_stage5(x), self._pixels_from_stage5(keyframe_x)

    def _euler_step(
        self, x_t: torch.Tensor, model_out: torch.Tensor, t_now: torch.Tensor, t_next: torch.Tensor
    ) -> torch.Tensor:
        compute_dtype = x_t.dtype
        dt = (t_now - t_next).view(-1, *([1] * (x_t.ndim - 1))).to(torch.float32)
        x_t_fp32 = x_t.to(torch.float32)
        v_pred = model_out if self.model_output_type == "v" else to_velocity(x_t_fp32, t_now, model_out)
        return (x_t_fp32 - dt * v_pred).to(compute_dtype)

    def _decode_one_tile(
        self,
        feat_tile: torch.Tensor,
        x_t_tile_init: torch.Tensor,
        *,
        is_origin: bool,
        timestep: torch.Tensor,
        pad_trailing: bool,
    ) -> torch.Tensor:
        context_tile = self.forward_stage_4(
            feat_tile,
            drop_leading_frame=is_origin,
            pad_trailing=pad_trailing,
        )

        x_t = x_t_tile_init
        _, num_steps = timestep.shape
        for i in range(num_steps - 1):
            t_now = timestep[:, i]
            t_next = timestep[:, i + 1]
            if self.deferred_stage4_upsample:
                x = self._x_for_diff_step(x_t)
                model_out = self.forward_diff_step_deferred(x, context_tile, t_now, drop_leading_frame=is_origin).to(
                    torch.float32
                )
            else:
                context_and_x = self._context_and_x_for_diff_step(context_tile, x_t)
                model_out = self.forward_diff_step(context_and_x, t_now).to(torch.float32)
            x_t = self._euler_step(x_t, model_out, t_now, t_next)

        t_now = timestep[:, -1]
        if self.deferred_stage4_upsample:
            x = self._x_for_diff_step(x_t)
            model_out = self.forward_diff_step_deferred(x, context_tile, t_now, drop_leading_frame=is_origin)
        else:
            context_and_x = self._context_and_x_for_diff_step(context_tile, x_t)
            model_out = self.forward_diff_step(context_and_x, t_now)
        if self.model_output_type == "x0":
            return model_out
        return self._euler_step(x_t, model_out.to(torch.float32), t_now, torch.zeros_like(t_now))

    def _stage5_canvas_from_context(
        self,
        context_tile: torch.Tensor,
        *,
        drop_leading_frame: bool,
    ) -> tuple[int, int, int]:
        t, h, w = context_tile.shape[1], context_tile.shape[2], context_tile.shape[3]
        if self.deferred_stage4_upsample:
            # Context is still pre-upsample, so this is the same geometry as a stage-4
            # input. Ghost crop already ran, so do not re-apply the kernel-T floor.
            return stage5_pixel_shape_from_stage4(
                t,
                h,
                w,
                upsample_stride=tuple(self.upsamples[3].stride),  # type: ignore[arg-type]
                patch_size=self.patch_size,
                stage5_kernel_t=self.stage5_kernel[0],
                drop_leading_frame=drop_leading_frame,
                pad_trailing=False,
            )
        return t, h * self.patch_size, w * self.patch_size

    def _decode_one_tile_with_keyframes(  # noqa: PLR0913
        self,
        feat_tile: torch.Tensor,
        keyframes: KeyframeStream,
        pixel_frame_indices: torch.Tensor,
        *,
        is_origin: bool,
        timestep: torch.Tensor,
        pad_trailing: bool,
        generator: torch.Generator | None,
        compute_dtype: torch.dtype,
        x_t_tile_init: torch.Tensor | None = None,
        stage4_time_origin: float = 0.0,
        pixel_time_origin: float = 0.0,
        clip_start_frame: int = 0,
    ) -> torch.Tensor:
        context_tile, keyframes = self.forward_stage_4_with_keyframes(
            feat_tile,
            keyframes,
            pixel_frame_indices,
            drop_leading_frame=is_origin,
            pad_trailing=pad_trailing,
            stage4_time_origin=stage4_time_origin,
            pixel_time_origin=pixel_time_origin,
            clip_start_frame=clip_start_frame,
        )

        batch = context_tile.shape[0]
        canvas_t, canvas_h, canvas_w = self._stage5_canvas_from_context(context_tile, drop_leading_frame=is_origin)
        randn_device = generator.device if generator is not None else feat_tile.device

        def _noise(frames: int) -> torch.Tensor:
            return torch.randn(
                (batch, self.out_channels, frames, canvas_h, canvas_w),
                dtype=compute_dtype,
                generator=generator,
                device=randn_device,
            ).to(feat_tile.device)

        x_t = _noise(canvas_t) if x_t_tile_init is None else x_t_tile_init
        keyframe_x_t = _noise(keyframes.num_planes)

        def _step(t_now: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
            if self.deferred_stage4_upsample:
                return self.forward_diff_step_deferred_with_keyframes(
                    self._x_for_diff_step(x_t),
                    context_tile,
                    self._keyframe_x_for_diff_step(keyframe_x_t, keyframes.valid),
                    keyframes.x,
                    t_now,
                    keyframes.times,
                    keyframes.valid,
                    drop_leading_frame=is_origin,
                )
            context_and_x = self._context_and_x_for_diff_step(context_tile, x_t)
            keyframe_context_and_x = self._keyframe_context_and_x_for_diff_step(
                keyframes.x, keyframe_x_t, keyframes.valid
            )
            return self.forward_diff_step_with_keyframes(
                context_and_x, keyframe_context_and_x, t_now, keyframes.times, keyframes.valid
            )

        _, num_steps = timestep.shape
        for i in range(num_steps - 1):
            t_now = timestep[:, i]
            t_next = timestep[:, i + 1]
            video_out, keyframe_out = _step(t_now)
            x_t = self._euler_step(x_t, video_out.to(torch.float32), t_now, t_next)
            keyframe_x_t = self._euler_step(keyframe_x_t, keyframe_out.to(torch.float32), t_now, t_next)

        t_now = timestep[:, -1]
        video_out, _ = _step(t_now)
        if self.model_output_type == "x0":
            return video_out
        return self._euler_step(x_t, video_out.to(torch.float32), t_now, torch.zeros_like(t_now))

    def _decode_temporal_group_isolated_with_keyframes(  # noqa: PLR0913
        self,
        tiles: List[Tile],
        feat_s4: torch.Tensor,
        stream: KeyframeStream,
        pixel_frame_indices: torch.Tensor,
        content_s4_frames: int,
        x_t_init: torch.Tensor | None,
        timestep: torch.Tensor,
        full_video_shape: VideoLatentShape,
        curr_temporal_slice: slice,
        generator: torch.Generator | None,
        *,
        complementary: bool,
        clip_start_frame: int = 0,
    ) -> Tuple[torch.Tensor, torch.Tensor | None]:
        group_temporal_len = curr_temporal_slice.stop - curr_temporal_slice.start
        group_shape = full_video_shape._replace(frames=group_temporal_len)
        full_torch_shape = full_video_shape.to_torch_shape()
        accum_dtype = torch.float16 if feat_s4.dtype == torch.bfloat16 else feat_s4.dtype
        buffer = torch.zeros(group_shape.to_torch_shape(), device=feat_s4.device, dtype=accum_dtype)
        weights: torch.Tensor | None = None if complementary else torch.zeros_like(buffer)
        local_temporal_slice = slice(0, group_temporal_len)

        compute_dtype = feat_s4.dtype
        up3_stride = tuple(self.upsamples[3].stride)

        for tile_index, tile in tqdm(enumerate(tiles), total=len(tiles), desc="DiffVAE decode"):
            feat_tile, is_origin, pad_trailing, content_thw = slice_stage4_tile(
                feat_s4, tile, content_frames=content_s4_frames
            )
            # Two origins at two scales -- see forward_stage_4_with_keyframes.
            stage4_origin = tile.in_coords[1].indices(content_s4_frames)[0]
            pixel_lo, pixel_hi, _ = tile.out_coords[2].indices(full_torch_shape[2])

            # ``out_coords`` are local to this latent. Dist slices keep global indices and
            # ``clip_start_frame`` as the origin, so a local ``[0, 72)`` still has to select
            # global ``[56, 127]``.
            keep = planes_for_tile(pixel_frame_indices, pixel_lo, pixel_hi - 1, clip_start_frame=clip_start_frame)
            if not bool(keep.any()):
                raise RuntimeError(
                    f"tile covering pixel frames [{pixel_lo + clip_start_frame}, "
                    f"{pixel_hi - 1 + clip_start_frame}] selected no keyframe planes "
                    f"out of {int(pixel_frame_indices.shape[0])}; planes_for_tile always keeps at least one"
                )
            tile_stream = stream.select_planes(keep.to(stream.valid.device)).crop_spatial(
                tile.in_coords[2], tile.in_coords[3]
            )
            # Not debug-only: the decode below needs this tile's plane positions.
            tile_indices = pixel_frame_indices[keep.to(pixel_frame_indices.device)]
            if logger.isEnabledFor(logging.INFO):
                logger.info(
                    "keyframe decode: tile %d/%d frames [%d, %d), stage-4 extent %dx%dx%d, %d of %d planes at %s",
                    tile_index + 1,
                    len(tiles),
                    pixel_lo,
                    pixel_hi,
                    feat_tile.shape[1],
                    feat_tile.shape[2],
                    feat_tile.shape[3],
                    tile_stream.num_planes,
                    int(pixel_frame_indices.shape[0]),
                    tile_indices.tolist(),
                )

            x_t_tile_init: torch.Tensor | None = None
            if x_t_init is not None:
                stage5_f, stage5_h, stage5_w = stage5_pixel_shape_from_stage4(
                    content_thw[0],
                    content_thw[1],
                    content_thw[2],
                    upsample_stride=up3_stride,  # type: ignore[arg-type]
                    patch_size=self.patch_size,
                    stage5_kernel_t=self.stage5_kernel[0],
                    drop_leading_frame=is_origin,
                    pad_trailing=pad_trailing,
                )
                # Same edge policy as the plain path: expand/crop the shared noise field
                # rather than drawing fresh noise, since NA mixes padded values inward.
                x_t_tile_init = x_t_init[tile.out_coords]
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 2, stage5_f, mode="repeat_last")
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 3, stage5_h, mode="symmetric")
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 4, stage5_w, mode="symmetric")

            pixel_tile = self._decode_one_tile_with_keyframes(
                feat_tile,
                tile_stream,
                tile_indices,
                is_origin=is_origin,
                timestep=timestep,
                pad_trailing=pad_trailing,
                generator=generator,
                compute_dtype=compute_dtype,
                x_t_tile_init=x_t_tile_init,
                stage4_time_origin=float(stage4_origin),
                pixel_time_origin=float(pixel_lo),
                clip_start_frame=clip_start_frame,
            )
            content_pixel_shape = pixel_tile_shape(full_torch_shape, tile.out_coords)
            pixel_tile = crop_pixels_to_content(
                pixel_tile,
                content_pixel_shape[2],
                content_pixel_shape[3],
                content_pixel_shape[4],
            ).to(buffer.dtype)

            masks = tuple(m.to(device=buffer.device, dtype=torch.float32) for m in tile.masks_1d)
            local_coords = (
                tile.out_coords[0],
                tile.out_coords[1],
                local_temporal_slice,
                tile.out_coords[3],
                tile.out_coords[4],
            )
            buffer[local_coords] += scale_by_masks_1d(pixel_tile, masks)
            if weights is not None:
                strength = torch.ones(pixel_tile.shape, device=buffer.device, dtype=buffer.dtype)
                weights[local_coords] += scale_by_masks_1d(strength, masks)

        return buffer, weights

    def _decode_groups_with_keyframes(  # noqa: PLR0913, PLR0915
        self,
        feat_s4: torch.Tensor,
        stream: KeyframeStream,
        pixel_frame_indices: torch.Tensor,
        latent: torch.Tensor,
        tiling_config: TilingConfig,
        timestep: torch.Tensor,
        generator: torch.Generator | None,
        *,
        content_pixel: VideoLatentShape,
        h_pad: AxisPad | None,
        w_pad: AxisPad | None,
        as_fhwc: bool,
        clip_start_frame: int = 0,
    ) -> Iterator[torch.Tensor]:
        full_video_shape = (
            VideoLatentShape.from_torch_shape(latent.shape)
            .upscale(self.video_downscale_factors)
            ._replace(channels=self.out_channels)
        )
        target_shape = full_video_shape.to_torch_shape()
        strides = [tuple(u.stride) for u in self.upsamples]
        s4_t, s4_h, s4_w = stage4_thw_from_latent(
            strides, latent.shape[2], latent.shape[3], latent.shape[4], drop_leading_frame=True
        )
        tiles = prepare_tile_schedule(
            torch.Size([latent.shape[0], latent.shape[1], s4_t, s4_h, s4_w]),
            tiling_config,
            upsample3_stride=tuple(self.upsamples[3].stride),  # type: ignore[arg-type]
            patch_size=self.patch_size,
            min_tile_size=self.tile_min_sizes,
            tile_halos=self.tile_halos,
        )
        complementary = masks_are_complementary(tiles, target_shape)
        groups = group_tiles_by_temporal_slice(tiles)
        group_slices = [slice(*group[0].out_coords[2].indices(target_shape[2])[:2]) for group in groups]

        single_step_x0 = timestep.shape[1] == 1 and self.model_output_type == "x0"
        x_t_init: torch.Tensor | None = None
        if not single_step_x0:
            randn_device = generator.device if generator is not None else latent.device
            x_t_init = torch.randn(
                tuple(target_shape), dtype=latent.dtype, generator=generator, device=randn_device
            ).to(latent.device)

        logger.info(
            "keyframe decode: %d tile(s) in %d temporal group(s), %d frames at %dx%d, %d planes",
            len(tiles),
            len(groups),
            content_pixel.frames,
            content_pixel.height,
            content_pixel.width,
            int(pixel_frame_indices.shape[0]),
        )

        scaled_h_pad = scale_axis_pad(h_pad, self.video_downscale_factors.height)
        scaled_w_pad = scale_axis_pad(w_pad, self.video_downscale_factors.width)
        overlap_stub: torch.Tensor | None = None
        overlap_stub_weights: torch.Tensor | None = None

        def _emit(buf: torch.Tensor, wts: torch.Tensor | None, global_start: int) -> torch.Tensor | None:
            if global_start >= content_pixel.frames or buf.shape[2] < 1:
                return None
            frames_keep = min(buf.shape[2], content_pixel.frames - global_start)
            if frames_keep < 1:
                return None
            chunk = buf[:, :, :frames_keep]
            if wts is not None:
                floor = _weight_floor(wts.dtype)
                chunk = chunk / wts[:, :, :frames_keep].clamp(min=floor)
            chunk = crop_pixels_to_content(
                chunk.to(latent.dtype),
                frames_keep,
                content_pixel.height,
                content_pixel.width,
                h_pad=scaled_h_pad,
                w_pad=scaled_w_pad,
            )
            return chunk[0].permute(1, 2, 3, 0).contiguous() if as_fhwc else chunk

        for group_index, group in enumerate(groups):
            curr_temporal_slice = group_slices[group_index]
            logger.info(
                "keyframe decode: group %d/%d, frames [%d, %d)",
                group_index + 1,
                len(groups),
                curr_temporal_slice.start,
                curr_temporal_slice.stop,
            )
            buffer, weights = self._decode_temporal_group_isolated_with_keyframes(
                group,
                feat_s4,
                stream,
                pixel_frame_indices,
                s4_t,
                x_t_init,
                timestep,
                full_video_shape,
                curr_temporal_slice,
                generator,
                complementary=complementary,
                clip_start_frame=clip_start_frame,
            )

            if overlap_stub is not None:
                overlap_len = int(overlap_stub.shape[2])
                if overlap_len > 0:
                    overlap_stub += buffer[:, :, :overlap_len]
                    buffer[:, :, :overlap_len] = overlap_stub
                    if not complementary:
                        assert overlap_stub_weights is not None
                        assert weights is not None
                        overlap_stub_weights += weights[:, :, :overlap_len]
                        weights[:, :, :overlap_len] = overlap_stub_weights
                overlap_stub = None
                overlap_stub_weights = None

            if group_index + 1 < len(groups):
                next_start = group_slices[group_index + 1].start
                exclusive_len = min(max(0, next_start - curr_temporal_slice.start), buffer.shape[2])
                emitted = _emit(
                    buffer[:, :, :exclusive_len],
                    None if weights is None else weights[:, :, :exclusive_len],
                    curr_temporal_slice.start,
                )
                if emitted is not None:
                    yield emitted
                # Retain only the trailing overlap for the next group's handoff.
                overlap_stub = buffer[:, :, exclusive_len:].clone()
                if not complementary:
                    assert weights is not None
                    overlap_stub_weights = weights[:, :, exclusive_len:].clone()
                del buffer, weights
            else:
                emitted = _emit(buffer, weights, curr_temporal_slice.start)
                if emitted is not None:
                    yield emitted

    def _decode_pixels_with_keyframes(
        self,
        latent: torch.Tensor,
        keyframes: DecodeKeyframes,
        tiling_config: TilingConfig | None = None,
        generator: torch.Generator | None = None,
        *,
        as_fhwc: bool = False,
    ) -> Iterator[torch.Tensor]:
        content_shape = VideoLatentShape.from_torch_shape(latent.shape)
        content_pixel = content_shape.upscale(self.video_downscale_factors)._replace(channels=self.out_channels)
        keyframes.validate(num_frames=content_pixel.frames)

        latent, (_t_pad, h_pad, w_pad) = ensure_min_latent_shape(latent, self.stage_min_tile_sizes)
        # Same spatial floor for the keyframe planes, with the plane axis pinned by a
        # temporal minimum of 1. The pad is symmetric, so padding only one stream would
        # offset every plane from the video by half of it.
        _min_t, min_h, min_w = self.stage_min_tile_sizes
        keyframe_latents, (_, keyframe_h_pad, keyframe_w_pad) = ensure_min_latent_shape(
            keyframes.latents, (1, min_h, min_w)
        )
        if (keyframe_h_pad, keyframe_w_pad) != (h_pad, w_pad):
            raise RuntimeError(
                f"keyframe spatial pad {(keyframe_h_pad, keyframe_w_pad)} != video pad {(h_pad, w_pad)}; "
                "the two streams must share one spatial origin"
            )
        padded_keyframes = dataclasses.replace(keyframes, latents=keyframe_latents)

        # Ghost pad is a temporal-border workaround for the video stream; keyframe planes have
        # no temporal extent to pad. The appendix is cropped off context before stage 5.
        latent_padded = pad_trailing_latent_for_natten_border(
            latent, self._natten_trailing_pad_latent_frames
        )
        feat_s4, stream = self.forward_stages_1_to_3_with_keyframes(
            latent_padded, padded_keyframes, drop_leading_frame=True
        )

        batch = latent.shape[0]
        timestep = self.default_inference_timesteps.to(latent.device).unsqueeze(0).expand(batch, -1)
        if tiling_config is not None:
            yield from self._decode_groups_with_keyframes(
                feat_s4,
                stream,
                keyframes.pixel_frame_indices,
                latent,
                tiling_config,
                timestep,
                generator,
                content_pixel=content_pixel,
                h_pad=h_pad,
                w_pad=w_pad,
                as_fhwc=as_fhwc,
                clip_start_frame=keyframes.clip_start_frame,
            )
            return

        logger.info("keyframe decode: untiled, %d frames, %d planes", content_pixel.frames, stream.num_planes)
        pixels = self._decode_one_tile_with_keyframes(
            feat_s4,
            stream,
            keyframes.pixel_frame_indices,
            is_origin=True,
            timestep=timestep,
            pad_trailing=True,
            generator=generator,
            compute_dtype=latent.dtype,
            clip_start_frame=keyframes.clip_start_frame,
        )
        pixels = crop_pixels_to_content(
            pixels,
            content_pixel.frames,
            content_pixel.height,
            content_pixel.width,
            h_pad=scale_axis_pad(h_pad, self.video_downscale_factors.height),
            w_pad=scale_axis_pad(w_pad, self.video_downscale_factors.width),
        ).to(latent.dtype)
        if as_fhwc:
            yield pixels[0].permute(1, 2, 3, 0).contiguous()
        else:
            yield pixels

    def _decode_video_with_keyframes(
        self,
        latent: torch.Tensor,
        keyframes: DecodeKeyframes,
        tiling_config: TilingConfig | None = None,
        generator: torch.Generator | None = None,
    ) -> Iterator[torch.Tensor]:
        def to_rgb(frames: torch.Tensor) -> torch.Tensor:
            return frames.add_(1).mul_(0.5).clamp_(0, 1)

        for chunk in self._decode_pixels_with_keyframes(
            latent, keyframes, tiling_config, generator=generator, as_fhwc=True
        ):
            yield to_rgb(chunk)

    def _decode_temporal_group_isolated(
        self,
        tiles: List[Tile],
        feat_s4: torch.Tensor,
        content_s4_frames: int,
        x_t_init: torch.Tensor | None,
        timestep: torch.Tensor,
        full_video_shape: VideoLatentShape,
        curr_temporal_slice: slice,
        generator: torch.Generator | None,
        *,
        complementary: bool,
    ) -> Tuple[torch.Tensor, torch.Tensor | None]:
        group_temporal_len = curr_temporal_slice.stop - curr_temporal_slice.start
        group_shape = full_video_shape._replace(frames=group_temporal_len)
        full_torch_shape = full_video_shape.to_torch_shape()
        accum_dtype = torch.float16 if feat_s4.dtype == torch.bfloat16 else feat_s4.dtype
        buffer = torch.zeros(group_shape.to_torch_shape(), device=feat_s4.device, dtype=accum_dtype)
        weights: torch.Tensor | None = None if complementary else torch.zeros_like(buffer)
        local_temporal_slice = slice(0, group_temporal_len)

        compute_dtype = feat_s4.dtype
        randn_device = generator.device if generator is not None else feat_s4.device
        up3_stride = tuple(self.upsamples[3].stride)

        for tile in tqdm(tiles, total=len(tiles), desc="DiffVAE decode"):
            feat_tile, is_origin, pad_trailing, content_thw = slice_stage4_tile(
                feat_s4, tile, content_frames=content_s4_frames
            )
            content_pixel_shape = pixel_tile_shape(full_torch_shape, tile.out_coords)
            stage5_f, stage5_h, stage5_w = stage5_pixel_shape_from_stage4(
                content_thw[0],
                content_thw[1],
                content_thw[2],
                upsample_stride=up3_stride,  # type: ignore[arg-type]
                patch_size=self.patch_size,
                stage5_kernel_t=self.stage5_kernel[0],
                drop_leading_frame=is_origin,
                pad_trailing=pad_trailing,
            )

            if x_t_init is None:
                x_t_tile_init = torch.randn(
                    (content_pixel_shape[0], content_pixel_shape[1], stage5_f, stage5_h, stage5_w),
                    dtype=compute_dtype,
                    generator=generator,
                    device=randn_device,
                ).to(feat_s4.device)
            else:
                # Expand/crop to stage-5 canvas with the same edge policy as latent
                # size-floor / ghost pad (not fresh noise - NA mixes padded values
                # into kept pixels near the boundary).
                x_t_tile_init = x_t_init[tile.out_coords]
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 2, stage5_f, mode="repeat_last")
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 3, stage5_h, mode="symmetric")
                x_t_tile_init, _ = resize_axis(x_t_tile_init, 4, stage5_w, mode="symmetric")

            pixel_tile = self._decode_one_tile(
                feat_tile,
                x_t_tile_init,
                is_origin=is_origin,
                timestep=timestep,
                pad_trailing=pad_trailing,
            )
            pixel_tile = crop_pixels_to_content(
                pixel_tile,
                content_pixel_shape[2],
                content_pixel_shape[3],
                content_pixel_shape[4],
            ).to(buffer.dtype)

            masks = tuple(m.to(device=buffer.device, dtype=torch.float32) for m in tile.masks_1d)
            local_coords = (
                tile.out_coords[0],
                tile.out_coords[1],
                local_temporal_slice,
                tile.out_coords[3],
                tile.out_coords[4],
            )
            buffer[local_coords] += scale_by_masks_1d(pixel_tile, masks)
            if weights is not None:
                strength = torch.ones(pixel_tile.shape, device=buffer.device, dtype=buffer.dtype)
                weights[local_coords] += scale_by_masks_1d(strength, masks)

        return buffer, weights

    def _decode_pixels(  # noqa: PLR0912, PLR0915
        self,
        latent: torch.Tensor,
        tiling_config: TilingConfig | None = None,
        generator: torch.Generator | None = None,
        *,
        as_fhwc: bool = False,
    ) -> Iterator[torch.Tensor]:
        content_shape = VideoLatentShape.from_torch_shape(latent.shape)
        content_pixel = content_shape.upscale(self.video_downscale_factors)._replace(channels=self.out_channels)

        latent, (_t_pad, h_pad, w_pad) = ensure_min_latent_shape(latent, self.stage_min_tile_sizes)
        spatial_scale = (self.video_downscale_factors.height, self.video_downscale_factors.width)
        work_shape = VideoLatentShape.from_torch_shape(latent.shape)
        full_video_shape = work_shape.upscale(self.video_downscale_factors)._replace(channels=self.out_channels)
        target_shape = full_video_shape.to_torch_shape()

        strides = [tuple(u.stride) for u in self.upsamples]
        s4_t, s4_h, s4_w = stage4_thw_from_latent(
            strides, latent.shape[2], latent.shape[3], latent.shape[4], drop_leading_frame=True
        )
        tiles = prepare_tile_schedule(
            torch.Size([latent.shape[0], latent.shape[1], s4_t, s4_h, s4_w]),
            tiling_config,
            upsample3_stride=tuple(self.upsamples[3].stride),  # type: ignore[arg-type]
            patch_size=self.patch_size,
            min_tile_size=self.tile_min_sizes,
            tile_halos=self.tile_halos,
        )

        latent_padded = pad_trailing_latent_for_natten_border(
            latent, self._natten_trailing_pad_latent_frames
        )
        if self.mark_dynamic_shapes:
            for dim in (2, 3, 4):
                torch._dynamo.mark_dynamic(latent_padded, dim)

        feat_s4 = self.forward_stages_1_to_3(latent_padded, drop_leading_frame=True)

        batch = latent.shape[0]
        timestep = self.default_inference_timesteps.to(latent.device).unsqueeze(0).expand(batch, -1)
        single_step_x0 = timestep.shape[1] == 1 and self.model_output_type == "x0"

        x_t_init: torch.Tensor | None = None
        if not single_step_x0:
            compute_dtype = latent.dtype
            randn_device = generator.device if generator is not None else latent.device
            x_t_init = torch.randn(
                tuple(target_shape), dtype=compute_dtype, generator=generator, device=randn_device
            ).to(latent.device)

        complementary = masks_are_complementary(tiles, target_shape)
        groups = group_tiles_by_temporal_slice(tiles)
        group_slices = [slice(*group[0].out_coords[2].indices(target_shape[2])[:2]) for group in groups]

        # Keep only the trailing temporal overlap of the previous group (not the full
        # chunk). Exclusive frames are yielded before the next group is decoded; the
        # consumer may still hold that emit while the next buffer is live (~2x tile).
        overlap_stub: torch.Tensor | None = None
        overlap_stub_weights: torch.Tensor | None = None

        def _finalize(buf: torch.Tensor, wts: torch.Tensor | None) -> torch.Tensor:
            if complementary:
                return buf.to(latent.dtype)
            assert wts is not None
            wts = wts.clamp(min=_weight_floor(wts.dtype))
            return (buf / wts).to(latent.dtype)

        def _narrow_content_cfhw(t: torch.Tensor, frames_keep: int) -> torch.Tensor:
            x = t[:, :, :frames_keep]
            th, tw = content_pixel.height, content_pixel.width
            scale_h, scale_w = spatial_scale
            if h_pad is not None:
                before = scale_axis_pad(h_pad, scale_h).before
                x = x.narrow(3, before, th)
            else:
                need = x.shape[3] - th
                if need > 0:
                    x = x.narrow(3, need // 2, th)
                elif need < 0:
                    x, _ = resize_axis(x, 3, th, mode="symmetric")
            if w_pad is not None:
                before = scale_axis_pad(w_pad, scale_w).before
                x = x.narrow(4, before, tw)
            else:
                need = x.shape[4] - tw
                if need > 0:
                    x = x.narrow(4, need // 2, tw)
                elif need < 0:
                    x, _ = resize_axis(x, 4, tw, mode="symmetric")
            return x

        def _crop_emit(buf: torch.Tensor, wts: torch.Tensor | None, global_start: int) -> torch.Tensor | None:
            if global_start >= content_pixel.frames or buf.shape[2] < 1:
                return None
            frames_keep = min(buf.shape[2], content_pixel.frames - global_start)
            if frames_keep < 1:
                return None
            if not as_fhwc:
                chunk = _finalize(buf[:, :, :frames_keep], None if wts is None else wts[:, :, :frames_keep])
                return crop_pixels_to_content(
                    chunk,
                    frames_keep,
                    content_pixel.height,
                    content_pixel.width,
                    h_pad=h_pad,
                    w_pad=w_pad,
                    spatial_scale=spatial_scale,
                )

            # One materialize: contiguous FHWC in latent.dtype, still [-1, 1].
            # CFHW→FHWC cannot be inplace; range mapping is left to to_rgb.
            cfhw = _narrow_content_cfhw(buf, frames_keep)
            src = cfhw[0]  # C, F, H, W (view into accumulator)
            video = torch.empty(
                src.shape[1],
                src.shape[2],
                src.shape[3],
                src.shape[0],
                dtype=latent.dtype,
                device=src.device,
            )
            video.copy_(src.permute(1, 2, 3, 0))
            if not complementary:
                assert wts is not None
                w_cfhw = _narrow_content_cfhw(wts, frames_keep)
                wview = w_cfhw[0].permute(1, 2, 3, 0)
                # Inplace floor on exclusive weight region only (discarded after emit).
                wview.clamp_min_(_weight_floor(w_cfhw.dtype))
                video.div_(wview)
            return video

        for gi, group in enumerate(groups):
            curr_temporal_slice = group_slices[gi]
            buffer, weights = self._decode_temporal_group_isolated(
                group,
                feat_s4,
                s4_t,
                x_t_init,
                timestep,
                full_video_shape,
                curr_temporal_slice,
                generator=generator,
                complementary=complementary,
            )

            if overlap_stub is not None:
                overlap_len = int(overlap_stub.shape[2])
                if overlap_len > 0:
                    # Stub is exactly the region overlapping this group (cloned when
                    # the previous group finished); blend then write back into buffer.
                    overlap_stub += buffer[:, :, :overlap_len]
                    if complementary:
                        buffer[:, :, :overlap_len] = overlap_stub
                    else:
                        assert overlap_stub_weights is not None
                        assert weights is not None
                        overlap_stub_weights += weights[:, :, :overlap_len]
                        buffer[:, :, :overlap_len] = overlap_stub
                        weights[:, :, :overlap_len] = overlap_stub_weights
                overlap_stub = None
                overlap_stub_weights = None

            if gi + 1 < len(groups):
                next_start = group_slices[gi + 1].start
                exclusive_len = min(max(0, next_start - curr_temporal_slice.start), buffer.shape[2])
                emitted = _crop_emit(
                    buffer[:, :, :exclusive_len],
                    None if weights is None else weights[:, :, :exclusive_len],
                    curr_temporal_slice.start,
                )
                if emitted is not None:
                    yield emitted
                # Retain only the trailing overlap for the next handoff.
                overlap_stub = buffer[:, :, exclusive_len:].clone()
                if not complementary:
                    assert weights is not None
                    overlap_stub_weights = weights[:, :, exclusive_len:].clone()
                del buffer, weights
            else:
                emitted = _crop_emit(buffer, weights, curr_temporal_slice.start)
                if emitted is not None:
                    yield emitted

    def forward(
        self,
        sample: torch.Tensor,
        generator: torch.Generator | None = None,
    ) -> torch.Tensor:
        return next(self._decode_pixels(sample, tiling_config=None, generator=generator))

    def decode_video(
        self,
        latent: torch.Tensor,
        tiling_config: TilingConfig | None = None,
        generator: torch.Generator | None = None,
        *,
        keyframes: DecodeKeyframes | None = None,
    ) -> Iterator[torch.Tensor]:
        if keyframes is not None:
            yield from self._decode_video_with_keyframes(latent, keyframes, tiling_config, generator)
            return

        def to_rgb(frames: torch.Tensor) -> torch.Tensor:
            return frames.add_(1).mul_(0.5).clamp_(0, 1)

        for chunk in self._decode_pixels(latent, tiling_config, generator=generator, as_fhwc=True):
            yield to_rgb(chunk)

class LTX25DiffusionVideoDecoder(DiffusionVideoDecoder):
    def forward(self, sample, generator=None, keyframes=None):
        return self.decode(sample, generator=generator, keyframes=keyframes)

    def auto_tiling_config(self, latent, keyframes=None):
        pixel_shape = (
            VideoLatentShape.from_torch_shape(latent.shape)
            .upscale(self.video_downscale_factors)
            ._replace(channels=self.out_channels)
        )
        upsample_strides = [tuple(upsample.stride) for upsample in self.upsamples]
        pixel_scale = stage4_to_pixel_scale_factors(upsample_strides[3], self.patch_size)
        overlap_t, overlap_hw = recommended_pixel_overlaps(self.tile_halos, pixel_scale)
        ft, fh, fw = pixel_scale.time, pixel_scale.height, pixel_scale.width
        step_t = math.lcm(ft, VIDEO_SCALE_FACTORS.time)
        step_h = math.lcm(fh, VIDEO_SCALE_FACTORS.height)
        step_w = math.lcm(fw, VIDEO_SCALE_FACTORS.width)
        min_t = _round_up(max(2 * ft, 2 * overlap_t, _round_up(self.tile_min_sizes[0] * ft, ft), 16), step_t)
        min_h = _round_up(max(2 * fh, 2 * overlap_hw, _round_up(self.tile_min_sizes[1] * fh, fh), 512), step_h)
        min_w = _round_up(max(2 * fw, 2 * overlap_hw, _round_up(self.tile_min_sizes[2] * fw, fw), 512), step_w)
        return TileSizeConfig(
            frames=DimensionSizeConfig(min_t, overlap_t),
            height=DimensionSizeConfig(min_h, overlap_hw),
            width=DimensionSizeConfig(min_w, overlap_hw),
        )

    def _resolve_tiling_config(self, latent, tiled, keyframes=None, tile_size_in_pixels=None, tile_size_in_frames=None):
        if not tiled:
            return None
        if tile_size_in_pixels is not None and tile_size_in_frames is not None:
            try:
                pixel_scale = stage4_to_pixel_scale_factors([tuple(upsample.stride) for upsample in self.upsamples][3], self.patch_size)
                overlap_t, overlap_hw = recommended_pixel_overlaps(self.tile_halos, pixel_scale)
                tiling_config = TileSizeConfig(
                    frames=DimensionSizeConfig(tile_size_in_frames, overlap_t),
                    height=DimensionSizeConfig(tile_size_in_pixels, overlap_hw),
                    width=DimensionSizeConfig(tile_size_in_pixels, overlap_hw),
                )
                pixel_shape = (
                    VideoLatentShape.from_torch_shape(latent.shape)
                    .upscale(self.video_downscale_factors)
                    ._replace(channels=self.out_channels)
                )
                tiling_config.validate(pixel_scale, pixel_shape)
                return tiling_config
            except ValueError:
                pass
        return self.auto_tiling_config(latent, keyframes=keyframes)

    def decode(
        self,
        latent,
        tiled=False,
        seed=None,
        rand_device="cpu",
        keyframes=None,
        tile_size_in_pixels=None,
        tile_size_in_frames=None,
        **kwargs,
    ):
        generator = torch.Generator(device=rand_device).manual_seed(42 if seed is None else seed + 42)
        tiling_config = self._resolve_tiling_config(latent, tiled, keyframes, tile_size_in_pixels, tile_size_in_frames)
        iterator = (
            self._decode_pixels_with_keyframes(latent, keyframes, tiling_config, generator=generator)
            if keyframes is not None
            else self._decode_pixels(latent, tiling_config, generator=generator)
        )
        chunks = list(iterator)
        if not chunks:
            raise RuntimeError("Diffusion decoder produced no output chunks")
        return torch.cat(chunks, dim=2)