Skip to content

armgit / LTX-2

Public Imported from GitHub
0

Official Python inference and LoRA trainer package for the LTX-2 audio–video generative model.

45.7 KB
Raw Blame
from __future__ import annotations

import itertools
import math
from dataclasses import dataclass, replace
from typing import Callable, NamedTuple, Sequence

import torch

from ltx_core.types import VIDEO_SCALE_FACTORS, SpatioTemporalScaleFactors, VideoPixelShape


def compute_trapezoidal_mask_1d(
    length: int,
    ramp_left: int,
    ramp_right: int,
    left_starts_from_0: bool = False,
) -> torch.Tensor:
    """
    Generate a 1D trapezoidal blending mask with linear ramps.
    Args:
        length: Output length of the mask.
        ramp_left: Fade-in length on the left.
        ramp_right: Fade-out length on the right.
        left_starts_from_0: Whether the ramp starts from 0 or first non-zero value.
            Useful for temporal tiles where the first tile is causal.
    Returns:
        A 1D tensor of shape `(length,)` with values in [0, 1].
    """
    if length <= 0:
        raise ValueError("Mask length must be positive.")

    ramp_left = max(0, min(ramp_left, length))
    ramp_right = max(0, min(ramp_right, length))

    mask = torch.ones(length)

    if ramp_left > 0:
        interval_length = ramp_left + 1 if left_starts_from_0 else ramp_left + 2
        fade_in = torch.linspace(0.0, 1.0, interval_length)[:-1]
        if not left_starts_from_0:
            fade_in = fade_in[1:]
        mask[:ramp_left] *= fade_in

    if ramp_right > 0:
        fade_out = torch.linspace(1.0, 0.0, steps=ramp_right + 2)[1:-1]
        mask[-ramp_right:] *= fade_out

    return mask.clamp_(0, 1)


def compute_rectangular_mask_1d(
    length: int,
    left_ramp: int,
    right_ramp: int,
) -> torch.Tensor:
    """
    Generate a 1D rectangular (pulse) mask.
    Args:
        length: Output length of the mask.
        left_ramp: Number of elements at the start of the mask to set to 0.
        right_ramp: Number of elements at the end of the mask to set to 0.
    Returns:
        A 1D tensor of shape `(length,)` with values 0 or 1.
    """
    if length <= 0:
        raise ValueError("Mask length must be positive.")

    mask = torch.ones(length)
    if left_ramp > 0:
        mask[:left_ramp] = 0
    if right_ramp > 0:
        mask[-right_ramp:] = 0
    return mask


@dataclass(frozen=True)
class DimensionInterval:
    start: int
    end: int
    left_ramp: int
    right_ramp: int


@dataclass(frozen=True)
class DimensionIntervals:
    """Intervals which a single dimension of the latent space is split into.
    Each interval is defined by its start, end, left ramp, and right ramp.
    The start and end are the indices of the first and last element (exclusive) in the interval.
    Ramps are regions of the interval where the value of the mask tensor is
    interpolated between 0 and 1 for blending with neighboring intervals.
    The left ramp and right ramp values are the lengths of the left and right ramps.
    """

    intervals: list[DimensionInterval]


@dataclass(frozen=True)
class LatentIntervals:
    """Intervals which the latent tensor of given shape is split into.
    Each dimension of the latent space is split into intervals based on the length along said dimension.
    """

    original_shape: torch.Size
    dimension_intervals: tuple[DimensionIntervals, ...]


# Operation to split a single dimension of the tensor into intervals based on the length along the dimension.
SplitOperation = Callable[[int], DimensionIntervals]
# Operation to map the intervals in input dimension to slices and masks along a corresponding output dimension.
MappingOperation = Callable[[DimensionIntervals], tuple[list[slice], list[torch.Tensor]]]


def default_split_operation(length: int) -> DimensionIntervals:
    return DimensionIntervals(intervals=[DimensionInterval(start=0, end=length, left_ramp=0, right_ramp=0)])


DEFAULT_SPLIT_OPERATION: SplitOperation = default_split_operation


def untiled_mask_1d() -> torch.Tensor:
    """Length-1 ones that broadcast over an untiled axis (historical ``None`` mask)."""
    return torch.ones(1)


def default_mapping_operation(
    _intervals: DimensionIntervals,
) -> tuple[list[slice], list[torch.Tensor]]:
    return [slice(0, None)], [untiled_mask_1d()]


DEFAULT_MAPPING_OPERATION: MappingOperation = default_mapping_operation


# ---------------------------------------------------------------------------
# Split functions
# ---------------------------------------------------------------------------


def _grow_last_tile_to_min(intervals: list[DimensionInterval], min_tile_size: int) -> list[DimensionInterval]:
    """Grow a short last tile left to ``min_tile_size``; widen penultimate ``right_ramp``."""
    if len(intervals) <= 1:
        return list(intervals)
    last = intervals[-1]
    if last.end - last.start >= min_tile_size:
        return list(intervals)
    new_start = last.end - min_tile_size
    prev = intervals[-2]
    new_overlap = prev.end - new_start
    return [
        *intervals[:-2],
        replace(prev, right_ramp=new_overlap),
        replace(last, start=new_start, left_ramp=new_overlap),
    ]


def _validate_tile_intervals(intervals: list[DimensionInterval], *, dim_size: int, min_tile_size: int) -> None:
    """Validate coverage, ramp/overlap consistency, and ``min_tile_size``."""
    if not intervals or intervals[0].start != 0 or intervals[-1].end != dim_size:
        raise ValueError(f"tiles must cover [0, {dim_size})")
    for i, iv in enumerate(intervals):
        length = iv.end - iv.start
        if length < min_tile_size:
            raise ValueError(f"tile {i} length {length} is below min_tile_size={min_tile_size}")
        if iv.left_ramp < 0 or iv.right_ramp < 0 or iv.left_ramp > length or iv.right_ramp > length:
            raise ValueError(f"tile {i} has invalid ramps: left={iv.left_ramp}, right={iv.right_ramp}, length={length}")
        if i == 0:
            continue
        overlap = intervals[i - 1].end - iv.start
        if overlap < 0 or intervals[i - 1].right_ramp != overlap or iv.left_ramp != overlap:
            raise ValueError(f"tiles {i - 1}/{i}: ramp/overlap mismatch (overlap={overlap})")


def split_by_size(size: int, overlap: int, min_tile_size: int | None = None) -> SplitOperation:
    """Split a dimension into overlapping tiles of a given size.
    Tiles are sized ``size`` with ``overlap`` shared elements between
    consecutive tiles.  The last tile may be shorter if the dimension
    doesn't divide evenly.  If ``min_tile_size`` is set and the last tile is
    shorter, it is grown leftward (penultimate ``right_ramp`` widens); the
    result is validated and invalid layouts raise ``ValueError``.
    Args:
        size: Target tile size (in axis units).
        overlap: Overlap between consecutive tiles.
        min_tile_size: Optional minimum tile length. ``None`` keeps legacy
            short-last-tile behavior.
    Returns:
        A split operation that divides a dimension into tiles.
    """
    if size <= 0:
        raise ValueError(f"size must be > 0, got {size}")
    if overlap < 0 or overlap >= size:
        raise ValueError(f"overlap must satisfy 0 <= overlap < size, got overlap={overlap}, size={size}")
    if min_tile_size is not None and min_tile_size < 1:
        raise ValueError(f"min_tile_size must be >= 1, got {min_tile_size}")

    def split(dimension_size: int) -> DimensionIntervals:
        if min_tile_size is not None and dimension_size < min_tile_size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        if dimension_size <= size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        amount = (dimension_size + size - 2 * overlap - 1) // (size - overlap)
        intervals = [
            DimensionInterval(start=0, end=size, left_ramp=0, right_ramp=overlap),
            *(
                DimensionInterval(
                    start=i * (size - overlap),
                    end=i * (size - overlap) + size,
                    left_ramp=overlap,
                    right_ramp=overlap,
                )
                for i in range(1, amount - 1)
            ),
            DimensionInterval(
                start=(amount - 1) * (size - overlap), end=dimension_size, left_ramp=overlap, right_ramp=0
            ),
        ]
        if min_tile_size is not None:
            intervals = _grow_last_tile_to_min(intervals, min_tile_size)
            _validate_tile_intervals(intervals, dim_size=dimension_size, min_tile_size=min_tile_size)
        return DimensionIntervals(intervals=intervals)

    return split


def split_by_size_pinned(size: int, min_overlap: int) -> SplitOperation:
    """Split into equal-size tiles with first and last origins pinned to the extent.
    Unlike :func:`split_by_size`, this operation never emits a short trailing
    tile. ``min_overlap`` is a target of at most half ``size``. The tile count
    grows until the target is met, capped so three tiles never cover one cell.
    The realized overlap therefore follows from the extent and can differ from
    the target in either direction: below it when the cap is hit, and up to
    ``size - 1`` when two tiles span an extent barely longer than one tile
    (33 cells, ``size=32``: overlap 31).
    This layout is intended for models trained on a fixed spatial window, where
    changing the final tile's shape changes the model's effective content scale.
    """
    if size <= 0:
        raise ValueError(f"size must be > 0, got {size}")
    if min_overlap < 0 or min_overlap * 2 > size:
        raise ValueError(f"min_overlap must be at most half the tile size, got min_overlap={min_overlap}, size={size}")

    def split(dimension_size: int) -> DimensionIntervals:
        if dimension_size <= size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)

        # Cap so a tile and the one two steps away cannot overlap.
        limit = max(2, 1 + (dimension_size - size) // ((size + 1) // 2))
        tiles_count = 2
        while tiles_count < limit and (tiles_count * size - dimension_size) / (tiles_count - 1) < min_overlap:
            tiles_count += 1
        base, remainder = divmod(dimension_size - size, tiles_count - 1)
        origins = [0]
        for index in range(tiles_count - 1):
            origins.append(origins[-1] + base + (1 if index < remainder else 0))

        intervals: list[DimensionInterval] = []
        for index, start in enumerate(origins):
            left_overlap = 0 if index == 0 else origins[index - 1] + size - start
            right_overlap = 0 if index == tiles_count - 1 else start + size - origins[index + 1]
            intervals.append(
                DimensionInterval(
                    start=start,
                    end=start + size,
                    left_ramp=left_overlap,
                    right_ramp=right_overlap,
                )
            )
        _validate_tile_intervals(intervals, dim_size=dimension_size, min_tile_size=size)
        return DimensionIntervals(intervals=intervals)

    return split


def split_temporal_causal(size: int, overlap: int, min_tile_size: int | None = None) -> SplitOperation:
    """Split a temporal axis into overlapping tiles with causal handling.
    Each tile after the first is shifted back by 1 and its left ramp is
    increased by 1, ensuring causal continuity through the blend ramps.
    Args:
        size: Tile size in axis units.
        overlap: Overlap between tiles in the same units.
        min_tile_size: Optional floor forwarded to :func:`split_by_size`.
    Returns:
        Split operation that divides temporal dimension with causal handling.
    """
    non_causal_split = split_by_size(size, overlap, min_tile_size=min_tile_size)

    def split(dimension_size: int) -> DimensionIntervals:
        if dimension_size <= size:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        dim_intervals = non_causal_split(dimension_size)
        if len(dim_intervals.intervals) <= 1:
            return dim_intervals
        modified_intervals = [dim_intervals.intervals[0]] + [
            replace(interval, start=interval.start - 1, left_ramp=interval.left_ramp + 1)
            for interval in dim_intervals.intervals[1:]
        ]
        return DimensionIntervals(intervals=modified_intervals)

    return split


def split_temporal(tile_size_frames: int, overlap_frames: int) -> SplitOperation:
    """Split a temporal axis in video frame space into overlapping tiles.
    Args:
        tile_size_frames: Tile length in frames.
        overlap_frames: Overlap between consecutive tiles in frames.
    Returns:
        Split operation that takes frame count and returns DimensionIntervals in frame indices.
    """
    non_causal_split = split_by_size(tile_size_frames, overlap_frames)

    def split(dimension_size: int) -> DimensionIntervals:
        if dimension_size <= tile_size_frames:
            return DEFAULT_SPLIT_OPERATION(dimension_size)
        dim_intervals = non_causal_split(dimension_size)
        modified_intervals = [
            replace(interval, end=interval.end + 1, right_ramp=0) for interval in dim_intervals.intervals[:-1]
        ] + [replace(dim_intervals.intervals[-1], right_ramp=0)]
        return DimensionIntervals(intervals=modified_intervals)

    return split


def split_by_count_temporal_causal(
    num_tiles: int, overlap: int = 0, min_tile_size: int | None = None
) -> SplitOperation:
    """Split a temporal dimension by count with causal handling.
    Wraps :func:`split_by_count` with the same causal adjustment as
    :func:`split_temporal_causal`: each tile after the first is shifted
    back by 1 and its left ramp is increased by 1.
    Args:
        num_tiles: Number of tiles. Must be >= 1.
        overlap: Overlap between adjacent tiles (default 0).
        min_tile_size: Optional floor forwarded to :func:`split_by_count`.
    Returns:
        A split operation that divides a temporal dimension into tiles.
    """
    non_causal_split = split_by_count(num_tiles, overlap, min_tile_size=min_tile_size)

    def split(dimension_size: int) -> DimensionIntervals:
        dim_intervals = non_causal_split(dimension_size)
        if len(dim_intervals.intervals) <= 1:
            return dim_intervals
        modified_intervals = [dim_intervals.intervals[0]] + [
            replace(interval, start=interval.start - 1, left_ramp=interval.left_ramp + 1)
            for interval in dim_intervals.intervals[1:]
        ]
        return DimensionIntervals(intervals=modified_intervals)

    return split


def split_at_seams(boundaries: Sequence[int], num_tiles: int, overlap: int = 0) -> SplitOperation:
    """Split a dimension on boundary cells whose content is already known, dropping the overlap.
    ``boundaries`` are the ``K + 1`` segment edges in grid cells, starting at 0 and ending at the
    last cell of the dimension. The ``K`` segments are dealt largest-first so leftover segments go to the leading tiles;
    ``num_tiles`` larger than ``K`` is clamped. Each tile but the first starts ``overlap`` cells
    before the boundary it resumes after. That run-up is context only: it lands in the interval's
    ``left_ramp``, which :func:`identity_mapping_operation` with ``rectangular=True`` masks to zero,
    so the earlier tile keeps the boundary cell and this one contributes strictly after it.
    The point of cutting here is that nothing needs blending. A ramp is what a pair of tiles needs
    when neither of them knows the truth at the seam; on a boundary cell both reproduce the same
    known frame, so averaging them only smears it.
    Args:
        boundaries: Segment edges in grid cells, strictly increasing, starting at 0.
        num_tiles: Number of tiles. Must be >= 1. Extra tiles beyond the segment count are dropped.
        overlap: Context cells each non-first tile denoises before the cell it resumes at, in grid
            units. Clamped at the start of the dimension.
    Returns:
        A split operation that divides a dimension on ``boundaries``.
    """
    boundaries = tuple(boundaries)
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    if overlap < 0:
        raise ValueError(f"overlap must be >= 0, got {overlap}")
    if len(boundaries) < 2 or boundaries[0] != 0:
        raise ValueError(f"boundaries must start at 0 and hold at least one segment, got {list(boundaries)}")
    if any(b <= a for a, b in itertools.pairwise(boundaries)):
        raise ValueError(f"boundaries must be strictly increasing, got {list(boundaries)}")
    n_segments = len(boundaries) - 1
    n_tiles = min(num_tiles, n_segments)
    base, leftover = divmod(n_segments, n_tiles)
    counts = [base + (1 if index < leftover else 0) for index in range(n_tiles)]

    def split(dim_size: int) -> DimensionIntervals:
        if boundaries[-1] != dim_size - 1:
            raise ValueError(f"boundaries must end at the last cell ({dim_size - 1}), got {boundaries[-1]}")
        intervals: list[DimensionInterval] = []
        cursor = 0
        for tile_index, count in enumerate(counts):
            resume = boundaries[cursor] + 1
            start = 0 if tile_index == 0 else max(0, resume - overlap)
            cursor += count
            intervals.append(
                DimensionInterval(
                    start=start,
                    end=boundaries[cursor] + 1,
                    left_ramp=0 if tile_index == 0 else resume - start,
                    right_ramp=0,
                )
            )
        return DimensionIntervals(intervals=intervals)

    return split


def split_by_count(num_tiles: int, overlap: int = 0, min_tile_size: int | None = None) -> SplitOperation:
    """Split a dimension into a given number of tiles with overlap.
    Computes the tile size as
    ``(dim_size + overlap * (num_tiles - 1)) // num_tiles`` so that
    ``num_tiles`` tiles of that size with ``overlap`` shared elements
    cover the dimension evenly.  Delegates to :func:`split_by_size` for
    the actual interval construction.
    When the total ``dim_size + overlap * (num_tiles - 1)`` is not evenly
    divisible by ``num_tiles``, the first ``remainder`` tiles each absorb
    one extra unit.
    Args:
        num_tiles: Number of tiles. Must be >= 1.
        overlap: Overlap between adjacent tiles (default 0). Must be >= 0
            and less than the computed tile size.
        min_tile_size: Optional floor forwarded to last-tile growth / validation.
    Returns:
        A split operation that divides a dimension into tiles.
    """
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    if overlap < 0:
        raise ValueError(f"overlap must be >= 0, got {overlap}")
    if min_tile_size is not None and min_tile_size < 1:
        raise ValueError(f"min_tile_size must be >= 1, got {min_tile_size}")

    def split(dim_size: int) -> DimensionIntervals:
        if num_tiles > dim_size:
            raise ValueError(
                f"num_tiles ({num_tiles}) exceeds dim_size ({dim_size}). Cannot assign at least 1 unit per tile."
            )
        if num_tiles == 1:
            return DEFAULT_SPLIT_OPERATION(dim_size)

        total = dim_size + overlap * (num_tiles - 1)
        tile_size = total // num_tiles
        if tile_size <= overlap:
            raise ValueError(
                f"split_by_count produced size={tile_size} <= overlap={overlap} "
                f"for dim_size={dim_size}, num_tiles={num_tiles}"
            )
        remainder = total % num_tiles

        base_intervals = split_by_size(tile_size, overlap)(dim_size - remainder).intervals

        # First `remainder` tiles each absorb 1 extra unit; shift subsequent boundaries.
        intervals: list[DimensionInterval] = []
        for i, iv in enumerate(base_intervals):
            shift = min(i, remainder)
            grow = 1 if i < remainder else 0
            intervals.append(replace(iv, start=iv.start + shift, end=iv.end + shift + grow))

        if min_tile_size is not None:
            intervals = _grow_last_tile_to_min(intervals, min_tile_size)
            _validate_tile_intervals(intervals, dim_size=dim_size, min_tile_size=min_tile_size)

        return DimensionIntervals(intervals=intervals)

    return split


# ---------------------------------------------------------------------------
# Mapping operations
# ---------------------------------------------------------------------------


def identity_mapping_operation(
    intervals: DimensionIntervals,
    *,
    rectangular: bool = False,
) -> tuple[list[slice], list[torch.Tensor]]:
    """Map each DimensionInterval to an output region at the same position.
    For every interval the output start/end matches the input start/end and a 1-D mask is built
    from the interval's left_ramp and right_ramp. The default mask is trapezoidal (blend on the
    ramps). ``rectangular=True`` drops the ramps outright: the overlap is context the tile denoised
    but does not contribute. Pair that with a split whose ramps are one-sided, such as
    :func:`split_at_seams` -- ramps on both sides of an interval would leave a hole between tiles.
    """
    mask_1d = compute_rectangular_mask_1d if rectangular else compute_trapezoidal_mask_1d
    out_slices: list[slice] = []
    masks: list[torch.Tensor] = []
    for iv in intervals.intervals:
        out_slices.append(slice(iv.start, iv.end))
        masks.append(mask_1d(iv.end - iv.start, iv.left_ramp, iv.right_ramp))
    return out_slices, masks


class Tile(NamedTuple):
    """
    Represents a single tile.
    Attributes:
        in_coords:
            Tuple of slices specifying where to cut the tile from the INPUT tensor.
        out_coords:
            Tuple of slices specifying where this tile's OUTPUT should be placed in the reconstructed OUTPUT tensor.
        masks_1d:
            Per-dimension masks in OUTPUT units.
            Untiled axes use a length-1 ones tensor (broadcasts). These are used
            for separable blending (and for the dense ``blend_mask`` property).
    Methods:
        blend_mask:
            Create a single N-D mask from the per-dimension masks.
    """

    in_coords: tuple[slice, ...]
    out_coords: tuple[slice, ...]
    masks_1d: tuple[torch.Tensor, ...]

    @property
    def blend_mask(self) -> torch.Tensor:
        num_dims = len(self.out_coords)
        per_dimension_masks: list[torch.Tensor] = []

        for dim_idx in range(num_dims):
            mask_1d = self.masks_1d[dim_idx]
            view_shape = [1] * num_dims
            # Reshape (L,) -> (1, ..., L, ..., 1) so masks across dimensions broadcast-multiply.
            view_shape[dim_idx] = mask_1d.shape[0]
            per_dimension_masks.append(mask_1d.view(*view_shape))

        # Multiply per-dimension masks to form the full N-D mask (separable blending window).
        combined_mask = per_dimension_masks[0]
        for mask in per_dimension_masks[1:]:
            combined_mask = combined_mask * mask

        return combined_mask


def scale_by_masks_1d(x: torch.Tensor, masks_1d: Sequence[torch.Tensor]) -> torch.Tensor:
    """Multiply ``x`` by separable 1d masks with broadcasting.
    ``len(masks_1d)`` must equal ``x.ndim``. Prefer float32 masks so bf16/fp16 ``x`` promotes.
    Length-1 masks (untiled axes) broadcast over that dimension.
    """
    if len(masks_1d) != x.ndim:
        raise ValueError(f"masks_1d length {len(masks_1d)} != x.ndim {x.ndim}")
    out = x
    for axis, mask in enumerate(masks_1d):
        view_shape = [1] * x.ndim
        view_shape[axis] = -1
        out = out * mask.reshape(*view_shape)
    return out


def masks_are_complementary(
    tiles: Sequence[Tile],
    full_shape: Sequence[int],
    *,
    atol: float = 1e-5,
) -> bool:
    """Return whether per-axis 1d blend masks partition unity (sum to 1).
    Checks each axis independently over the unique out-slices on that axis
    (cartesian tile products would otherwise multi-count the same 1d interval).
    When True, weighted accumulation needs no denominator.
    """
    if not tiles:
        return True
    ndim = len(full_shape)
    for tile in tiles:
        if len(tile.out_coords) != ndim or len(tile.masks_1d) != ndim:
            raise ValueError(
                f"Tile out_coords/masks_1d rank {len(tile.out_coords)}/{len(tile.masks_1d)} != full_shape rank {ndim}"
            )
    for axis, length in enumerate(full_shape):
        # Explicit CPU float32: masks may live on CUDA; a non-CPU default device
        # must not place ``acc`` on GPU (device-mismatch on ``acc[sl] +=``).
        acc = torch.zeros(length, dtype=torch.float32, device="cpu")
        seen: set[tuple[int | None, int | None]] = set()
        for tile in tiles:
            sl = tile.out_coords[axis]
            key = (sl.start, sl.stop)
            if key in seen:
                continue
            seen.add(key)
            # Length-1 untiled masks broadcast over ``acc[sl]``.
            acc[sl] += tile.masks_1d[axis].detach().float().cpu()
        if not torch.allclose(acc, torch.ones(length, dtype=torch.float32), atol=atol, rtol=0.0):
            return False
    return True


def compute_summed_weights(
    tiles: Sequence[Tile],
    full_shape: Sequence[int],
) -> torch.Tensor:
    """Build the dense denominator for weighted blending over ``full_shape``.
    Uses separable per-axis mask broadcasts — never ``Tile.blend_mask``.
    Requires concrete ``out_coords`` (``stop`` not ``None``) on every axis.
    Always builds on CPU float32 so CUDA masks / a non-CPU default device cannot
    place a multi-GB ``[F,H,W]`` tensor on GPU.
    """
    weights = torch.zeros(*full_shape, dtype=torch.float32, device="cpu")
    for tile in tiles:
        masks = tuple(m.detach().float().cpu() for m in tile.masks_1d)
        region_shape = tuple(s.stop - s.start for s in tile.out_coords)
        region = torch.ones(region_shape, dtype=torch.float32, device="cpu")
        weights[tile.out_coords] += scale_by_masks_1d(region, masks)
    return weights.clamp(min=1e-8)


def create_tiles_from_intervals_and_mappers(
    intervals: LatentIntervals,
    mappers: list[MappingOperation],
) -> list[Tile]:
    full_dim_input_slices: list[list[slice]] = []
    full_dim_output_slices: list[list[slice]] = []
    full_dim_masks_1d: list[list[torch.Tensor]] = []
    for axis_index in range(len(intervals.original_shape)):
        dimension_intervals = intervals.dimension_intervals[axis_index]
        input_slices = [slice(interval.start, interval.end) for interval in dimension_intervals.intervals]
        output_slices, masks_1d = mappers[axis_index](dimension_intervals)
        n_intervals = len(input_slices)
        if len(output_slices) != n_intervals or len(masks_1d) != n_intervals:
            raise ValueError(
                f"Axis {axis_index}: mapper produced {len(output_slices)} output slices and "
                f"{len(masks_1d)} masks for {n_intervals} input intervals"
            )
        full_dim_input_slices.append(input_slices)
        full_dim_output_slices.append(output_slices)
        full_dim_masks_1d.append(masks_1d)

    return [
        Tile(in_coords=in_coord, out_coords=out_coord, masks_1d=mask_1d)
        for in_coord, out_coord, mask_1d in zip(
            itertools.product(*full_dim_input_slices),
            itertools.product(*full_dim_output_slices),
            itertools.product(*full_dim_masks_1d),
            strict=True,
        )
    ]


def create_tiles(
    latent_shape: torch.Size,
    splitters: list[SplitOperation],
    mappers: list[MappingOperation],
) -> list[Tile]:
    if len(splitters) != len(latent_shape):
        raise ValueError(
            f"Number of splitters must be equal to number of dimensions in latent shape, "
            f"got {len(splitters)} and {len(latent_shape)}"
        )
    if len(mappers) != len(latent_shape):
        raise ValueError(
            f"Number of mappers must be equal to number of dimensions in latent shape, "
            f"got {len(mappers)} and {len(latent_shape)}"
        )
    intervals = [splitter(length) for splitter, length in zip(splitters, latent_shape, strict=True)]
    latent_intervals = LatentIntervals(original_shape=latent_shape, dimension_intervals=tuple(intervals))
    return create_tiles_from_intervals_and_mappers(latent_intervals, mappers)


def group_tiles_by_temporal_slice(tiles: list[Tile]) -> list[list[Tile]]:
    """Group consecutive tiles that share the same temporal ``out_coords`` slice.
    Assumes ``tiles`` is ordered with the temporal axis varying slowest (true
    for every tile list this codebase builds via ``itertools.product`` with
    the temporal axis first), so equal temporal slices are always contiguous.
    """
    if not tiles:
        return []

    groups = []
    current_slice = tiles[0].out_coords[2]
    current_group = []

    for tile in tiles:
        tile_slice = tile.out_coords[2]
        if tile_slice == current_slice:
            current_group.append(tile)
        else:
            groups.append(current_group)
            current_slice = tile_slice
            current_group = [tile]

    if current_group:
        groups.append(current_group)

    return groups


# ---------------------------------------------------------------------------
# Video-grid tiling configs
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class DimensionTilingConfig:
    """Tiling parameters for a single dimension of the patchified grid.
    Attributes:
        num_tiles: Number of tiles along this dimension. ``1`` with ``overlap=0``
            means the axis is not tiled.
        overlap: Overlap between adjacent tiles, in latent grid units.
            Adjacent tiles share ``overlap`` grid cells at their
            boundary, producing an overlap zone blended with
            trapezoidal masks.
    """

    num_tiles: int = 1
    overlap: int = 0

    def __post_init__(self) -> None:
        if self.num_tiles < 1:
            raise ValueError(f"num_tiles must be >= 1, got {self.num_tiles}")
        if self.overlap < 0:
            raise ValueError(f"overlap must be >= 0, got {self.overlap}")

    def is_tiled(self) -> bool:
        """True when this axis is split into more than one tile (or has overlap)."""
        return self.num_tiles > 1 or self.overlap > 0

    @classmethod
    def from_tile_size(cls, dim_size: int, tile_size: int, overlap: int = 0) -> DimensionTilingConfig:
        """Create config by computing ``num_tiles`` from dimension size and tile size.
        Args:
            dim_size: Total length of the dimension.
            tile_size: Desired tile size.
            overlap: Overlap between consecutive tiles.
        Returns:
            A ``DimensionTilingConfig`` with the computed ``num_tiles``.
        """
        split_op = split_by_size(tile_size, overlap)
        intervals = split_op(dim_size)
        return cls(num_tiles=len(intervals.intervals), overlap=overlap)


@dataclass(frozen=True)
class DimensionSizeConfig:
    """Tile size and overlap for a single video axis (frames / height / width).
    Mirrors :class:`DimensionTilingConfig`, but specifies tile *size* rather than
    tile *count*. ``tile_size=0`` means the axis is not tiled (covers the whole
    length).     Axis-specific VAE pixel constraints (divisibility / minimums) are
    enforced by :meth:`TileSizeConfig.validate` for tiled axes only.
    """

    tile_size: int = 0
    overlap: int = 0

    def __post_init__(self) -> None:
        if self.tile_size < 0:
            raise ValueError(f"tile_size must be >= 0, got {self.tile_size}")
        if self.overlap < 0:
            raise ValueError(f"overlap must be >= 0, got {self.overlap}")
        if self.tile_size == 0:
            if self.overlap != 0:
                raise ValueError("untiled axis (tile_size=0) must have overlap=0")
            return
        if self.overlap >= self.tile_size:
            raise ValueError(f"Overlap must be less than tile size, got {self.overlap} and {self.tile_size}")

    def is_tiled(self) -> bool:
        """True when this axis has a positive tile size (caller intends to split it)."""
        return self.tile_size > 0


@dataclass(frozen=True)
class TileCountConfig:
    """Tiling layout for a ``(F, H, W)`` grid by tile *counts*.
    Overlaps are in latent-grid units. Mirror of :class:`TileSizeConfig`.
    Attributes:
        frames: Tiling along the temporal (frames) dimension.
        height: Tiling along the latent height dimension.
        width: Tiling along the latent width dimension.
    """

    frames: DimensionTilingConfig = DimensionTilingConfig()
    height: DimensionTilingConfig = DimensionTilingConfig()
    width: DimensionTilingConfig = DimensionTilingConfig()

    def validate(self, scale_factors: SpatioTemporalScaleFactors, video_shape: VideoPixelShape) -> None:
        """Raise if this count layout cannot tile ``video_shape`` under ``scale_factors``.
        Counts/overlaps are in latent-grid units. ``video_shape.frames <= 0`` skips the
        temporal axis (duration not yet known). Spatial axes always checked.
        """
        check_temporal = _assert_video_on_vae_grid(scale_factors, video_shape)
        latent_h = video_shape.height // scale_factors.height
        latent_w = video_shape.width // scale_factors.width
        _validate_count_axis(self.height, latent_h, "height")
        _validate_count_axis(self.width, latent_w, "width")
        if check_temporal:
            latent_f = (video_shape.frames - 1) // scale_factors.time + 1
            _validate_count_axis(self.frames, latent_f, "frames")

    def to_splitters(
        self,
        scale_factors: SpatioTemporalScaleFactors,
        min_tile_size: tuple[int, int, int] | None = None,
        *,
        causal_temporal: bool = True,
    ) -> tuple[SplitOperation, SplitOperation, SplitOperation]:
        """Build ``(T, H, W)`` latent-grid split operations for this count layout.
        ``scale_factors`` is accepted for signature parity with
        :meth:`TileSizeConfig.to_splitters` and ignored — counts are already in
        grid units. When ``causal_temporal`` is True (VAE encode/decode), the
        frames axis uses :func:`split_by_count_temporal_causal`; otherwise plain
        :func:`split_by_count`. ``min_tile_size`` is a per-axis floor in the same
        units as the split.
        """
        del scale_factors
        min_t = min_h = min_w = None
        if min_tile_size is not None:
            min_t, min_h, min_w = min_tile_size

        def axis_split(cfg: DimensionTilingConfig, axis_min: int | None, *, temporal: bool) -> SplitOperation:
            if not cfg.is_tiled():
                return DEFAULT_SPLIT_OPERATION
            if temporal and causal_temporal:
                return split_by_count_temporal_causal(cfg.num_tiles, cfg.overlap, min_tile_size=axis_min)
            return split_by_count(cfg.num_tiles, cfg.overlap, min_tile_size=axis_min)

        return (
            axis_split(self.frames, min_t, temporal=True),
            axis_split(self.height, min_h, temporal=False),
            axis_split(self.width, min_w, temporal=False),
        )

    def video_chunks_number(self, num_frames: int) -> int:
        """Number of temporal decode chunks for ``num_frames`` under this layout."""
        del num_frames
        return max(1, self.frames.num_tiles)


@dataclass(frozen=True)
class TileSizeConfig:
    """Size-based tiling layout for a ``(F, H, W)`` video — mirror of ``TileCountConfig``.
    Each axis is a non-optional :class:`DimensionSizeConfig`; ``tile_size=0`` means
    untiled on that axis (:meth:`DimensionSizeConfig.is_tiled`). Sizes and overlaps
    are in pixel / frame units. Conversion to a split grid is an explicit
    ``scale_factors`` argument to :meth:`to_splitters` (not stored on the config).
    Legality vs a VAE grid is checked by :meth:`validate` (same factors decode will
    pass to :meth:`to_splitters`), not at construction.
    Attributes:
        frames: Temporal tile size/overlap in frames.
        height: Spatial height tile size/overlap in pixels.
        width: Spatial width tile size/overlap in pixels.
    """

    frames: DimensionSizeConfig = DimensionSizeConfig()
    height: DimensionSizeConfig = DimensionSizeConfig()
    width: DimensionSizeConfig = DimensionSizeConfig()

    def validate(self, scale_factors: SpatioTemporalScaleFactors, video_shape: VideoPixelShape) -> None:
        """Raise if this size layout is illegal for ``video_shape`` under ``scale_factors``.
        Checks tile/overlap divisibility and minimums against the VAE grid, and that
        the video extents are compatible with that grid. ``video_shape.frames <= 0``
        skips the temporal axis (duration not yet known); height/width always checked.
        """
        check_temporal = _assert_video_on_vae_grid(scale_factors, video_shape)
        _validate_size_axis(self.height, scale_factors.height, "height")
        _validate_size_axis(self.width, scale_factors.width, "width")
        if check_temporal:
            _validate_size_axis(self.frames, scale_factors.time, "frames")

    @classmethod
    def default(cls) -> TileSizeConfig:
        return cls(
            frames=DimensionSizeConfig(tile_size=80, overlap=24),
            height=DimensionSizeConfig(tile_size=768, overlap=64),
            width=DimensionSizeConfig(tile_size=768, overlap=64),
        )

    @classmethod
    def from_long_side(
        cls,
        *,
        long_side: DimensionSizeConfig,
        height: int,
        width: int,
        scale_factors: SpatioTemporalScaleFactors,
        frames: DimensionSizeConfig | None = None,
    ) -> TileSizeConfig:
        """Aspect-coupled construction — old single-spatial long-side behavior, explicit.
        Matches main-era ``latent_tile_splitters``: scale the long-side tile in
        *latent* units with ``round(size_lat * axis_lat / long_lat)``, then
        multiply back by the VAE factor. Pixel-space ``round`` + ceil-snap would
        bias the short axis up by almost one latent (e.g. 680 → 704 vs 672).
        Both axes share ``long_side.overlap``.
        """
        if height < 1 or width < 1:
            raise ValueError(f"height/width must be >= 1, got {height}x{width}")
        if not long_side.is_tiled():
            raise ValueError("long_side must be tiled (tile_size > 0)")
        if scale_factors.height < 1 or scale_factors.width < 1:
            raise ValueError(f"scale_factors height/width must be >= 1, got {scale_factors}")
        span = max(height, width)

        def axis_size(axis_len: int, factor: int) -> int:
            # Latent-grid round (same as main decode enable_on_axis), not pixel ceil.
            axis_lat = axis_len // factor
            long_lat = span // factor
            size_lat = long_side.tile_size // factor
            overlap_lat = long_side.overlap // factor
            lower_threshold = max(2, overlap_lat + 1)
            tile_lat = max(lower_threshold, round(size_lat * axis_lat / long_lat))
            tile_px = tile_lat * factor
            min_legal = max(2 * factor, long_side.overlap + factor)
            return max(tile_px, min_legal)

        return cls(
            frames=DimensionSizeConfig() if frames is None else frames,
            height=DimensionSizeConfig(tile_size=axis_size(height, scale_factors.height), overlap=long_side.overlap),
            width=DimensionSizeConfig(tile_size=axis_size(width, scale_factors.width), overlap=long_side.overlap),
        )

    def to_splitters(
        self,
        scale_factors: SpatioTemporalScaleFactors,
        min_tile_size: tuple[int, int, int] | None = None,
        *,
        causal_temporal: bool = True,
    ) -> tuple[SplitOperation, SplitOperation, SplitOperation]:
        """Build ``(T, H, W)`` grid split ops from pixel/frame sizes via ``scale_factors``.
        When ``causal_temporal`` is True, frames use :func:`split_temporal_causal`.
        """
        min_t = min_h = min_w = None
        if min_tile_size is not None:
            min_t, min_h, min_w = min_tile_size

        def enable_size_axis(
            factor: int,
            axis_min: int | None,
            cfg: DimensionSizeConfig,
            axis_name: str,
            *,
            temporal: bool,
        ) -> SplitOperation:
            if not cfg.is_tiled():
                return DEFAULT_SPLIT_OPERATION
            _validate_size_axis(cfg, factor, axis_name)
            size = cfg.tile_size // factor
            overlap = cfg.overlap // factor
            lower_threshold = max(2, overlap + 1)
            tile = max(lower_threshold, size)
            if temporal and causal_temporal:
                return split_temporal_causal(tile, overlap, min_tile_size=axis_min)
            return split_by_size(tile, overlap, min_tile_size=axis_min)

        return (
            enable_size_axis(scale_factors.time, min_t, self.frames, "frames", temporal=True),
            enable_size_axis(scale_factors.height, min_h, self.height, "height", temporal=False),
            enable_size_axis(scale_factors.width, min_w, self.width, "width", temporal=False),
        )

    def video_chunks_number(self, num_frames: int, *, time_scale: int = VIDEO_SCALE_FACTORS.time) -> int:
        """Number of temporal decode chunks for ``num_frames`` under this layout.
        Mirrors what decode actually does: :meth:`to_splitters` converts this axis to the
        latent grid and hands it to :func:`split_by_size`, so the count must be taken there
        too. Doing the arithmetic in pixel units instead over-reports by one whenever the
        trailing tile is absorbed -- including the common case of a tile larger than the
        clip, which is a single tile but used to report two.
        """
        if not self.frames.is_tiled():
            return 1
        # Same derivation as ``to_splitters.enable_size_axis``.
        overlap = self.frames.overlap // time_scale
        size = max(2, overlap + 1, self.frames.tile_size // time_scale)
        latent_frames = (num_frames - 1) // time_scale + 1
        if latent_frames <= size:
            return 1
        # Same tile count as ``split_by_size``.
        return (latent_frames + size - 2 * overlap - 1) // (size - overlap)


# Either size-based (single-GPU VAE) or count-based (MGPU / explicit counts).
TilingConfig = TileSizeConfig | TileCountConfig


class AutoTiling:
    """Sentinel: pipeline should recommend decode tiling (DiffVAE-aware / Conv default).
    Distinct from ``None``, which means untiled decode.
    """

    __slots__ = ()

    def __repr__(self) -> str:
        return "AUTO_TILING"


AUTO_TILING = AutoTiling()

# Pipeline ``tiling_config`` argument: explicit layout, auto-recommend, or untiled.
PipelineTiling = TilingConfig | AutoTiling | None


def _assert_video_on_vae_grid(
    scale_factors: SpatioTemporalScaleFactors,
    video_shape: VideoPixelShape,
) -> bool:
    """Raise if ``video_shape`` is incompatible with the VAE ``scale_factors`` grid.
    Returns whether the temporal axis is known (``video_shape.frames > 0``). When
    False, callers skip frames-axis checks (duration not yet resolved).
    """
    if scale_factors.time < 1 or scale_factors.height < 1 or scale_factors.width < 1:
        raise ValueError(f"scale_factors must be >= 1 on each axis, got {scale_factors}")
    if video_shape.height < 1 or video_shape.width < 1:
        raise ValueError(f"video_shape height/width must be >= 1, got {video_shape.height}x{video_shape.width}")
    if video_shape.height % scale_factors.height != 0:
        raise ValueError(f"video height {video_shape.height} must be divisible by scale {scale_factors.height}")
    if video_shape.width % scale_factors.width != 0:
        raise ValueError(f"video width {video_shape.width} must be divisible by scale {scale_factors.width}")
    if video_shape.frames <= 0:
        return False
    if (video_shape.frames - 1) % scale_factors.time != 0:
        raise ValueError(f"video frames {video_shape.frames} must satisfy (frames - 1) % {scale_factors.time} == 0")
    return True


def _validate_size_axis(cfg: DimensionSizeConfig, factor: int, axis_name: str) -> None:
    """Pixel/frame size-axis legality vs VAE ``factor``."""
    if not cfg.is_tiled():
        return
    min_size = 2 * factor
    if cfg.tile_size < min_size:
        raise ValueError(f"{axis_name}.tile_size must be at least {min_size}, got {cfg.tile_size}")
    if cfg.tile_size % factor != 0:
        raise ValueError(f"{axis_name}.tile_size must be divisible by {factor}, got {cfg.tile_size}")
    if cfg.overlap % factor != 0:
        raise ValueError(f"{axis_name}.overlap must be divisible by {factor}, got {cfg.overlap}")


def _validate_count_axis(cfg: DimensionTilingConfig, latent_extent: int, axis_name: str) -> None:
    """Latent count-axis legality vs latent ``extent``."""
    if not cfg.is_tiled():
        return
    if cfg.num_tiles > latent_extent:
        raise ValueError(f"{axis_name}.num_tiles {cfg.num_tiles} exceeds latent {axis_name} extent {latent_extent}")
    # split_by_count requires overlap < tile_size; tile_size grows with extent/n.
    max_overlap = latent_extent - cfg.num_tiles
    if cfg.overlap > max_overlap:
        raise ValueError(
            f"{axis_name}.overlap {cfg.overlap} exceeds latent bound {max_overlap} "
            f"for extent {latent_extent} with {cfg.num_tiles} tiles"
        )


def _validate_overlap(
    tiling_config: TilingConfig,
    *,
    min_overlap_frames: int,
    min_overlap_pixels: int,
) -> None:
    """Raise if any tiled ``TileSizeConfig`` axis overlap is below the given floors."""
    if not isinstance(tiling_config, TileSizeConfig):
        return

    for axis_name, cfg, recommended, unit in (
        ("frames", tiling_config.frames, min_overlap_frames, "frames"),
        ("height", tiling_config.height, min_overlap_pixels, "px"),
        ("width", tiling_config.width, min_overlap_pixels, "px"),
    ):
        if cfg.is_tiled() and cfg.overlap < recommended:
            raise ValueError(f"{axis_name} overlap {cfg.overlap} {unit} is below the required {recommended} {unit}.")


def balanced_tile_split(num_tiles: int) -> tuple[int, int]:
    """Factor ``num_tiles`` into ``(small, large)`` as square as possible.
    ``small`` is the largest divisor not exceeding the square root, so
    ``small * large == num_tiles`` and ``small <= large``. E.g. 2 -> (1, 2),
    4 -> (2, 2), 8 -> (2, 4), 16 -> (4, 4). The caller decides which tiled
    dimension gets which factor.
    """
    if num_tiles < 1:
        raise ValueError(f"num_tiles must be >= 1, got {num_tiles}")
    small = next(d for d in range(math.isqrt(num_tiles), 0, -1) if num_tiles % d == 0)
    return small, num_tiles // small