Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: seam-free LTX-2.3 video decode (untiled when it fits, VRAM-sized tiles otherwise) (#12698)

Daniel Han32b8d92a041c
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index d0f844c1f78..444c35d582b 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -6630,6 +6630,14 @@ class VideoBackend:
vae_tiling = True
except Exception as exc: # noqa: BLE001 -- tiling is an optimisation only
logger.warning("video.vae_tiling_failed: %s", exc)
+ # LTX-2's stock tiles (16 latents, a 2-latent blend) leave seam lines: tiles sized to free VRAM, wide overlaps.
+ if vae_tiling:
+ try:
+ from .video_ltx2_vae_tiles import install as install_ltx2_vae_tiles
+ if install_ltx2_vae_tiles(getattr(pipe, "vae", None), logger):
+ speed_optims += ("vae_wide_tiles",)
+ except Exception as exc: # noqa: BLE001 - keep the stock tiled decode
+ logger.warning("video.vae_wide_tiles: not installed: %s", exc)
# Resident: decode untiled when it fits, tiled as fallback. Not on SPEED_OFF, which must stay bit-identical.
if (
offload_policy == "none"
diff --git a/studio/backend/core/inference/video_ltx2_vae_tiles.py b/studio/backend/core/inference/video_ltx2_vae_tiles.py
new file mode 100644
index 00000000000..2d516bc96ec
--- /dev/null
+++ b/studio/backend/core/inference/video_ltx2_vae_tiles.py
@@ -0,0 +1,326 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Seam-free LTX-2 / 2.3 VAE decode: the fewest, largest tiles that fit free VRAM (untiled when it fits).
+
+Stock tiles (16 latents, 2-latent blend) are narrower than the decoder's receptive field, so every seam shows a line.
+Kill switch ``UNSLOTH_VIDEO_VAE_WIDE_TILES=0``; ``UNSLOTH_VIDEO_VAE_UNTILED=0`` keeps >= 2 tiles. Tiles span every
+frame: Studio never decodes LTX framewise.
+"""
+
+from __future__ import annotations
+
+import os
+import types
+from typing import Any, Optional
+
+WIDE_TILES_ENV = "UNSLOTH_VIDEO_VAE_WIDE_TILES"
+
+# Stock tile side: the smallest tile, so the tightest tier never needs more per tile than stock.
+MIN_TILE_LATENTS = 16
+OVERLAP_LATENTS = 8
+MARGIN_LATENTS = 3
+RAMP_LATENTS = 2
+
+# Decode peak MiB per output frame x latent pixel (bf16, B200, 2.3 VAE): measured 0.102-0.105 stock, ~0.071 fused.
+DECODE_MIB_PER_FRAME_LATENT = 0.11
+DECODE_MIB_PER_FRAME_LATENT_FUSED = 0.075
+# Coefficients already sit ~5% over measured peaks; an OOM still retries in stock-size tiles.
+_MARGIN = 1.05
+_MARGIN_BYTES = 512 * 2**20
+
+_VAE_CLASSES = frozenset({"AutoencoderKLLTX2Video"})
+
+
+def wide_tiles_disabled() -> bool:
+ return (os.environ.get(WIDE_TILES_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _untiled_allowed() -> bool:
+ from .video_vae_untiled import UNTILED_ENV
+ return (os.environ.get(UNTILED_ENV) or "").strip().lower() not in ("0", "false", "no", "off")
+
+
+def tile_starts(
+ length: int,
+ tile: int,
+ overlap: Optional[int] = None,
+) -> list[int]:
+ """Evenly spread starts of the fewest tiles covering ``length`` with overlaps >= ``overlap``; last ends at length."""
+ overlap = OVERLAP_LATENTS if overlap is None else overlap
+ length, tile, overlap = int(length), int(tile), int(overlap)
+ if length <= tile:
+ return [0]
+ if tile <= overlap or overlap < 0:
+ raise ValueError(f"tile {tile} must exceed overlap {overlap}")
+ count = -(-(length - overlap) // (tile - overlap))
+ span = length - tile
+ return [(i * span) // (count - 1) for i in range(count)]
+
+
+def axis_weights(
+ starts: list[int],
+ tile: int,
+ length: int,
+ scale: int,
+ torch: Any,
+ device: Any,
+ margin: Optional[int] = None,
+ ramp: Optional[int] = None,
+) -> list:
+ """Per-tile fp32 weights along one axis summing to 1: 0 within ``margin`` of a shared edge, then a linear ``ramp``."""
+ margin = MARGIN_LATENTS if margin is None else margin
+ ramp = RAMP_LATENTS if ramp is None else ramp
+ size = min(tile, length) * scale
+ pos = (torch.arange(size, dtype = torch.float64, device = "cpu") + 0.5) / scale
+ total = torch.zeros(length * scale, dtype = torch.float64, device = "cpu")
+ weights = []
+ for s in starts:
+ w = torch.ones(size, dtype = torch.float64, device = "cpu")
+ if s > 0:
+ w = torch.minimum(w, ((pos - margin) / ramp).clamp(0, 1))
+ if s + tile < length:
+ w = torch.minimum(w, ((min(tile, length) - pos - margin) / ramp).clamp(0, 1))
+ total[s * scale : s * scale + size] += w
+ weights.append(w)
+ if float(total.min()) <= 0.0:
+ raise ValueError(f"tiles {starts} leave pixels without weight on a {length}-latent axis")
+ # float64 on CPU: MPS has no float64.
+ return [
+ (w / total[s * scale : s * scale + size]).float().to(device)
+ for w, s in zip(weights, starts)
+ ]
+
+
+def output_frames(vae: Any, latent_frames: int) -> int:
+ ratio = int(getattr(vae, "temporal_compression_ratio", 8) or 8)
+ return (max(1, int(latent_frames)) - 1) * ratio + 1
+
+
+def tile_bytes(
+ frames: int,
+ th: int,
+ tw: int,
+ batch: int = 1,
+ itemsize: int = 2,
+ fused: bool = False,
+) -> int:
+ """Estimated extra bytes of one decode of a (th, tw)-latent tile over ``frames`` output frames."""
+ coef = DECODE_MIB_PER_FRAME_LATENT_FUSED if fused else DECODE_MIB_PER_FRAME_LATENT
+ return int(coef * 2**20 * max(2, itemsize) / 2 * max(1, batch) * frames * th * tw)
+
+
+def _axis_side(length: int, count: int) -> Optional[int]:
+ """Smallest tile side (>= MIN_TILE_LATENTS) covering ``length`` in ``count`` tiles with the overlap."""
+ if count == 1:
+ return length
+ side = max(MIN_TILE_LATENTS, -(-(length + (count - 1) * OVERLAP_LATENTS) // count))
+ return side if side < length else None
+
+
+def choose_tiles(
+ height: int,
+ width: int,
+ fits: Any,
+ allow_single: bool = True,
+) -> tuple[int, int]:
+ """Tile (height, width) in latents: the fewest decoded latents (overlaps counted) among tiles ``fits(th, tw)``
+ accepts. Nothing fits: the stock 16-latent tile."""
+ floor = (min(MIN_TILE_LATENTS, height), min(MIN_TILE_LATENTS, width))
+ best, best_cost = floor, None
+ for nh in range(1, len(tile_starts(height, floor[0])) + 1):
+ th = _axis_side(height, nh)
+ if th is None:
+ continue
+ for nw in range(1, len(tile_starts(width, floor[1])) + 1):
+ tw = _axis_side(width, nw)
+ if (
+ tw is None
+ or (not allow_single and th == height and tw == width)
+ or not fits(th, tw)
+ ):
+ continue
+ n = len(tile_starts(height, th)) * len(tile_starts(width, tw))
+ cost = (n * th * tw, n)
+ if best_cost is None or cost < best_cost:
+ best, best_cost = (th, tw), cost
+ return best
+
+
+def _free_bytes(device: Any) -> Optional[int]:
+ from .video_vae_untiled import _free_bytes as free_bytes
+ return free_bytes(device)
+
+
+def _itemsize(vae: Any) -> int:
+ from .video_vae_untiled import _decoder_itemsize
+ return _decoder_itemsize(vae)
+
+
+def plan_tiles(
+ vae: Any,
+ z: Any,
+ free: Optional[int] = None,
+) -> tuple[int, int]:
+ """The decode tile for ``z`` (B, C, T, H, W) given ``free`` bytes (read from the device when None)."""
+ return _plan(vae, z, free)[0]
+
+
+def _plan(
+ vae: Any,
+ z: Any,
+ free: Optional[int] = None,
+) -> tuple[tuple[int, int], bool]:
+ """``(tile, budgeted)``; not budgeted (nothing fit) means no room was priced in for the fp32 accumulator."""
+ batch, _, latent_frames, height, width = (int(x) for x in z.shape)
+ frames = output_frames(vae, latent_frames)
+ ratio = int(vae.spatial_compression_ratio)
+ if free is None:
+ free = _free_bytes(getattr(z, "device", None))
+ allow_single = _untiled_allowed()
+ if free is None:
+ return choose_tiles(height, width, lambda th, tw: False, allow_single), False
+ itemsize = _itemsize(vae)
+ fused = bool(getattr(vae, "_unsloth_vae_fused_installed", 0))
+ accum = 4 * batch * 3 * frames * height * ratio * width * ratio
+
+ def fits(th: int, tw: int) -> bool:
+ need = tile_bytes(frames, th, tw, batch, itemsize, fused) * _MARGIN + _MARGIN_BYTES
+ return need + (0 if (th == height and tw == width) else accum) <= free
+
+ tile = choose_tiles(height, width, fits, allow_single)
+ return tile, fits(*tile)
+
+
+def _decode_tiles(
+ vae: Any,
+ z: Any,
+ temb: Any,
+ causal: Any,
+ th: int,
+ tw: int,
+ fp32_accum: bool = True,
+) -> Any:
+ import torch
+
+ _, _, _, height, width = z.shape
+ ratio = int(vae.spatial_compression_ratio)
+ hs, ws = tile_starts(height, th), tile_starts(width, tw)
+ if len(hs) == 1 and len(ws) == 1:
+ return vae.decoder(z, temb, causal = causal)
+ wy = axis_weights(hs, th, height, ratio, torch, z.device)
+ wx = axis_weights(ws, tw, width, ratio, torch, z.device)
+ out, dtype = None, None
+ for i, y in enumerate(hs):
+ for j, x in enumerate(ws):
+ tile = vae.decoder(z[:, :, :, y : y + th, x : x + tw], temb, causal = causal)
+ if out is None:
+ dtype = tile.dtype
+ # Unbudgeted (nothing fit): accumulate in the tile dtype, else the fp32 buffer exceeds stock's peak.
+ out = torch.zeros(
+ (*tile.shape[:3], height * ratio, width * ratio),
+ dtype = torch.float32 if fp32_accum else dtype,
+ device = tile.device,
+ )
+ w = wy[i].view(-1, 1) * wx[j].view(1, -1)
+ out[..., y * ratio : (y + th) * ratio, x * ratio : (x + tw) * ratio].addcmul_(
+ tile.float() if fp32_accum else tile, w
+ )
+ del tile
+ dec = out.to(dtype)
+ del out
+ return dec
+
+
+def tiled_decode(
+ vae: Any,
+ z: Any,
+ temb: Any = None,
+ causal: Any = None,
+ return_dict: bool = True,
+) -> Any:
+ """Decode ``z`` in the tiles ``plan_tiles`` picks; on OOM, once more in the stock-size 16-latent tiles."""
+ from diffusers.models.autoencoders.vae import DecoderOutput
+
+ from .diffusion_batched import is_oom_error
+
+ (th, tw), budgeted = _plan(vae, z)
+ floor = (min(MIN_TILE_LATENTS, int(z.shape[-2])), min(MIN_TILE_LATENTS, int(z.shape[-1])))
+ stats = vae.__dict__.setdefault(
+ "_unsloth_wide_tiles_stats", {"untiled": 0, "tiled": 0, "oom_fallback": 0}
+ )
+ vae._unsloth_last_decode_tile = (th, tw)
+ failed = False
+ try:
+ dec = _decode_tiles(vae, z, temb, causal, th, tw, budgeted)
+ except Exception as exc: # noqa: BLE001
+ if (th, tw) == floor or not is_oom_error(exc):
+ raise
+ failed = True
+ if failed:
+ # Retried outside the handler so the failed attempt's tensors (held by the traceback) are gone first.
+ import torch
+
+ stats["oom_fallback"] += 1
+ torch.cuda.empty_cache()
+ th, tw = floor
+ vae._unsloth_last_decode_tile = floor
+ dec = _decode_tiles(vae, z, temb, causal, th, tw, False)
+ stats["untiled" if (th, tw) == tuple(int(x) for x in z.shape[-2:]) else "tiled"] += 1
+ if not return_dict:
+ return (dec,)
+ return DecoderOutput(sample = dec)
+
+
+def _covered(vae: Any) -> bool:
+ if vae is None or type(vae).__name__ not in _VAE_CLASSES:
+ return False
+ try:
+ ratio = int(vae.spatial_compression_ratio)
+ except (AttributeError, TypeError, ValueError):
+ return False
+ return ratio == 32 and callable(getattr(vae, "decoder", None)) and hasattr(vae, "use_tiling")
+
+
+def install(vae: Any, logger: Any = None) -> bool:
+ """Route ``vae``'s tiled decode through tiles sized to the free VRAM. Idempotent; False when not covered."""
+ if not _covered(vae) or wide_tiles_disabled():
+ return False
+ if getattr(vae, "_unsloth_wide_tiles", False):
+ return True
+ stock = vae.tiled_decode
+
+ def _tiled_decode(
+ self: Any,
+ z: Any,
+ temb: Any = None,
+ causal: Any = None,
+ return_dict: bool = True,
+ ) -> Any:
+ if wide_tiles_disabled():
+ return stock(z, temb, causal = causal, return_dict = return_dict)
+ return tiled_decode(self, z, temb, causal = causal, return_dict = return_dict)
+
+ vae._unsloth_wide_tiles_own = vae.__dict__.get("tiled_decode")
+ vae.tiled_decode = types.MethodType(_tiled_decode, vae)
+ vae._unsloth_wide_tiles = True
+ if logger is not None:
+ logger.info(
+ "video.vae_wide_tiles: %s decodes in tiles sized to free VRAM (>= %d latents, %d-latent overlaps)",
+ type(vae).__name__,
+ MIN_TILE_LATENTS,
+ OVERLAP_LATENTS,
+ )
+ return True
+
+
+def uninstall(vae: Any) -> None:
+ if not getattr(vae, "_unsloth_wide_tiles", False):
+ return
+ own = vae.__dict__.pop("_unsloth_wide_tiles_own", None)
+ if own is None:
+ vae.__dict__.pop("tiled_decode", None)
+ else:
+ vae.tiled_decode = own
+ vae.__dict__.pop("_unsloth_last_decode_tile", None)
+ vae._unsloth_wide_tiles = False
diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.py
index 29d10d9ff18..10248493fb9 100644
--- a/studio/backend/core/inference/video_vae_untiled.py
+++ b/studio/backend/core/inference/video_vae_untiled.py
@@ -23,6 +23,13 @@ _OUTPUT_SCALE = {
"wan2.2-ti2v-5b": (16, 4),
"wan2.2-t2v-a14b": (8, 4),
}
+# LTX-2 / 2.3 holds every frame: peak per output frame x latent pixel, 0.102-0.105 MiB measured bf16 (B200).
+_BYTES_PER_FRAME_LATENT_PIXEL = {
+ "ltx-2": 0.11 * 2**20,
+}
+_TEMPORAL_SCALE = {
+ "ltx-2": 8,
+}
_MARGIN = 1.25
_MARGIN_BYTES = 2 * 2**30
@@ -37,11 +44,17 @@ def untiled_decode_bytes(
itemsize: int = 2,
) -> Optional[int]:
"""Extra bytes of an untiled decode of ``latent_shape`` (B, C, T, h, w), or None if unmeasured."""
- coef = _BYTES_PER_LATENT_PIXEL.get(family)
- if coef is None or len(latent_shape) != 5:
+ if len(latent_shape) != 5:
return None
batch, _, latent_frames, height, width = (int(x) for x in latent_shape)
batch, itemsize = max(1, batch), max(2, itemsize)
+ per_frame = _BYTES_PER_FRAME_LATENT_PIXEL.get(family)
+ if per_frame is not None:
+ frames = (max(1, latent_frames) - 1) * _TEMPORAL_SCALE[family] + 1
+ return int(per_frame * itemsize / 2 * batch * frames * height * width)
+ coef = _BYTES_PER_LATENT_PIXEL.get(family)
+ if coef is None:
+ return None
spatial, temporal = _OUTPUT_SCALE[family]
frames = (max(1, latent_frames) - 1) * temporal + 1
output = 2 * batch * 3 * frames * height * spatial * width * spatial * itemsize
@@ -82,7 +95,9 @@ def install_untiled_decode(
decode = getattr(vae, "decode", None)
if not _enabled() or vae is None or not callable(decode):
return False
- if family not in _BYTES_PER_LATENT_PIXEL or not getattr(vae, "use_tiling", False):
+ if (
+ family not in _BYTES_PER_LATENT_PIXEL and family not in _BYTES_PER_FRAME_LATENT_PIXEL
+ ) or not getattr(vae, "use_tiling", False):
return False
if getattr(decode, "_unsloth_untiled_decode", False):
return True
diff --git a/studio/backend/tests/test_video_ltx2_vae_tiles.py b/studio/backend/tests/test_video_ltx2_vae_tiles.py
new file mode 100644
index 00000000000..865c1e2b6ae
--- /dev/null
+++ b/studio/backend/tests/test_video_ltx2_vae_tiles.py
@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LTX-2 / 2.3 VAE decode without tile seams (video_ltx2_vae_tiles) and the ltx-2 untiled estimate."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+if not hasattr(diffusers, "AutoencoderKLLTX2Video"):
+ pytest.skip("diffusers without LTX-2", allow_module_level = True)
+
+from core.inference import video_ltx2_vae_tiles as vt # noqa: E402
+from core.inference import video_vae_untiled as U # noqa: E402
+
+GIB = 2**30
+
+
+def _ltx_vae():
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKLLTX2Video(
+ in_channels = 3,
+ out_channels = 3,
+ latent_channels = 8,
+ block_out_channels = (8, 8, 8, 8),
+ decoder_block_out_channels = (8, 16, 32),
+ layers_per_block = (1, 1, 1, 1, 1),
+ decoder_layers_per_block = (1, 1, 1, 1),
+ spatial_compression_ratio = 32,
+ temporal_compression_ratio = 8,
+ ).eval()
+ assert vae.spatial_compression_ratio == 32
+ return vae
+
+
+def _line_error(x, ref):
+ d = (x - ref).abs().mean(1)[0] # (T, H, W)
+ return max(float(d.mean(1).max()), float(d.mean(2).max()))
+
+
+def _decode(vae, z, *args, **kwargs):
+ with torch.no_grad():
+ return vae.decode(z, *args, return_dict = False, **kwargs)[0]
+
+
+def free(monkeypatch):
+ """Set the free bytes the planner sees."""
+ box = {"bytes": 0}
+ monkeypatch.setattr(vt, "_free_bytes", lambda device: box["bytes"])
+ monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)
+ monkeypatch.delenv(U.UNTILED_ENV, raising = False)
+ return box
+
+
+def test_tight_memory_tiles_have_no_seam_lines(free, monkeypatch):
+ vae = _ltx_vae()
+ z = torch.randn(
+ 1, 8, 2, 22, 38, generator = torch.Generator().manual_seed(1)
+ ) # 1216x704, 9 frames
+ vae.use_tiling = False
+ untiled = _decode(vae, z)
+ vae.enable_tiling() # what Studio does on every LTX load
+ stock = _decode(vae, z)
+ free["bytes"] = 0 # nothing fits: the stock-size 16-latent tiles, with the wide overlap
+ assert vt.install(vae)
+ wide = _decode(vae, z)
+ assert vae._unsloth_last_decode_tile == (16, 16)
+ assert wide.shape == untiled.shape == stock.shape
+ stock_err, wide_err = _line_error(stock, untiled), _line_error(wide, untiled)
+ assert stock_err > 3 * wide_err, (stock_err, wide_err)
+ monkeypatch.setenv(vt.WIDE_TILES_ENV, "0")
+ assert torch.equal(
+ _decode(vae, z), stock
+ ) # kill switch set after install: the stock tiled decode
+
+
+def test_whole_latent_fits_decodes_untiled(free, monkeypatch):
+ vae = _ltx_vae()
+ z = torch.randn(
+ 1, 8, 2, 16, 24, generator = torch.Generator().manual_seed(2)
+ ) # 768x512, the default preset
+ vae.use_tiling = False
+ untiled = _decode(vae, z)
+ vae.enable_tiling()
+ assert vt.install(vae)
+ free["bytes"] = 1000 * GIB
+ assert torch.equal(_decode(vae, z), untiled)
+ assert vae._unsloth_last_decode_tile == (16, 24)
+ assert vae._unsloth_wide_tiles_stats["untiled"] == 1
+ monkeypatch.setenv(U.UNTILED_ENV, "0")
+ _decode(vae, z)
+ assert vae._unsloth_last_decode_tile != (16, 24)
+
+
+def test_more_free_memory_means_fewer_larger_tiles(free, monkeypatch):
+ vae = _ltx_vae()
+ monkeypatch.setattr(vt, "_itemsize", lambda vae: 2) # the bf16 decoder Studio loads
+ z = torch.zeros(1, 128, 16, 22, 38) # 1216x704x121, the largest the UI offers
+ for fused in (False, True):
+ vae._unsloth_vae_fused_installed = int(fused)
+ sizes = []
+ for gib in (0, 4, 6, 7, 9, 12, 20):
+ tile = vt.plan_tiles(vae, z, free = gib * GIB)
+ n = len(vt.tile_starts(22, tile[0])) * len(vt.tile_starts(38, tile[1]))
+ need = vt.tile_bytes(121, *tile, fused = fused) * vt._MARGIN + vt._MARGIN_BYTES
+ accum = 0 if n == 1 else 4 * 3 * 121 * 704 * 1216
+ if tile != (16, 16):
+ assert need + accum <= gib * GIB
+ sizes.append(n)
+ assert sizes == sorted(sizes, reverse = True) and sizes[0] > 1 and sizes[-1] == 1, sizes
+
+
+def test_temb_and_causal_reach_every_tile(free):
+ vae = _ltx_vae()
+ vae.enable_tiling()
+ assert vt.install(vae)
+ seen = []
+ decoder = vae.decoder
+ forward = decoder.forward
+
+ def spy(
+ x,
+ temb = None,
+ causal = None,
+ ):
+ seen.append((temb, causal))
+ return forward(x, temb, causal = causal)
+
+ decoder.forward = spy
+ temb = torch.tensor([0.05])
+ _decode(vae, torch.randn(1, 8, 1, 16, 40), temb, causal = False)
+ assert len(seen) > 1 and all(t is temb and c is False for t, c in seen)
+
+
+def test_oom_retries_in_stock_size_tiles(free, monkeypatch):
+ vae = _ltx_vae()
+ z = torch.randn(1, 8, 1, 22, 38)
+ vae.enable_tiling()
+ assert vt.install(vae)
+ free["bytes"] = 1000 * GIB
+ calls = []
+ real = vt._decode_tiles
+
+ def flaky(
+ vae_,
+ z_,
+ temb,
+ causal,
+ th,
+ tw,
+ fp32_accum = True,
+ ):
+ calls.append((th, tw, fp32_accum))
+ if len(calls) == 1:
+ raise torch.cuda.OutOfMemoryError("fake")
+ return real(vae_, z_, temb, causal, th, tw, fp32_accum)
+
+ monkeypatch.setattr(vt, "_decode_tiles", flaky)
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ out = _decode(vae, z)
+ assert calls == [(22, 38, True), (16, 16, False)] and out.shape[-2:] == (704, 1216)
+ assert vae._unsloth_wide_tiles_stats["oom_fallback"] == 1
+
+
[email protected]("free_bytes, fp32_accum", [(0, False), (None, False), (1000 * GIB, True)])
+def test_fp32_accumulator_only_when_budgeted(free, monkeypatch, free_bytes, fp32_accum):
+ # Unbudgeted plans must not add a full-frame fp32 buffer on top of stock-size tiles.
+ vae = _ltx_vae().to(torch.bfloat16)
+ z = torch.randn(1, 8, 1, 22, 38, dtype = torch.bfloat16)
+ vae.enable_tiling()
+ assert vt.install(vae)
+ free["bytes"] = free_bytes
+ monkeypatch.setenv(U.UNTILED_ENV, "0") # keep >= 2 tiles so the accumulator is used
+ seen = []
+ zeros = torch.zeros
+
+ def spy(*args, **kwargs):
+ seen.append(kwargs.get("dtype"))
+ return zeros(*args, **kwargs)
+
+ monkeypatch.setattr(torch, "zeros", spy)
+ out = _decode(vae, z)
+ assert out.dtype == torch.bfloat16 and torch.isfinite(out.float()).all()
+ assert (torch.float32 if fp32_accum else torch.bfloat16) in seen
+ assert (torch.bfloat16 if fp32_accum else torch.float32) not in seen
+
+
+def test_install_is_scoped_idempotent_and_reversible(monkeypatch):
+ monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)
+ vae = _ltx_vae()
+ assert vt.install(vae) and vt.install(vae)
+ assert "tiled_decode" in vae.__dict__
+ vt.uninstall(vae)
+ assert "tiled_decode" not in vae.__dict__
+ monkeypatch.setenv(vt.WIDE_TILES_ENV, "0")
+ assert not vt.install(_ltx_vae())
+ monkeypatch.delenv(vt.WIDE_TILES_ENV)
+ assert not vt.install(
+ diffusers.AutoencoderKLWan(base_dim = 8, z_dim = 4, dim_mult = [1, 1], num_res_blocks = 1)
+ )
+
+
[email protected]("length", list(range(1, 130)))
+def test_tile_layout_full_tiles_and_weights_sum_to_one(length):
+ for tile in sorted({16, 20, 24, 32, max(16, length - 1)}):
+ starts = vt.tile_starts(length, tile)
+ if length > tile:
+ assert starts[0] == 0 and starts[-1] + tile == length
+ assert all(tile - (b - a) >= vt.OVERLAP_LATENTS for a, b in zip(starts, starts[1:]))
+ weights = vt.axis_weights(starts, tile, length, 4, torch, "cpu")
+ total = torch.zeros(length * 4, dtype = torch.float64)
+ for s, w in zip(starts, weights):
+ total[s * 4 : s * 4 + w.numel()] += w.double()
+ assert torch.allclose(total, torch.ones_like(total), atol = 1e-6)
+
+
+# Untiled decode peaks of the LTX-2.3 VAE (bf16, B200, MiB): (w, h, frames) -> peak. The estimate must cover each.
+MEASURED = {
+ (768, 512, 25): 1011,
+ (768, 512, 121): 4755,
+ (1216, 704, 49): 4239,
+ (1216, 704, 121): 10352,
+ (896, 896, 121): 9708,
+ (1216, 704, 241): 20541,
+}
+
+
[email protected]("shape,peak", list(MEASURED.items()))
+def test_ltx2_untiled_estimate_covers_measured_peaks(shape, peak):
+ w, h, frames = shape
+ latent = (1, 128, (frames - 1) // 8 + 1, h // 32, w // 32)
+ need = U.untiled_decode_bytes("ltx-2", latent, itemsize = 2)
+ assert need is not None and peak * 2**20 <= need <= 1.2 * peak * 2**20
+ assert vt.tile_bytes(frames, h // 32, w // 32) == need
+ assert U.untiled_decode_bytes("ltx-2", latent, itemsize = 4) == 2 * need
+
+
+class _FakeLTX:
+ def __init__(self):
+ self.decoder = torch.nn.Conv3d(1, 1, 1).to(torch.bfloat16)
+ self.use_tiling = True
+ self.calls = []
+
+ def decode(
+ self,
+ z,
+ temb = None,
+ return_dict = False,
+ ):
+ self.calls.append((self.use_tiling, temb))
+ return ("tiled" if self.use_tiling else "untiled",)
+
+
+def test_resident_ltx2_decodes_untiled_when_it_fits(monkeypatch):
+ import types
+
+ monkeypatch.delenv(U.UNTILED_ENV, raising = False)
+ z = torch.zeros(1, 128, 16, 22, 38)
+ need = U.untiled_decode_bytes("ltx-2", tuple(z.shape), itemsize = 2)
+ vae = _FakeLTX()
+ monkeypatch.setattr(U, "_free_bytes", lambda device: 100 * GIB)
+ assert U.install_untiled_decode(types.SimpleNamespace(vae = vae), "ltx-2")
+ assert vae.decode(z, "t", return_dict = False) == ("untiled",) and vae.calls == [(False, "t")]
+ monkeypatch.setattr(U, "_free_bytes", lambda device: int(need * 1.25 + 2 * GIB) - 1)
+ assert vae.decode(z, "t") == ("tiled",)
+
+
+def test_video_load_installs_wide_tiles_on_every_tiling_load():
+ src = open(os.path.join(os.path.dirname(vt.__file__), "video.py"), encoding = "utf-8").read()
+ tiling = src.index("pipe.vae.enable_tiling()")
+ install = src.index("from .video_ltx2_vae_tiles import install")
+ resident = src.index("from .video_vae_untiled import install_untiled_decode")
+ assert (
+ tiling < install < resident
+ ) # after tiling is enabled, before (so beneath) the resident untiled wrapper
+
+
+def test_blend_weights_never_put_float64_on_the_device(monkeypatch):
+ made = []
+ for name in ("arange", "zeros", "ones"):
+ real = getattr(torch, name)
+
+ def spy(
+ *args,
+ _real = real,
+ **kwargs,
+ ):
+ made.append((kwargs.get("dtype"), str(kwargs.get("device", "cpu"))))
+ return _real(*args, **kwargs)
+
+ monkeypatch.setattr(torch, name, spy)
+ w = vt.axis_weights(vt.tile_starts(38, 16), 16, 38, 4, torch, torch.device("meta"))
+ assert all(x.device.type == "meta" and x.dtype == torch.float32 for x in w)
+ assert all(dev == "cpu" for dtype, dev in made if dtype == torch.float64)