Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reuse VAE tile blend weights in tiled encode and LTX-2 tiled decode (#15700)

Daniel Han87ea22f12443
diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.py
index ade63fb6948..327605b3cc3 100644
--- a/studio/backend/core/inference/diffusion_vae_tiling.py
+++ b/studio/backend/core/inference/diffusion_vae_tiling.py
@@ -167,16 +167,24 @@ def _geometry(vae: Any) -> tuple[int, int, int]:
def _cached_axis_weights(
- vae: Any, starts: list[int], tile: int, length: int, ratio: int, torch: Any, device: Any
+ vae: Any,
+ starts: list[int],
+ tile: int,
+ length: int,
+ ratio: int,
+ torch: Any,
+ device: Any,
+ margin: int = MARGIN_LATENTS,
+ ramp: int = RAMP_LATENTS,
):
- """``axis_weights`` memoised per VAE (a repeat decode syncs no device)."""
+ """``axis_weights`` memoised per VAE (a repeat decode or encode syncs no device)."""
cache = vae.__dict__.setdefault("_unsloth_wide_weights", {}) if hasattr(vae, "__dict__") else {}
- key = (tuple(starts), int(tile), int(length), int(ratio), str(device))
+ key = (tuple(starts), int(tile), int(length), int(ratio), str(device), int(margin), int(ramp))
hit = cache.get(key)
if hit is None:
if len(cache) >= 64:
cache.clear()
- hit = cache[key] = axis_weights(starts, tile, length, ratio, torch, device)
+ hit = cache[key] = axis_weights(starts, tile, length, ratio, torch, device, margin, ramp)
return hit
@@ -441,8 +449,8 @@ def tiled_encode(vae: Any, x: Any) -> Any:
if len(hs) == 1 and len(ws) == 1:
return _encode_tile(vae, x)
margin, ramp = ENCODE_MARGIN_LATENTS, ENCODE_RAMP_LATENTS
- wy = axis_weights(hs, tile, height, 1, torch, x.device, margin, ramp)
- wx = axis_weights(ws, tile, width, 1, torch, x.device, margin, ramp)
+ wy = _cached_axis_weights(vae, hs, tile, height, 1, torch, x.device, margin, ramp)
+ wx = _cached_axis_weights(vae, ws, tile, width, 1, torch, x.device, margin, ramp)
out, dtype = None, None
for i, y in enumerate(hs):
for j, xs in enumerate(ws):
diff --git a/studio/backend/core/inference/video_ltx2_vae_tiles.py b/studio/backend/core/inference/video_ltx2_vae_tiles.py
index f69bdc07741..720edc5ed44 100644
--- a/studio/backend/core/inference/video_ltx2_vae_tiles.py
+++ b/studio/backend/core/inference/video_ltx2_vae_tiles.py
@@ -82,6 +82,23 @@ def axis_weights(
)
+_WEIGHTS_ATTR = "_unsloth_ltx2_tile_weights"
+
+
+def _cached_weights(
+ vae: Any, starts: list[int], tile: int, length: int, scale: int, torch: Any, device: Any
+) -> list:
+ """``axis_weights`` memoised per VAE (a repeat decode syncs no device)."""
+ cache = vae.__dict__.setdefault(_WEIGHTS_ATTR, {}) if hasattr(vae, "__dict__") else {}
+ key = (tuple(starts), int(tile), int(length), int(scale), str(device))
+ hit = cache.get(key)
+ if hit is None:
+ if len(cache) >= 64:
+ cache.clear()
+ hit = cache[key] = axis_weights(starts, tile, length, scale, torch, device)
+ return hit
+
+
def output_frames(vae: Any, latent_frames: int) -> int:
ratio = int(getattr(vae, "temporal_compression_ratio", 8) or 8)
return (max(1, int(latent_frames)) - 1) * ratio + 1
@@ -198,8 +215,8 @@ def _decode_tiles(
hs, ws = tile_starts(height, th), tile_starts(width, tw)
if len(hs) == 1 and len(ws) == 1:
return vae.decoder(z, temb, causal = causal)
- wy = axis_weights(hs, th, height, ratio, torch, z.device)
- wx = axis_weights(ws, tw, width, ratio, torch, z.device)
+ wy = _cached_weights(vae, hs, th, height, ratio, torch, z.device)
+ wx = _cached_weights(vae, ws, tw, width, ratio, torch, z.device)
out, dtype = None, None
for i, y in enumerate(hs):
for j, x in enumerate(ws):
@@ -324,4 +341,5 @@ def uninstall(vae: Any) -> None:
else:
vae.tiled_decode = own
vae.__dict__.pop("_unsloth_last_decode_tile", None)
+ vae.__dict__.pop(_WEIGHTS_ATTR, None)
vae._unsloth_wide_tiles = False
diff --git a/studio/backend/tests/test_diffusion_vae_tiling.py b/studio/backend/tests/test_diffusion_vae_tiling.py
index fd47bcdd022..e642c84b9c6 100644
--- a/studio/backend/tests/test_diffusion_vae_tiling.py
+++ b/studio/backend/tests/test_diffusion_vae_tiling.py
@@ -382,6 +382,42 @@ def test_encode_within_one_tile_is_untiled_and_kill_switch_keeps_stock(monkeypat
assert "tiled_encode" not in vae.__dict__
+def test_repeat_tiled_encode_reuses_its_blend_weights(monkeypatch):
+ monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)
+ vae = _diffusers_vae()
+ x = torch.rand(1, 3, 1, 256, 1088, generator = torch.Generator().manual_seed(7)) * 2 - 1
+ real, calls = vt.axis_weights, []
+ monkeypatch.setattr(vt, "axis_weights", lambda *a: calls.append(a) or real(*a))
+ with torch.no_grad():
+ vae.enable_tiling()
+ assert vt.install(vae)
+ first = vae._encode(x)
+ built = len(calls)
+ second = vae._encode(x)
+ assert built >= 1 and len(calls) == built
+ assert torch.equal(first, second)
+ fresh = vt.axis_weights(
+ vt.tile_starts(68, vt.ENCODE_TILE_LATENTS, vt.ENCODE_OVERLAP_LATENTS),
+ vt.ENCODE_TILE_LATENTS,
+ 68,
+ 1,
+ torch,
+ x.device,
+ vt.ENCODE_MARGIN_LATENTS,
+ vt.ENCODE_RAMP_LATENTS,
+ )
+ cached = [v for k, v in vae.__dict__["_unsloth_wide_weights"].items() if k[2] == 68]
+ assert len(cached) == 1 and all(torch.equal(a, b) for a, b in zip(cached[0], fresh))
+ # Decode blend over the same tiles: own entry (narrower margin / ramp).
+ ws = vt.tile_starts(68, vt.ENCODE_TILE_LATENTS, vt.ENCODE_OVERLAP_LATENTS)
+ decode = vt._cached_axis_weights(vae, ws, vt.ENCODE_TILE_LATENTS, 68, 1, torch, x.device)
+ want = real(ws, vt.ENCODE_TILE_LATENTS, 68, 1, torch, x.device)
+ assert all(torch.equal(a, b) for a, b in zip(decode, want))
+ assert not all(torch.equal(a, b) for a, b in zip(decode, fresh))
+ vt.uninstall(vae)
+ assert "_unsloth_wide_weights" not in vae.__dict__
+
+
def _decoded_latents(height, width, th, tw):
return len(vt.tile_starts(height, th)) * th * len(vt.tile_starts(width, tw)) * tw
diff --git a/studio/backend/tests/test_video_ltx2_vae_tiles.py b/studio/backend/tests/test_video_ltx2_vae_tiles.py
index 57b6ef28b16..fe06ccc8a8f 100644
--- a/studio/backend/tests/test_video_ltx2_vae_tiles.py
+++ b/studio/backend/tests/test_video_ltx2_vae_tiles.py
@@ -183,6 +183,27 @@ def test_fp32_accumulator_only_when_budgeted(free, monkeypatch, free_bytes, fp32
assert (torch.bfloat16 if fp32_accum else torch.float32) not in seen
+def test_repeat_tiled_decode_reuses_its_blend_weights(free, monkeypatch):
+ vae = _ltx_vae()
+ z = torch.randn(1, 8, 2, 22, 38, generator = torch.Generator().manual_seed(1))
+ vae.enable_tiling()
+ free["bytes"] = 0
+ assert vt.install(vae)
+ real, calls = vt.axis_weights, []
+ monkeypatch.setattr(vt, "axis_weights", lambda *a: calls.append(a) or real(*a))
+ first = _decode(vae, z)
+ assert vae._unsloth_last_decode_tile == (16, 16) and len(calls) == 2
+ second = _decode(vae, z)
+ assert len(calls) == 2 and torch.equal(first, second)
+ for (starts, tile, length, scale, _t, device), got in zip(
+ calls, vae.__dict__[vt._WEIGHTS_ATTR].values()
+ ):
+ want = real(starts, tile, length, scale, torch, device)
+ assert all(torch.equal(a, b) and a.device == b.device for a, b in zip(got, want))
+ vt.uninstall(vae)
+ assert vt._WEIGHTS_ATTR not in vae.__dict__
+
+
def test_install_is_scoped_idempotent_and_reversible(monkeypatch):
monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)
vae = _ltx_vae()