Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio Images: decide VAE tiling at the decode and stream resident groups there, so offloaded edits decode untiled (#15006)
Daniel Hanb0c7f6829c20
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 27d2fe9fa68..c1778abd7ea 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -2543,6 +2543,302 @@ def _drop_measured_reserve_for_layer_split(pipe: Any, plan: Any) -> bool:return True+RELEASE_BEFORE_TILING_ENV = "UNSLOTH_DIFFUSION_RELEASE_BEFORE_TILING"+RELEASE_BEFORE_TILING_LAYER = "release_before_tiling"+RELEASE_BEFORE_TILING_STATS = "_unsloth_release_before_tiling_stats"+++def _release_before_tiling_off() -> bool:+ return os.environ.get(RELEASE_BEFORE_TILING_ENV, "").strip().lower() in (+ "0",+ "false",+ "no",+ "off",+ )+++def _untiled_decode_shortfall_mib(vae: Any, z: Any) -> Optional[int]:+ """MiB short for decoding ``z`` untiled, by ``decode_tile_budget``'s rule on the whole frame. None off CUDA,+ on unknown figures, or when ``decode_platform_cap`` is under the frame."""+ from .diffusion_vae_tiling import (+ FREE_FRACTION,+ _free_mib,+ _fused,+ _geometry,+ decode_mib_per_latent,+ decode_platform_cap,+ )+ try:+ cap = decode_platform_cap(vae, z)+ if cap is not None and cap < int(z.shape[-2]) * int(z.shape[-1]):+ return None+ free = _free_mib(vae, z)+ if free is None:+ return None+ mib, scale = free+ frames = int(z.shape[2]) if z.dim() == 5 else 1+ batch = 1 if bool(getattr(vae, "use_slicing", False)) else int(z.shape[0])+ per_latent = decode_mib_per_latent(_geometry(vae)[0], _fused(vae), type(vae).__name__)+ need = batch * frames * int(z.shape[-2]) * int(z.shape[-1]) * per_latent * scale+ return int(-(-(need / FREE_FRACTION - mib) // 1))+ except Exception: # noqa: BLE001 - unknown figures: decode as planned (tiled)+ return None+++def _untiled_encode_shortfall_mib(vae: Any, x: Any) -> Optional[int]:+ """MiB short for encoding ``x`` untiled, the decode's rule; None off CUDA or on unknown figures."""+ from .diffusion_vae_tiling import FREE_FRACTION, _fused, _geometry, encode_mib_per_latent+ try:+ import torch++ if getattr(x, "device", None) is None or x.device.type != "cuda":+ return None+ ratio = int(_geometry(vae)[0])+ per_latent = encode_mib_per_latent(ratio, _fused(vae), type(vae).__name__)+ if per_latent is None or ratio <= 0:+ return None+ free, _ = torch.cuda.mem_get_info(x.device)+ free += torch.cuda.memory_reserved(x.device) - torch.cuda.memory_allocated(x.device)+ elem = next(vae.encoder.parameters()).element_size()+ # enable_fp16_vae_decode's fp32_encode upcasts the fp16 encoder and input for the encode itself+ if getattr(vae, "_unsloth_fp16_decode", False):+ elem = max(elem, 4)+ scale = elem / 2+ frames = int(x.shape[2]) if x.dim() == 5 else 1+ batch = 1 if bool(getattr(vae, "use_slicing", False)) else int(x.shape[0])+ latents = -(-int(x.shape[-2]) // ratio) * -(-int(x.shape[-1]) // ratio)+ need = batch * frames * latents * per_latent * scale+ return int(-(-(need / FREE_FRACTION - free / 2**20) // 1))+ except Exception: # noqa: BLE001 - unknown figures: encode tiled+ return None+++def _release_for_untiled(+ state_pipe: Any, shortfall: Callable[[], Optional[int]], device: Any, logger: Any, reason: str+) -> tuple[bool, Optional[Callable[[], None]]]:+ """(untiled?, restore). Streams the shortfall plus margin, then the rest once; nothing when even every resident+ group could not cover it (a freed MiB frees at most a MiB). Still short: pin back and tile."""+ short = shortfall()+ if short is None:+ return False, None+ if short <= 0:+ return True, None+ resident = resident_group_mib(state_pipe)+ if short > resident:+ return False, None+ restore: Optional[Callable[[], None]] = None+ try:+ for need in (min(resident, short + short // 4 + 256), None):+ if need is None:+ need = resident_group_mib(state_pipe)+ if need <= 0:+ break+ step = release_resident_groups(state_pipe, need, logger, reason = reason, device = device)+ if step is None:+ break+ restore = _chain_restores(step, restore)+ again = shortfall()+ if again is not None and again <= 0:+ return True, restore+ if need >= resident:+ break+ except Exception as exc: # noqa: BLE001 - a broken release keeps the tiled pass+ if logger is not None:+ logger.warning("diffusion.memory: streaming before %s failed (%s)", reason, exc)+ except BaseException:+ # a cancel between a release and its handover: pin back before propagating+ if restore is not None:+ restore()+ raise+ if restore is not None:+ restore()+ return False, None+++def _release_for_untiled_decode(+ state_pipe: Any, vae: Any, z: Any, device: Any, logger: Any+) -> tuple[bool, Optional[Callable[[], None]]]:+ return _release_for_untiled(+ state_pipe,+ lambda: _untiled_decode_shortfall_mib(vae, z),+ device,+ logger,+ "an untiled VAE decode",+ )+++def _release_for_untiled_encode(+ state_pipe: Any, vae: Any, x: Any, device: Any, logger: Any+) -> tuple[bool, Optional[Callable[[], None]]]:+ """Untiled only on a positive fit check: on WDDM an overrun spills to host memory instead of raising."""+ return _release_for_untiled(+ state_pipe,+ lambda: _untiled_encode_shortfall_mib(vae, x),+ device,+ logger,+ "an untiled VAE encode",+ )+++def _defer_tiling_to_decode(+ pipe: Any,+ state_pipe: Any,+ verdict: Any,+ guard_target: Any,+ guard_kwargs: dict,+ restore_vae: Any,+ logger: Any,+) -> Any:+ """Tile verdict with resident offload groups: keep them through the denoise (as ComfyUI does) and decide at each+ VAE decode / encode, on its own input, whether streaming lets it run untiled (tiling changes pixels, streaming+ only costs time); groups pin back right after. An untiled encode OOM retries tiled. Wrappers sit on the public+ ``decode`` / ``encode`` slots, outside compiled regions. Returns the VAE restore extended to drop them."""+ vae = getattr(pipe, "vae", None)+ if (+ vae is None+ or restore_vae is None+ or verdict is None+ or verdict.action != ACTIVATION_TILE+ or verdict.overridden+ or _release_before_tiling_off()+ or not bool(getattr(vae, "use_tiling", False))+ or not callable(getattr(vae, "decode", None))+ ):+ return restore_vae+ resident = resident_group_mib(state_pipe)+ if resident <= 0:+ return restore_vae+ # No request-wide "hopeless" gate: the verdict prices the denoise, gone by decode time.+ try:+ if _vae_on_other_card(pipe, guard_target) is not None:+ return restore_vae+ except Exception: # noqa: BLE001 - unreadable split: the primary's card decodes+ pass+ from . import decode_layers+ from .diffusion_vae_oom import _compiling, clear_partial_decode, drop_tracebacks, is_oom++ device = getattr(guard_target, "torch_device", None)+ stats = {"decodes": 0, "untiled": 0, "tiled": 0, "release_s": 0.0, "restore_s": 0.0,+ "released_mib": 0, "encodes": 0, "encode_untiled": 0, "encode_tiled": 0,+ "encode_released_mib": 0, "encode_oom": 0} # fmt: skip+ vae.__dict__[RELEASE_BEFORE_TILING_STATS] = stats++ def make_decode(inner: Callable, layer: Any) -> Callable:+ depth = [0]++ def decode(*args: Any, **kwargs: Any) -> Any:+ if depth[0] or _compiling():+ return inner(*args, **kwargs)+ depth[0] += 1+ restore = None+ untiled = False+ try:+ z = args[0] if args else kwargs.get("z")+ t0 = time.perf_counter()+ before = resident_group_mib(state_pipe)+ untiled, restore = _release_for_untiled_decode(state_pipe, vae, z, device, logger)+ stats["decodes"] += 1+ stats["release_s"] += time.perf_counter() - t0+ stats["untiled" if untiled else "tiled"] += 1+ if restore is not None:+ stats["released_mib"] += max(0, before - resident_group_mib(state_pipe))+ vae.use_tiling = not untiled+ return inner(*args, **kwargs)+ finally:+ vae.use_tiling = False+ if restore is not None:+ t1 = time.perf_counter()+ try:+ restore()+ except Exception as exc: # noqa: BLE001 - the groups keep streaming+ if logger is not None:+ logger.warning(+ "diffusion.memory: re-residency after the decode failed: %s", exc+ )+ stats["restore_s"] += time.perf_counter() - t1+ depth[0] -= 1++ return decode++ def make_encode(inner: Callable, layer: Any) -> Callable:+ depth = [0]++ def encode(*args: Any, **kwargs: Any) -> Any:+ if depth[0] or _compiling() or bool(getattr(vae, "use_tiling", False)):+ return inner(*args, **kwargs)+ depth[0] += 1+ restore = None+ try:+ x = args[0] if args else kwargs.get("x")+ before = resident_group_mib(state_pipe)+ untiled, restore = _release_for_untiled_encode(state_pipe, vae, x, device, logger)+ stats["encodes"] += 1+ stats["encode_untiled" if untiled else "encode_tiled"] += 1+ if restore is not None:+ stats["encode_released_mib"] += max(0, before - resident_group_mib(state_pipe))+ vae.use_tiling = not untiled+ if not untiled:+ return inner(*args, **kwargs) # oversized or unknown: tiled, as main+ try:+ return inner(*args, **kwargs)+ except Exception as exc: # noqa: BLE001 - the backstop behind a fit check that was wrong+ if not is_oom(exc):+ raise+ drop_tracebacks(exc)+ stats["encode_oom"] += 1+ clear_partial_decode(vae, x)+ vae.use_tiling = True+ return inner(*args, **kwargs)+ finally:+ vae.use_tiling = False+ if restore is not None:+ try:+ restore()+ except Exception as exc: # noqa: BLE001 - the groups keep streaming+ if logger is not None:+ logger.warning(+ "diffusion.memory: re-residency after the encode failed: %s", exc+ )+ depth[0] -= 1++ return encode++ layers = [decode_layers.wrap(vae, "decode", RELEASE_BEFORE_TILING_LAYER, make_decode)]+ if callable(getattr(vae, "encode", None)):+ layers.append(decode_layers.wrap(vae, "encode", RELEASE_BEFORE_TILING_LAYER, make_encode))+ vae.use_tiling = False+ if logger is not None:+ logger.info(+ "diffusion.memory: VAE tiling waits for the decode; resident groups stay resident for the denoise"+ )++ def restore_all() -> None:+ for layer in reversed(layers):+ decode_layers.unwrap(layer)+ vae.use_tiling = True # what the request's tiling left, for its own undo+ restore_vae()++ return restore_all+++def _chain_restores(+ first: Optional[Callable[[], None]], then: Optional[Callable[[], None]]+) -> Optional[Callable[[], None]]:+ """One restore running ``first`` (the later release) and then ``then``; None when both are None."""+ if first is None:+ return then+ if then is None:+ return first++ def restore() -> None:+ try:+ first()+ finally:+ then()++ return restore++def _tile_for_verdict(pipe: Any,verdict: Any,@@ -13307,9 +13603,21 @@ class DiffusionBackend:if _hires_pass is not None and _hires_pass.preflight is not None:_hires_pass.preflight(guard_kwargs)_guard_layer_cards(state.pipe, guard_target, guard_kwargs)+ tiled_before = bool(getattr(getattr(pipe, "vae", None), "use_tiling", False))restore_vae = _tile_for_verdict(pipe, verdict, guard_target, guard_kwargs, guard_batch, restore_vae, logger)+ # tiling changes pixels, streaming only costs time: defer the choice to the decode+ if not tiled_before:+ restore_vae = _defer_tiling_to_decode(+ pipe,+ state.pipe,+ verdict,+ guard_target,+ guard_kwargs,+ restore_vae,+ logger,+ )except ValueError:raise # the refusal itself: the route turns this into a 400 with the reasonexcept Exception as exc: # noqa: BLE001 - fail OPEN on a broken probediff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.pyindex ff34e53564a..3c49e7be04a 100644--- a/studio/backend/core/inference/diffusion_vae_tiling.py+++ b/studio/backend/core/inference/diffusion_vae_tiling.py@@ -132,6 +132,30 @@ def decode_mib_per_latent(return DECODE_MIB_PER_LATENT * (int(ratio) / DECODE_MIB_RATIO) ** 2+# bf16 untiled encode peak MiB per latent of the encoded frame, (unfused, fused): B200, real weights, 512 to 1536 px+# (linear in area: Qwen-Image-2.1 0.565 / 0.283, Qwen-Image 0.165 / 0.121), rounded up.+ENCODE_MIB_PER_LATENT_BY_CLASS = {+ "AutoencoderKLQwenImage": (0.17, 0.13),+ "AutoencoderKLQwenImage21": (0.6, 0.3),+}+++def encode_mib_per_latent(+ ratio: int,+ fused: bool = False,+ cls: Optional[str] = None,+) -> Optional[float]:+ """bf16 untiled encode peak MiB per latent: measured, else the decode figure (an upper bound), else None."""+ known = (+ ENCODE_MIB_PER_LATENT_BY_CLASS.get(cls or "")+ or DECODE_MIB_PER_LATENT_BY_CLASS.get(cls or "")+ or DECODE_MIB_PER_LATENT_BY_RATIO.get(int(ratio))+ )+ if known is None:+ return None+ return known[1] if fused else known[0]++def _geometry(vae: Any) -> tuple[int, int, int]:"""(ratio, floor tile, floor overlap), cached on the VAE at install."""cached = getattr(vae, "__dict__", {}).get("_unsloth_wide_geometry")@@ -277,21 +301,29 @@ def _fused(vae: Any) -> bool:)-def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:- """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now; None off CUDA."""- raw = (os.environ.get(MAX_TILE_ENV) or "").strip()- if raw.isdigit() and int(raw) > 0:- return int(raw) ** 2+def decode_platform_cap(vae: Any, z: Any) -> Optional[int]:+ """The decode tile area (latents) a platform pins regardless of free memory, else None."""# CUDA-calibrated coefficients under-predict Qwen-Image-2.1 decode on ROCm (40x40, 64x32 and 96x56 OOM at# 17/19 GiB limits): start at the tested 32x32 floor rather than retrying an oversized tile on every image.import torchif (- z.device.type == "cuda"+ getattr(getattr(z, "device", None), "type", None) == "cuda"and is_rocm_torch(torch)and type(vae).__name__ == "AutoencoderKLQwenImage21"):return TILE_LATENTS**2+ return None+++def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:+ """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now; None off CUDA."""+ raw = (os.environ.get(MAX_TILE_ENV) or "").strip()+ if raw.isdigit() and int(raw) > 0:+ return int(raw) ** 2+ cap = decode_platform_cap(vae, z)+ if cap is not None:+ return captry:_, tile, _ = _geometry(vae)# Stock tile == floor: grow only into device-free memory; decoding out of the allocator cache stalled ondiff --git a/studio/backend/tests/test_diffusion_release_before_tiling.py b/studio/backend/tests/test_diffusion_release_before_tiling.pynew file mode 100644index 00000000000..b1d2042aae2--- /dev/null+++ b/studio/backend/tests/test_diffusion_release_before_tiling.py@@ -0,0 +1,504 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A tile verdict with offload groups still resident streams them at the decode, not before the denoise.++Seen on a 15 GB T4 (Qwen-Image-2.1 fp16 edit, 1024x1024): streaming before the denoise left the decode tiled+(PSNR 52.6 dB vs untiled). CPU-only: decode figures, snapshot and group release are stubbed.+"""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion as diffusion+from core.inference.diffusion_memory import (+ ACTIVATION_RUN,+ ACTIVATION_TILE,+ ImageActivationVerdict,+)++UNTILED_NEED = 9000 # MiB the whole-frame decode needs, past FREE_FRACTION+ENCODE_NEED = (+ 3000 # MiB the untiled source encode needs, past FREE_FRACTION (fits the 6000 MiB free)+)+++class _Card:+ """Free MiB the decode-time check sees, raised by a release."""++ def __init__(self, free_mib: int, resident_mib: int) -> None:+ self.free = free_mib+ self.resident = resident_mib+ self.releases: list = []+ self.restored = 0+++def card(monkeypatch):+ c = _Card(free_mib = 6000, resident_mib = 4283)++ def release(+ pipe,+ need,+ logger = None,+ reason = "",+ device = None,+ **kw,+ ):+ if c.resident <= 0:+ return None+ c.releases.append((need, reason))+ freed = min(need, c.resident)+ c.resident -= freed+ c.free += freed++ def restore():+ c.restored += 1+ c.resident += freed+ c.free -= freed++ return restore++ monkeypatch.setattr(diffusion, "resident_group_mib", lambda pipe: c.resident)+ monkeypatch.setattr(diffusion, "release_resident_groups", release)+ monkeypatch.setattr(+ diffusion, "_untiled_decode_shortfall_mib", lambda vae, z: UNTILED_NEED - c.free+ )+ c.encode_need = ENCODE_NEED+ monkeypatch.setattr(+ diffusion, "_untiled_encode_shortfall_mib", lambda vae, x: c.encode_need - c.free+ )+ monkeypatch.delenv(diffusion.RELEASE_BEFORE_TILING_ENV, raising = False)+ return c+++def _decide(card):+ return diffusion._release_for_untiled_decode(object(), None, "z", "cuda:0", None)+++def test_the_shortfall_streams_and_the_decode_runs_untiled(card):+ untiled, restore = _decide(card)+ # short 3000 MiB, plus a quarter for the budget's reserve and 256 MiB: 4006 of the 4283 resident+ assert untiled and card.releases == [(4006, "an untiled VAE decode")]+ assert card.resident == 277+ restore()+ assert card.resident == 4283 and card.restored == 1+++def test_a_frame_that_already_fits_decodes_untiled_without_streaming(card):+ card.free = 9500+ assert _decide(card) == (True, None)+ assert card.releases == []+++def test_a_short_first_release_streams_the_rest_once(card, monkeypatch):+ real = diffusion.release_resident_groups+ monkeypatch.setattr(+ diffusion,+ "release_resident_groups",+ lambda pipe, need, *a, **k: real(pipe, need if card.releases else need // 4, *a, **k),+ )+ untiled, restore = _decide(card)+ assert untiled and len(card.releases) == 2 and card.resident == 0+ restore()+ assert card.resident == 4283 and card.restored == 2+++def test_still_short_after_every_group_pins_back_and_tiles(card, monkeypatch):+ monkeypatch.setattr(diffusion, "_untiled_decode_shortfall_mib", lambda vae, z: 1)+ assert _decide(card) == (False, None)+ assert card.resident == 4283 and card.restored >= 1+++def test_a_shortfall_past_every_resident_group_streams_nothing(card):+ card.resident = 2000 # short 3000 MiB: even all of it leaves the decode tiled+ assert _decide(card) == (False, None)+ assert card.releases == []+++def test_unknown_decode_figures_tile_without_streaming(card, monkeypatch):+ monkeypatch.setattr(diffusion, "_untiled_decode_shortfall_mib", lambda vae, z: None)+ assert _decide(card) == (False, None)+ assert card.releases == []+++def test_a_broken_release_tiles_and_pins_back(card, monkeypatch):+ calls = {"n": 0}+ real = diffusion.release_resident_groups++ def flaky(pipe, need, *a, **k):+ calls["n"] += 1+ if calls["n"] == 2:+ raise RuntimeError("copy stream broke")+ return real(pipe, need // 4, *a, **k)++ monkeypatch.setattr(diffusion, "release_resident_groups", flaky)+ assert _decide(card) == (False, None)+ assert card.resident == 4283 and card.restored == 1+++def test_a_cancel_mid_release_pins_back_and_propagates(card, monkeypatch):+ real = diffusion._untiled_decode_shortfall_mib+ calls = {"n": 0}++ def cancel(vae, z):+ calls["n"] += 1+ if calls["n"] == 2:+ raise KeyboardInterrupt+ return real(vae, z)++ monkeypatch.setattr(diffusion, "_untiled_decode_shortfall_mib", cancel)+ with pytest.raises(KeyboardInterrupt):+ _decide(card)+ assert card.resident == 4283 and card.restored == 1+++def test_chained_restore_runs_the_later_release_first():+ order: list = []+ chained = diffusion._chain_restores(+ lambda: order.append("later"), lambda: order.append("first")+ )+ chained()+ assert order == ["later", "first"]+ only = lambda: None # noqa: E731+ assert diffusion._chain_restores(None, only) is only+ assert diffusion._chain_restores(only, None) is only+ assert diffusion._chain_restores(None, None) is None+++def test_generate_defers_the_tiling_to_the_decode():+ import inspect++ src = inspect.getsource(diffusion.DiffusionBackend.generate)+ # the request keeps its tiling engagement (and its refusals) and hands the tiling to the decode+ assert src.index("restore_vae = _tile_for_verdict(") < src.index("_defer_tiling_to_decode(")+ assert "release_for_untiled_decode" not in src # nothing streams before the denoise+++def test_the_untiled_decode_shortfall_uses_the_wide_tile_rule(monkeypatch):+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.setattr(tiling, "_free_mib", lambda vae, z: (1000.0, 1.0))+ monkeypatch.setattr(tiling, "_geometry", lambda vae: (16, 32, 4))+ monkeypatch.setattr(tiling, "_fused", lambda vae: True)+ z = types.SimpleNamespace(shape = (1, 16, 1, 64, 64), dim = lambda: 5)+ vae = types.SimpleNamespace(use_slicing = True)+ per_latent = tiling.decode_mib_per_latent(16, True, "SimpleNamespace")+ want = 64 * 64 * per_latent / tiling.FREE_FRACTION - 1000+ assert abs(diffusion._untiled_decode_shortfall_mib(vae, z) - want) <= 1+ monkeypatch.setattr(tiling, "_free_mib", lambda vae, z: None)+ assert diffusion._untiled_decode_shortfall_mib(vae, z) is None+++class _Vae:+ """A VAE whose decode and encode record the tiling and the resident MiB they ran with."""++ def __init__(+ self,+ card: _Card,+ fail: BaseException | None = None,+ encode_oom: int = 0,+ ) -> None:+ self.card = card+ self.use_tiling = True # what engage_vae_tiling left+ self.fail = fail+ self.encode_oom = encode_oom+ self.decodes: list = []+ self.encodes: list = []+ self.encode_resident: list = []++ def decode(self, z):+ self.decodes.append((self.use_tiling, self.card.resident))+ if self.fail is not None:+ raise self.fail+ return z++ def encode(self, x):+ self.encodes.append(self.use_tiling)+ self.encode_resident.append(self.card.resident)+ if self.encode_oom:+ self.encode_oom -= 1+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")+ return x++ def disable_tiling(self):+ self.use_tiling = False+++def _priced(free: int) -> ImageActivationVerdict:+ # untiled: 6952 MiB of activations + the 2048 MiB fixed overhead = 9000 MiB+ return ImageActivationVerdict(+ ACTIVATION_TILE, needed_mib = 6952, tiled_needed_mib = 2952, budget_mib = free+ )+++TARGET = types.SimpleNamespace(torch_device = "cuda:0")+++def _armed(+ card,+ vae,+ verdict = None,+):+ pipe = types.SimpleNamespace(vae = vae)+ undone: list = []++ def restore_vae():+ undone.append(True)+ vae.disable_tiling()++ restore = diffusion._defer_tiling_to_decode(+ pipe, pipe, verdict or _priced(6000), TARGET, {}, restore_vae, None+ )+ return restore, undone+++def test_the_denoise_keeps_the_groups_and_the_decode_streams_them_untiled(card):+ vae = _Vae(card)+ restore, undone = _armed(card, vae)+ assert vae.use_tiling is False and card.releases == [] # nothing streamed before the denoise+ assert vae.encode("x") == "x" and vae.encodes == [False] # the edit's encode runs untiled too+ assert vae.decode("z") == "z"+ assert vae.decodes == [(False, 277)] # the shortfall streamed, the decode untiled+ assert (+ card.resident == 4283 and card.restored == 1+ ) # pinned back as soon as the decode returned+ stats = vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]+ assert stats["untiled"] == 1 and stats["tiled"] == 0 and stats["released_mib"] == 4006+ restore()+ assert "decode" not in vae.__dict__ and "encode" not in vae.__dict__ # the wrappers are gone+ assert undone == [True] and vae.use_tiling is False+++def test_every_decode_of_a_request_streams_and_pins_back_on_its_own(card):+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ vae.decode("a")+ vae.decode("b")+ assert vae.decodes == [(False, 277), (False, 277)] and card.restored == 2+ restore()+++def test_a_decode_that_still_does_not_fit_tiles_as_before(card, monkeypatch):+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ monkeypatch.setattr(diffusion, "_untiled_decode_shortfall_mib", lambda vae, z: 1)+ vae.decode("z")+ assert vae.decodes == [(True, 4283)] and vae.use_tiling is False+ assert vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]["tiled"] == 1+ restore()+++def test_a_failing_or_cancelled_decode_pins_the_groups_back(card):+ for exc in (RuntimeError("decode broke"), KeyboardInterrupt()):+ vae = _Vae(card, fail = exc)+ restore, _ = _armed(card, vae)+ with pytest.raises(type(exc)):+ vae.decode("z")+ assert card.resident == 4283 and vae.use_tiling is False+ restore()+++def test_an_untiled_encode_out_of_memory_retries_tiled(card):+ vae = _Vae(card, encode_oom = 1)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [False, True] and vae.use_tiling is False+ assert vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]["encode_oom"] == 1+ restore()+++ "case",+ ["kill", "run", "overridden", "nothing_resident", "split", "not_engaged"],+)+def test_cases_that_do_not_stream_keep_the_request_tiling(card, monkeypatch, case):+ verdict = _priced(6000)+ vae = _Vae(card)+ if case == "kill":+ monkeypatch.setenv(diffusion.RELEASE_BEFORE_TILING_ENV, "0")+ elif case == "run":+ verdict = ImageActivationVerdict(ACTIVATION_RUN)+ elif case == "overridden":+ verdict = ImageActivationVerdict(ACTIVATION_TILE, overridden = True)+ elif case == "nothing_resident":+ card.resident = 0+ elif case == "split":+ monkeypatch.setattr(diffusion, "_vae_on_other_card", lambda pipe, target: "cuda:1")+ elif case == "not_engaged":+ vae.use_tiling = False+ pipe = types.SimpleNamespace(vae = vae)+ restore_vae = vae.disable_tiling+ out = diffusion._defer_tiling_to_decode(pipe, pipe, verdict, TARGET, {}, restore_vae, None)+ assert out is restore_vae and "decode" not in vae.__dict__ and "encode" not in vae.__dict__+ assert card.releases == []++[email protected]("decode_free", [9500, 6000])+def test_a_conditioned_request_short_past_the_groups_still_decodes_untiled(card, decode_free):+ # Verdict short by more than all resident groups, but the frame alone fits at the decode: must decode untiled.+ verdict = ImageActivationVerdict(+ ACTIVATION_TILE, needed_mib = 23952, tiled_needed_mib = 2952, budget_mib = 6000+ )+ vae = _Vae(card)+ restore, _ = _armed(card, vae, verdict)+ card.free = decode_free+ assert vae.decode("z") == "z"+ assert vae.decodes == [(False, 4283 if decode_free >= UNTILED_NEED else 277)]+ assert card.releases == (+ [] if decode_free >= UNTILED_NEED else [(4006, "an untiled VAE decode")]+ )+ assert card.resident == 4283 and vae.use_tiling is False+ restore()+++def test_a_frame_short_past_every_group_tiles_at_the_decode_with_nothing_streamed(card):+ card.resident = 2000 # the decode is short 3000 MiB: even every resident group leaves it tiled+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.decode("z") == "z"+ assert vae.decodes == [(True, 2000)] and card.releases == [] and vae.use_tiling is False+ assert vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]["tiled"] == 1+ restore()+++def test_an_oversized_encode_keeps_tiling_with_no_untiled_attempt(card):+ card.encode_need = 20000 # short 14000 MiB, past every resident group+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [True] and vae.use_tiling is False # tiled straight away, as main+ assert card.releases == []+ stats = vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]+ assert stats["encode_tiled"] == 1 and stats["encode_untiled"] == 0 and stats["encode_oom"] == 0+ restore()+++def test_a_fitting_encode_runs_untiled_without_streaming(card):+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [False] and vae.encode_resident == [4283] and card.releases == []+ stats = vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]+ assert stats["encode_untiled"] == 1 and stats["encode_tiled"] == 0+ restore()+++def test_a_short_encode_streams_runs_untiled_and_pins_back(card):+ card.encode_need = (+ 8000 # short 2000 MiB: plus a quarter and 256 MiB, 2756 of the 4283 resident stream+ )+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [False] and vae.encode_resident == [4283 - 2756]+ assert card.releases == [(2756, "an untiled VAE encode")]+ assert (+ card.resident == 4283 and card.restored == 1+ ) # pinned back as soon as the encode returned+ stats = vae.__dict__[diffusion.RELEASE_BEFORE_TILING_STATS]+ assert stats["encode_untiled"] == 1 and stats["encode_released_mib"] == 2756+ assert stats["released_mib"] == 0 # the decode's own counter is untouched+ restore()+++def test_unknown_encode_figures_tile(card, monkeypatch):+ monkeypatch.setattr(diffusion, "_untiled_encode_shortfall_mib", lambda vae, x: None)+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [True] and card.releases == []+ restore()++[email protected]("need", [None, 20000])+def test_on_wddm_no_untiled_encode_without_a_fit(card, monkeypatch, need):+ # WDDM spills an overrun to host RAM instead of raising: only a positive fit check may encode untiled.+ monkeypatch.setattr(diffusion.sys, "platform", "win32")+ monkeypatch.setattr(+ diffusion,+ "_untiled_encode_shortfall_mib",+ lambda vae, x: None if need is None else need - card.free,+ )+ vae = _Vae(card)+ restore, _ = _armed(card, vae)+ assert vae.encode("x") == "x"+ assert vae.encodes == [True] and card.releases == []+ restore()+++def test_the_untiled_encode_shortfall_uses_the_measured_encode_figures(monkeypatch):+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.setattr(tiling, "_geometry", lambda vae: (16, 32, 4))+ monkeypatch.setattr(tiling, "_fused", lambda vae: False)+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ param = types.SimpleNamespace(element_size = lambda: 2)+ vae = type("AutoencoderKLQwenImage21", (), {})()+ vae.encoder = types.SimpleNamespace(parameters = lambda: iter([param]))+ x = types.SimpleNamespace(+ shape = (1, 4, 1, 1024, 1024), dim = lambda: 5, device = types.SimpleNamespace(type = "cuda")+ )+ want = 64 * 64 * 0.6 / tiling.FREE_FRACTION - 1000+ assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - want) <= 1+ assert tiling.encode_mib_per_latent(16, True, "AutoencoderKLQwenImage21") == 0.3+ assert tiling.encode_mib_per_latent(8, False, "AutoencoderKL") == tiling.decode_mib_per_latent(+ 8, False, "AutoencoderKL"+ )+ assert tiling.encode_mib_per_latent(12, False, "SomeVae") is None # unknown class and ratio+ x.device = types.SimpleNamespace(type = "cpu")+ assert diffusion._untiled_encode_shortfall_mib(vae, x) is None+++def test_the_untiled_decode_check_keeps_the_budget_caps(monkeypatch):+ # The whole-frame check must respect the ROCm Qwen-Image-2.1 floor cap.+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.delenv(tiling.MAX_TILE_ENV, raising = False)+ monkeypatch.setattr(tiling, "_free_mib", lambda vae, z: (10**7, 1.0))+ monkeypatch.setattr(tiling, "_geometry", lambda vae: (16, 32, 4))+ monkeypatch.setattr(tiling, "_fused", lambda vae: False)+ vae = type("AutoencoderKLQwenImage21", (), {"use_slicing": True})()+ z = types.SimpleNamespace(+ shape = (1, 16, 1, 64, 64), dim = lambda: 5, device = types.SimpleNamespace(type = "cuda")+ )+ monkeypatch.setattr(tiling, "is_rocm_torch", lambda torch: False)+ assert diffusion._untiled_decode_shortfall_mib(vae, z) <= 0 # CUDA: fits, untiled+ monkeypatch.setattr(tiling, "is_rocm_torch", lambda torch: True)+ assert tiling.decode_tile_budget(vae, z) == tiling.TILE_LATENTS**2+ assert diffusion._untiled_decode_shortfall_mib(vae, z) is None # ROCm: keep the floor tiles+ small = types.SimpleNamespace(shape = (1, 16, 1, 32, 32), dim = lambda: 5, device = z.device)+ assert diffusion._untiled_decode_shortfall_mib(vae, small) <= 0 # the frame is the floor tile+ monkeypatch.setattr(tiling, "is_rocm_torch", lambda torch: False)+ monkeypatch.setenv(tiling.MAX_TILE_ENV, "40") # an explicit max tile sizes tiled decodes only+ assert diffusion._untiled_decode_shortfall_mib(vae, z) <= 0+++def test_the_untiled_encode_check_prices_the_fp32_encode_of_an_fp16_decode_vae(monkeypatch):+ # fp32_encode upcasts the fp16 SDXL encoder: price the encode at fp32.+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.setattr(tiling, "_geometry", lambda vae: (8, 32, 4))+ monkeypatch.setattr(tiling, "_fused", lambda vae: False)+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ param = types.SimpleNamespace(element_size = lambda: 2)+ vae = type("AutoencoderKL", (), {})()+ vae.encoder = types.SimpleNamespace(parameters = lambda: iter([param]))+ x = types.SimpleNamespace(+ shape = (1, 3, 1024, 1024), dim = lambda: 4, device = types.SimpleNamespace(type = "cuda")+ )+ per_latent = tiling.encode_mib_per_latent(8, False, "AutoencoderKL")+ fp16 = 128 * 128 * per_latent / tiling.FREE_FRACTION - 1000+ fp32 = 128 * 128 * per_latent * 2 / tiling.FREE_FRACTION - 1000+ assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp16) <= 1+ vae._unsloth_fp16_decode = True+ assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp32) <= 1