Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: decode image VAEs into one expandable allocator segment (SDXL holds 1.15 GiB less VRAM) (#17045)
Daniel Han570619d455a6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 3a1ea937789..d762f3cba5a 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -10880,7 +10880,11 @@ class DiffusionBackend:# After the speed layer, so it wraps the fused / single-frame _decode the layer installed.try:from .diffusion_vae_oom import install as install_vae_oom_retry+install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)+ from .diffusion_decode_segments import enable as enable_decode_expandable++ enable_decode_expandable(getattr(pipe, "vae", None))except Exception as exc: # noqa: BLE001 - a decode OOM then fails as beforelogger.warning("diffusion.vae_oom: not installed: %s", exc)self._raise_if_load_cancelled(_load_token)diff --git a/studio/backend/core/inference/diffusion_decode_segments.py b/studio/backend/core/inference/diffusion_decode_segments.pynew file mode 100644index 00000000000..9957f46eda5--- /dev/null+++ b/studio/backend/core/inference/diffusion_decode_segments.py@@ -0,0 +1,167 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Run each image VAE decode in one expandable allocator segment (Linux CUDA, native allocator): its growing+activations and cuDNN workspaces otherwise each map a new fixed segment (SDXL 1024^2: 11.3 -> 10.1 GiB reserved).+UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off."""++from __future__ import annotations++import contextlib+import os+import sys+import threading+from typing import Any, Iterator, Optional++from .rocm_bf16 import is_rocm_torch++DECODE_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_DECODE_EXPANDABLE"+ENABLED_ATTR = "_unsloth_decode_expandable"+_ALLOC_CONF_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")++_lock = threading.Lock()+_depth = 0+_usable: Optional[bool] = None+STATS = {"engaged": 0, "skipped": 0}+++def _env_off() -> bool:+ return os.environ.get(DECODE_EXPANDABLE_ENV, "").strip().lower() in ("0", "false", "no", "off")+++def _conf_keys(value: str) -> dict:+ out, part, depth = {}, "", 0+ for ch in value + ",":+ if ch == "[":+ depth += 1+ elif ch == "]":+ depth = max(0, depth - 1)+ if ch == "," and depth == 0:+ key, _, val = part.partition(":")+ if key.strip():+ out[key.strip().lower()] = val.strip().lower()+ part = ""+ else:+ part += ch+ return out+++def _allocator_configured() -> bool:+ """Options a runtime setting would reset; a bare ``expandable_segments:True`` (unsloth_zoo, after CUDA init) is not."""+ for name in _ALLOC_CONF_VARS:+ keys = _conf_keys(os.environ.get(name) or "")+ if any(k != "expandable_segments" for k in keys):+ return True+ if keys.get("expandable_segments", "true") != "true":+ return True+ return False+++def _platform_ok() -> bool:+ if not sys.platform.startswith("linux"):+ return False+ if os.environ.get("WSL_DISTRO_NAME") or os.environ.get("WSL_INTEROP"):+ return False+ try:+ release = os.uname().release.lower()+ except Exception: # noqa: BLE001+ return False+ if "microsoft" in release or "tegra" in release or os.path.exists("/etc/nv_tegra_release"):+ return False+ return True+++def _setter(torch: Any) -> Any:+ c = getattr(torch, "_C", None)+ return getattr(c, "_accelerator_setAllocatorSettings", None) or getattr(+ c, "_cuda_cudaCachingAllocator_set_allocator_settings", None+ )+++def _decide(torch: Any) -> bool:+ if not _platform_ok() or _allocator_configured():+ return False+ if is_rocm_torch(torch):+ return False+ if _setter(torch) is None:+ return False+ try:+ if torch.cuda.memory.get_allocator_backend() != "native":+ return False+ # Already on process-wide: switching it off after the decode would be wrong.+ if any(seg.get("is_expandable") for seg in torch.cuda.memory_snapshot()):+ return False+ except Exception: # noqa: BLE001+ return False+ return True+++def _torch() -> Any:+ return sys.modules.get("torch")+++def _cuda_tensor(z: Any) -> bool:+ device = getattr(z, "device", None)+ return getattr(device, "type", None) == "cuda"+++def _busy(torch: Any) -> bool:+ try:+ compiler = getattr(torch, "compiler", None)+ if compiler is not None and compiler.is_compiling():+ return True+ return bool(torch.cuda.is_current_stream_capturing())+ except Exception: # noqa: BLE001+ return True+++def expandable_decode(vae: Any, z: Any) -> Iterator[None]:+ global _depth, _usable+ if not vae.__dict__.get(ENABLED_ATTR, False) or _env_off() or not _cuda_tensor(z):+ yield+ return+ torch = _torch()+ if torch is None or _busy(torch):+ yield+ return+ engaged = False+ with _lock:+ if _usable is None:+ _usable = _decide(torch)+ if _usable:+ try:+ if _depth == 0:+ _setter(torch)("expandable_segments:True")+ _depth += 1+ engaged = True+ except Exception: # noqa: BLE001 - refused: never retry+ _usable = False+ if engaged:+ STATS["engaged"] += 1+ else:+ STATS["skipped"] += 1+ try:+ yield+ finally:+ if engaged:+ with _lock:+ _depth -= 1+ if _depth == 0:+ try:+ _setter(torch)("expandable_segments:False")+ except Exception: # noqa: BLE001+ _usable = False+++def enable(vae: Any) -> bool:+ if vae is None or _env_off():+ return False+ try:+ vae.__dict__[ENABLED_ATTR] = True+ except Exception: # noqa: BLE001+ return False+ return True+++__all__ = ["DECODE_EXPANDABLE_ENV", "ENABLED_ATTR", "STATS", "enable", "expandable_decode"]diff --git a/studio/backend/core/inference/diffusion_vae_oom.py b/studio/backend/core/inference/diffusion_vae_oom.pyindex 344db8b5284..e6a028fd8a9 100644--- a/studio/backend/core/inference/diffusion_vae_oom.py+++ b/studio/backend/core/inference/diffusion_vae_oom.py@@ -14,6 +14,7 @@ import weakreffrom typing import Any, Callable, Optionalfrom . import decode_layers+from .diffusion_decode_segments import expandable_decodeOOM_RETRY_ENV = "UNSLOTH_VAE_OOM_RETRY"HOST_MARGIN_MIB = 2048@@ -1115,6 +1116,10 @@ def _install_scope(vae: Any, logger: Any = None) -> bool:def decode(*args: Any, **kwargs: Any) -> Any:if _compiling() or oom_retry_disabled() or _scope_session(vae) is not None:return inner(*args, **kwargs)+ with expandable_decode(vae, args[0] if args else kwargs.get("z")):+ return _scoped(args, kwargs)++ def _scoped(args: tuple, kwargs: dict) -> Any:holder: dict = {"session": None}vae.__dict__[_SESSION_ATTR] = holderz = args[0] if args else kwargs.get("z")diff --git a/studio/backend/tests/test_diffusion_decode_segments.py b/studio/backend/tests/test_diffusion_decode_segments.pynew file mode 100644index 00000000000..17626f88fc3--- /dev/null+++ b/studio/backend/tests/test_diffusion_decode_segments.py@@ -0,0 +1,293 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""diffusion_decode_segments: expandable allocator segments for the span of an image VAE decode (CPU, fake torch)."""++import sys+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_decode_segments as D # noqa: E402+from core.inference import diffusion_vae_oom as O # noqa: E402+++class _FakeTorch:+ def __init__(self, *, segments = (), backend = "native", hip = None, compiling = False, capturing = False,+ setter = True, legacy = False, fail = False): # fmt: skip+ self.calls = []+ self.version = types.SimpleNamespace(hip = hip)++ def set_settings(value):+ if fail:+ raise RuntimeError("refused")+ self.calls.append(value)++ c = types.SimpleNamespace()+ if setter:+ setattr(c, "_cuda_cudaCachingAllocator_set_allocator_settings" if legacy+ else "_accelerator_setAllocatorSettings", set_settings) # fmt: skip+ self._C = c+ self.compiler = types.SimpleNamespace(is_compiling = lambda: compiling)+ self.cuda = types.SimpleNamespace(+ is_current_stream_capturing = lambda: capturing,+ memory_snapshot = lambda: [dict(s) for s in segments],+ memory = types.SimpleNamespace(get_allocator_backend = lambda: backend),+ )+++class _Vae:+ pass+++def _cuda_z():+ return types.SimpleNamespace(device = types.SimpleNamespace(type = "cuda"))+++def fresh(monkeypatch):+ monkeypatch.setattr(D, "_usable", None)+ monkeypatch.setattr(D, "_depth", 0)+ monkeypatch.setattr(D, "_platform_ok", lambda: True)+ for name in D._ALLOC_CONF_VARS + (D.DECODE_EXPANDABLE_ENV,):+ monkeypatch.delenv(name, raising = False)++ def install(fake):+ monkeypatch.setattr(D, "_torch", lambda: fake)+ return fake++ return install+++def _vae():+ vae = _Vae()+ assert D.enable(vae)+ return vae+++def _decode(+ vae,+ fake,+ z = None,+):+ seen = []+ with D.expandable_decode(vae, z if z is not None else _cuda_z()):+ seen.append(list(fake.calls))+ return seen[0]+++def test_flips_on_for_the_decode_and_back_off(fresh):+ fake = fresh(_FakeTorch())+ inside = _decode(_vae(), fake)+ assert inside == ["expandable_segments:True"]+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+++def test_legacy_setter_on_older_torch(fresh):+ fake = fresh(_FakeTorch(legacy = True))+ _decode(_vae(), fake)+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+++def test_nested_decodes_flip_once(fresh):+ fake = fresh(_FakeTorch())+ vae = _vae()+ with D.expandable_decode(vae, _cuda_z()):+ with D.expandable_decode(vae, _cuda_z()):+ assert fake.calls == ["expandable_segments:True"]+ assert fake.calls == ["expandable_segments:True"]+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+++def test_restores_after_a_failing_decode(fresh):+ fake = fresh(_FakeTorch())+ with pytest.raises(ValueError):+ with D.expandable_decode(_vae(), _cuda_z()):+ raise ValueError("decode failed")+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+++ "fake_kwargs",+ [+ {"segments": [{"is_expandable": True}]}, # already on process-wide+ {"backend": "cudaMallocAsync"},+ {"hip": "6.4"},+ {"compiling": True},+ {"capturing": True},+ {"setter": False},+ ],+)+def test_never_touches_the_allocator_when_not_allowed(fresh, fake_kwargs):+ fake = fresh(_FakeTorch(**fake_kwargs))+ _decode(_vae(), fake)+ assert fake.calls == []+++ "conf",+ [+ "max_split_size_mb:128",+ "expandable_segments:False",+ "expandable_segments:True,roundup_power2_divisions:[32:256,64:128,256:64,>:32]",+ "backend:cudaMallocAsync",+ "garbage_collection_threshold:0.6",+ ],+)+def test_user_allocator_config_wins(fresh, monkeypatch, conf):+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", conf)+ fake = fresh(_FakeTorch())+ _decode(_vae(), fake)+ assert fake.calls == []+++def test_bare_expandable_true_export_after_init_still_engages(fresh, monkeypatch):+ monkeypatch.setenv("PYTORCH_ALLOC_CONF", "expandable_segments:True")+ fake = fresh(_FakeTorch(segments = [{"is_expandable": False}]))+ _decode(_vae(), fake)+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+++def test_kill_switch_and_opt_in(fresh, monkeypatch):+ fake = fresh(_FakeTorch())+ _decode(_Vae(), fake) # not opted in: video and other VAEs+ assert fake.calls == []+ vae = _vae()+ monkeypatch.setenv(D.DECODE_EXPANDABLE_ENV, "0")+ _decode(vae, fake)+ assert fake.calls == []+ assert not D.enable(_Vae())+++def test_cpu_and_mps_inputs_untouched(fresh):+ fake = fresh(_FakeTorch())+ for kind in ("cpu", "mps", "xpu"):+ _decode(_vae(), fake, types.SimpleNamespace(device = types.SimpleNamespace(type = kind)))+ assert fake.calls == []+++def test_refused_setting_disables_for_the_process(fresh):+ fake = fresh(_FakeTorch(fail = True))+ vae = _vae()+ ran = []+ with D.expandable_decode(vae, _cuda_z()):+ ran.append(1)+ assert ran == [1] and D._usable is False+++def test_platform_gate(monkeypatch):+ monkeypatch.setattr(D.sys, "platform", "win32")+ assert not D._platform_ok()+ monkeypatch.setattr(D.sys, "platform", "linux")+ monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu")+ assert not D._platform_ok()+ monkeypatch.delenv("WSL_DISTRO_NAME")+ monkeypatch.delenv("WSL_INTEROP", raising = False)+ monkeypatch.setattr(+ D.os,+ "uname",+ lambda: types.SimpleNamespace(release = "5.15.167.4-microsoft-standard-WSL2"),+ raising = False,+ )+ assert not D._platform_ok()+ monkeypatch.setattr(+ D.os, "uname", lambda: types.SimpleNamespace(release = "6.8.0-1015-aws"), raising = False+ )+ monkeypatch.setattr(D.os.path, "exists", lambda p: False)+ assert D._platform_ok()+++def test_conf_keys_keeps_bracketed_lists():+ keys = D._conf_keys("expandable_segments:True, roundup_power2_divisions:[32:256,64:128]")+ assert keys == {"expandable_segments": "true", "roundup_power2_divisions": "[32:256,64:128]"}+++class _ScopeVae(torch.nn.Module):+ def __init__(self, fake):+ super().__init__()+ self.use_tiling = False+ self.fake = fake+ self.during = []++ def enable_tiling(self):+ self.use_tiling = True++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ self.during.append(list(self.fake.calls))+ return z++ def decode(+ self,+ z,+ return_dict = True,+ ):+ return self._decode(z, return_dict = return_dict)+++def test_oom_scope_wraps_the_whole_decode(fresh):+ fake = fresh(_FakeTorch())+ vae = _ScopeVae(fake)+ assert O.install(vae)+ D.enable(vae)+ z = _cuda_z()+ assert vae.decode(z) is z+ assert vae.during == [["expandable_segments:True"]]+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]+ O.uninstall(vae)+++_real_gpu = torch.cuda.is_available() and sys.platform.startswith("linux")+++ not (_real_gpu and getattr(torch.version, "hip", None)), reason = "ROCm device only"+)+def test_rocm_device_never_engages(monkeypatch):+ monkeypatch.setattr(D, "_usable", None)+ monkeypatch.setattr(D, "_depth", 0)+ monkeypatch.setattr(D, "STATS", {"engaged": 0, "skipped": 0})+ vae = _Vae()+ D.enable(vae)+ z = torch.zeros(1, device = "cuda")+ with D.expandable_decode(vae, z):+ torch.empty(64 << 20, dtype = torch.uint8, device = "cuda")+ assert D._usable is False and D.STATS["engaged"] == 0+++ not (_real_gpu and not getattr(torch.version, "hip", None))+ or not D._platform_ok()+ or D._allocator_configured(),+ reason = "Linux CUDA device with the default allocator only",+)+def test_cuda_device_maps_the_decode_into_an_expandable_segment(monkeypatch):+ monkeypatch.setattr(D, "_usable", None)+ monkeypatch.setattr(D, "_depth", 0)+ torch.cuda.init()+ if any(seg.get("is_expandable") for seg in torch.cuda.memory_snapshot()):+ pytest.skip("expandable segments already on in this process")+ vae = _Vae()+ D.enable(vae)+ z = torch.zeros(1, device = "cuda")+ torch.cuda.synchronize()+ torch.cuda.empty_cache() # a cached fixed block would serve the allocation below++ def expandable(t):+ ptr = t.data_ptr()+ for seg in torch.cuda.memory_snapshot():+ if seg["address"] <= ptr < seg["address"] + seg["total_size"]:+ return bool(seg.get("is_expandable"))+ raise AssertionError("tensor not in any segment")++ with D.expandable_decode(vae, z):+ inside = torch.empty(96 << 20, dtype = torch.uint8, device = "cuda")+ assert expandable(inside)+ after = torch.empty(80 << 20, dtype = torch.uint8, device = "cuda")+ assert not expandable(after)+ del inside, after