Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: decode image VAEs into one expandable allocator segment (SDXL holds 1.15 GiB less VRAM) (#17045)

Daniel Han570619d455a6
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 3a1ea937789..d762f3cba5a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -10880,7 +10880,11 @@ class DiffusionBackend:
# After the speed layer, so it wraps the fused / single-frame _decode the layer installed.
try:
from .diffusion_vae_oom import install as install_vae_oom_retry
+
install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)
+ from .diffusion_decode_segments import enable as enable_decode_expandable
+
+ enable_decode_expandable(getattr(pipe, "vae", None))
except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
logger.warning("diffusion.vae_oom: not installed: %s", exc)
self._raise_if_load_cancelled(_load_token)
diff --git a/studio/backend/core/inference/diffusion_decode_segments.py b/studio/backend/core/inference/diffusion_decode_segments.py
new file mode 100644
index 00000000000..9957f46eda5
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_decode_segments.py
@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Run each image VAE decode in one expandable allocator segment (Linux CUDA, native allocator): its growing
+activations and cuDNN workspaces otherwise each map a new fixed segment (SDXL 1024^2: 11.3 -> 10.1 GiB reserved).
+UNSLOTH_DIFFUSION_DECODE_EXPANDABLE=0 turns it off."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import sys
+import threading
+from typing import Any, Iterator, Optional
+
+from .rocm_bf16 import is_rocm_torch
+
+DECODE_EXPANDABLE_ENV = "UNSLOTH_DIFFUSION_DECODE_EXPANDABLE"
+ENABLED_ATTR = "_unsloth_decode_expandable"
+_ALLOC_CONF_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF")
+
+_lock = threading.Lock()
+_depth = 0
+_usable: Optional[bool] = None
+STATS = {"engaged": 0, "skipped": 0}
+
+
+def _env_off() -> bool:
+ return os.environ.get(DECODE_EXPANDABLE_ENV, "").strip().lower() in ("0", "false", "no", "off")
+
+
+def _conf_keys(value: str) -> dict:
+ out, part, depth = {}, "", 0
+ for ch in value + ",":
+ if ch == "[":
+ depth += 1
+ elif ch == "]":
+ depth = max(0, depth - 1)
+ if ch == "," and depth == 0:
+ key, _, val = part.partition(":")
+ if key.strip():
+ out[key.strip().lower()] = val.strip().lower()
+ part = ""
+ else:
+ part += ch
+ return out
+
+
+def _allocator_configured() -> bool:
+ """Options a runtime setting would reset; a bare ``expandable_segments:True`` (unsloth_zoo, after CUDA init) is not."""
+ for name in _ALLOC_CONF_VARS:
+ keys = _conf_keys(os.environ.get(name) or "")
+ if any(k != "expandable_segments" for k in keys):
+ return True
+ if keys.get("expandable_segments", "true") != "true":
+ return True
+ return False
+
+
+def _platform_ok() -> bool:
+ if not sys.platform.startswith("linux"):
+ return False
+ if os.environ.get("WSL_DISTRO_NAME") or os.environ.get("WSL_INTEROP"):
+ return False
+ try:
+ release = os.uname().release.lower()
+ except Exception: # noqa: BLE001
+ return False
+ if "microsoft" in release or "tegra" in release or os.path.exists("/etc/nv_tegra_release"):
+ return False
+ return True
+
+
+def _setter(torch: Any) -> Any:
+ c = getattr(torch, "_C", None)
+ return getattr(c, "_accelerator_setAllocatorSettings", None) or getattr(
+ c, "_cuda_cudaCachingAllocator_set_allocator_settings", None
+ )
+
+
+def _decide(torch: Any) -> bool:
+ if not _platform_ok() or _allocator_configured():
+ return False
+ if is_rocm_torch(torch):
+ return False
+ if _setter(torch) is None:
+ return False
+ try:
+ if torch.cuda.memory.get_allocator_backend() != "native":
+ return False
+ # Already on process-wide: switching it off after the decode would be wrong.
+ if any(seg.get("is_expandable") for seg in torch.cuda.memory_snapshot()):
+ return False
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+def _torch() -> Any:
+ return sys.modules.get("torch")
+
+
+def _cuda_tensor(z: Any) -> bool:
+ device = getattr(z, "device", None)
+ return getattr(device, "type", None) == "cuda"
+
+
+def _busy(torch: Any) -> bool:
+ try:
+ compiler = getattr(torch, "compiler", None)
+ if compiler is not None and compiler.is_compiling():
+ return True
+ return bool(torch.cuda.is_current_stream_capturing())
+ except Exception: # noqa: BLE001
+ return True
+
+
+def expandable_decode(vae: Any, z: Any) -> Iterator[None]:
+ global _depth, _usable
+ if not vae.__dict__.get(ENABLED_ATTR, False) or _env_off() or not _cuda_tensor(z):
+ yield
+ return
+ torch = _torch()
+ if torch is None or _busy(torch):
+ yield
+ return
+ engaged = False
+ with _lock:
+ if _usable is None:
+ _usable = _decide(torch)
+ if _usable:
+ try:
+ if _depth == 0:
+ _setter(torch)("expandable_segments:True")
+ _depth += 1
+ engaged = True
+ except Exception: # noqa: BLE001 - refused: never retry
+ _usable = False
+ if engaged:
+ STATS["engaged"] += 1
+ else:
+ STATS["skipped"] += 1
+ try:
+ yield
+ finally:
+ if engaged:
+ with _lock:
+ _depth -= 1
+ if _depth == 0:
+ try:
+ _setter(torch)("expandable_segments:False")
+ except Exception: # noqa: BLE001
+ _usable = False
+
+
+def enable(vae: Any) -> bool:
+ if vae is None or _env_off():
+ return False
+ try:
+ vae.__dict__[ENABLED_ATTR] = True
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+__all__ = ["DECODE_EXPANDABLE_ENV", "ENABLED_ATTR", "STATS", "enable", "expandable_decode"]
diff --git a/studio/backend/core/inference/diffusion_vae_oom.py b/studio/backend/core/inference/diffusion_vae_oom.py
index 344db8b5284..e6a028fd8a9 100644
--- a/studio/backend/core/inference/diffusion_vae_oom.py
+++ b/studio/backend/core/inference/diffusion_vae_oom.py
@@ -14,6 +14,7 @@ import weakref
from typing import Any, Callable, Optional
from . import decode_layers
+from .diffusion_decode_segments import expandable_decode
OOM_RETRY_ENV = "UNSLOTH_VAE_OOM_RETRY"
HOST_MARGIN_MIB = 2048
@@ -1115,6 +1116,10 @@ def _install_scope(vae: Any, logger: Any = None) -> bool:
def decode(*args: Any, **kwargs: Any) -> Any:
if _compiling() or oom_retry_disabled() or _scope_session(vae) is not None:
return inner(*args, **kwargs)
+ with expandable_decode(vae, args[0] if args else kwargs.get("z")):
+ return _scoped(args, kwargs)
+
+ def _scoped(args: tuple, kwargs: dict) -> Any:
holder: dict = {"session": None}
vae.__dict__[_SESSION_ATTR] = holder
z = args[0] if args else kwargs.get("z")
diff --git a/studio/backend/tests/test_diffusion_decode_segments.py b/studio/backend/tests/test_diffusion_decode_segments.py
new file mode 100644
index 00000000000..17626f88fc3
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_decode_segments.py
@@ -0,0 +1,293 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""diffusion_decode_segments: expandable allocator segments for the span of an image VAE decode (CPU, fake torch)."""
+
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_decode_segments as D # noqa: E402
+from core.inference import diffusion_vae_oom as O # noqa: E402
+
+
+class _FakeTorch:
+ def __init__(self, *, segments = (), backend = "native", hip = None, compiling = False, capturing = False,
+ setter = True, legacy = False, fail = False): # fmt: skip
+ self.calls = []
+ self.version = types.SimpleNamespace(hip = hip)
+
+ def set_settings(value):
+ if fail:
+ raise RuntimeError("refused")
+ self.calls.append(value)
+
+ c = types.SimpleNamespace()
+ if setter:
+ setattr(c, "_cuda_cudaCachingAllocator_set_allocator_settings" if legacy
+ else "_accelerator_setAllocatorSettings", set_settings) # fmt: skip
+ self._C = c
+ self.compiler = types.SimpleNamespace(is_compiling = lambda: compiling)
+ self.cuda = types.SimpleNamespace(
+ is_current_stream_capturing = lambda: capturing,
+ memory_snapshot = lambda: [dict(s) for s in segments],
+ memory = types.SimpleNamespace(get_allocator_backend = lambda: backend),
+ )
+
+
+class _Vae:
+ pass
+
+
+def _cuda_z():
+ return types.SimpleNamespace(device = types.SimpleNamespace(type = "cuda"))
+
+
+def fresh(monkeypatch):
+ monkeypatch.setattr(D, "_usable", None)
+ monkeypatch.setattr(D, "_depth", 0)
+ monkeypatch.setattr(D, "_platform_ok", lambda: True)
+ for name in D._ALLOC_CONF_VARS + (D.DECODE_EXPANDABLE_ENV,):
+ monkeypatch.delenv(name, raising = False)
+
+ def install(fake):
+ monkeypatch.setattr(D, "_torch", lambda: fake)
+ return fake
+
+ return install
+
+
+def _vae():
+ vae = _Vae()
+ assert D.enable(vae)
+ return vae
+
+
+def _decode(
+ vae,
+ fake,
+ z = None,
+):
+ seen = []
+ with D.expandable_decode(vae, z if z is not None else _cuda_z()):
+ seen.append(list(fake.calls))
+ return seen[0]
+
+
+def test_flips_on_for_the_decode_and_back_off(fresh):
+ fake = fresh(_FakeTorch())
+ inside = _decode(_vae(), fake)
+ assert inside == ["expandable_segments:True"]
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+
+
+def test_legacy_setter_on_older_torch(fresh):
+ fake = fresh(_FakeTorch(legacy = True))
+ _decode(_vae(), fake)
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+
+
+def test_nested_decodes_flip_once(fresh):
+ fake = fresh(_FakeTorch())
+ vae = _vae()
+ with D.expandable_decode(vae, _cuda_z()):
+ with D.expandable_decode(vae, _cuda_z()):
+ assert fake.calls == ["expandable_segments:True"]
+ assert fake.calls == ["expandable_segments:True"]
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+
+
+def test_restores_after_a_failing_decode(fresh):
+ fake = fresh(_FakeTorch())
+ with pytest.raises(ValueError):
+ with D.expandable_decode(_vae(), _cuda_z()):
+ raise ValueError("decode failed")
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+
+
+ "fake_kwargs",
+ [
+ {"segments": [{"is_expandable": True}]}, # already on process-wide
+ {"backend": "cudaMallocAsync"},
+ {"hip": "6.4"},
+ {"compiling": True},
+ {"capturing": True},
+ {"setter": False},
+ ],
+)
+def test_never_touches_the_allocator_when_not_allowed(fresh, fake_kwargs):
+ fake = fresh(_FakeTorch(**fake_kwargs))
+ _decode(_vae(), fake)
+ assert fake.calls == []
+
+
+ "conf",
+ [
+ "max_split_size_mb:128",
+ "expandable_segments:False",
+ "expandable_segments:True,roundup_power2_divisions:[32:256,64:128,256:64,>:32]",
+ "backend:cudaMallocAsync",
+ "garbage_collection_threshold:0.6",
+ ],
+)
+def test_user_allocator_config_wins(fresh, monkeypatch, conf):
+ monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", conf)
+ fake = fresh(_FakeTorch())
+ _decode(_vae(), fake)
+ assert fake.calls == []
+
+
+def test_bare_expandable_true_export_after_init_still_engages(fresh, monkeypatch):
+ monkeypatch.setenv("PYTORCH_ALLOC_CONF", "expandable_segments:True")
+ fake = fresh(_FakeTorch(segments = [{"is_expandable": False}]))
+ _decode(_vae(), fake)
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+
+
+def test_kill_switch_and_opt_in(fresh, monkeypatch):
+ fake = fresh(_FakeTorch())
+ _decode(_Vae(), fake) # not opted in: video and other VAEs
+ assert fake.calls == []
+ vae = _vae()
+ monkeypatch.setenv(D.DECODE_EXPANDABLE_ENV, "0")
+ _decode(vae, fake)
+ assert fake.calls == []
+ assert not D.enable(_Vae())
+
+
+def test_cpu_and_mps_inputs_untouched(fresh):
+ fake = fresh(_FakeTorch())
+ for kind in ("cpu", "mps", "xpu"):
+ _decode(_vae(), fake, types.SimpleNamespace(device = types.SimpleNamespace(type = kind)))
+ assert fake.calls == []
+
+
+def test_refused_setting_disables_for_the_process(fresh):
+ fake = fresh(_FakeTorch(fail = True))
+ vae = _vae()
+ ran = []
+ with D.expandable_decode(vae, _cuda_z()):
+ ran.append(1)
+ assert ran == [1] and D._usable is False
+
+
+def test_platform_gate(monkeypatch):
+ monkeypatch.setattr(D.sys, "platform", "win32")
+ assert not D._platform_ok()
+ monkeypatch.setattr(D.sys, "platform", "linux")
+ monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu")
+ assert not D._platform_ok()
+ monkeypatch.delenv("WSL_DISTRO_NAME")
+ monkeypatch.delenv("WSL_INTEROP", raising = False)
+ monkeypatch.setattr(
+ D.os,
+ "uname",
+ lambda: types.SimpleNamespace(release = "5.15.167.4-microsoft-standard-WSL2"),
+ raising = False,
+ )
+ assert not D._platform_ok()
+ monkeypatch.setattr(
+ D.os, "uname", lambda: types.SimpleNamespace(release = "6.8.0-1015-aws"), raising = False
+ )
+ monkeypatch.setattr(D.os.path, "exists", lambda p: False)
+ assert D._platform_ok()
+
+
+def test_conf_keys_keeps_bracketed_lists():
+ keys = D._conf_keys("expandable_segments:True, roundup_power2_divisions:[32:256,64:128]")
+ assert keys == {"expandable_segments": "true", "roundup_power2_divisions": "[32:256,64:128]"}
+
+
+class _ScopeVae(torch.nn.Module):
+ def __init__(self, fake):
+ super().__init__()
+ self.use_tiling = False
+ self.fake = fake
+ self.during = []
+
+ def enable_tiling(self):
+ self.use_tiling = True
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ self.during.append(list(self.fake.calls))
+ return z
+
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return self._decode(z, return_dict = return_dict)
+
+
+def test_oom_scope_wraps_the_whole_decode(fresh):
+ fake = fresh(_FakeTorch())
+ vae = _ScopeVae(fake)
+ assert O.install(vae)
+ D.enable(vae)
+ z = _cuda_z()
+ assert vae.decode(z) is z
+ assert vae.during == [["expandable_segments:True"]]
+ assert fake.calls == ["expandable_segments:True", "expandable_segments:False"]
+ O.uninstall(vae)
+
+
+_real_gpu = torch.cuda.is_available() and sys.platform.startswith("linux")
+
+
+ not (_real_gpu and getattr(torch.version, "hip", None)), reason = "ROCm device only"
+)
+def test_rocm_device_never_engages(monkeypatch):
+ monkeypatch.setattr(D, "_usable", None)
+ monkeypatch.setattr(D, "_depth", 0)
+ monkeypatch.setattr(D, "STATS", {"engaged": 0, "skipped": 0})
+ vae = _Vae()
+ D.enable(vae)
+ z = torch.zeros(1, device = "cuda")
+ with D.expandable_decode(vae, z):
+ torch.empty(64 << 20, dtype = torch.uint8, device = "cuda")
+ assert D._usable is False and D.STATS["engaged"] == 0
+
+
+ not (_real_gpu and not getattr(torch.version, "hip", None))
+ or not D._platform_ok()
+ or D._allocator_configured(),
+ reason = "Linux CUDA device with the default allocator only",
+)
+def test_cuda_device_maps_the_decode_into_an_expandable_segment(monkeypatch):
+ monkeypatch.setattr(D, "_usable", None)
+ monkeypatch.setattr(D, "_depth", 0)
+ torch.cuda.init()
+ if any(seg.get("is_expandable") for seg in torch.cuda.memory_snapshot()):
+ pytest.skip("expandable segments already on in this process")
+ vae = _Vae()
+ D.enable(vae)
+ z = torch.zeros(1, device = "cuda")
+ torch.cuda.synchronize()
+ torch.cuda.empty_cache() # a cached fixed block would serve the allocation below
+
+ def expandable(t):
+ ptr = t.data_ptr()
+ for seg in torch.cuda.memory_snapshot():
+ if seg["address"] <= ptr < seg["address"] + seg["total_size"]:
+ return bool(seg.get("is_expandable"))
+ raise AssertionError("tensor not in any segment")
+
+ with D.expandable_decode(vae, z):
+ inside = torch.empty(96 << 20, dtype = torch.uint8, device = "cuda")
+ assert expandable(inside)
+ after = torch.empty(80 << 20, dtype = torch.uint8, device = "cuda")
+ assert not expandable(after)
+ del inside, after