Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: size CPU / MPS VAE decode tiles from free host memory (CPU 512 px decode 2.1x faster) (#17186)
Daniel Han3be02a67fa2e
diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.pyindex 327605b3cc3..58d5260d5d5 100644--- a/studio/backend/core/inference/diffusion_vae_tiling.py+++ b/studio/backend/core/inference/diffusion_vae_tiling.py@@ -116,6 +116,8 @@ DECODE_MIB_PER_LATENT = 1.7DECODE_MIB_RATIO = 16FREE_FRACTION = 0.75MAX_TILE_ENV = "UNSLOTH_DIFFUSION_VAE_MAX_TILE"+# "0" keeps CPU / MPS decodes at the floor tile (no memory reading), as before the host budget.+HOST_BUDGET_ENV = "UNSLOTH_DIFFUSION_VAE_HOST_BUDGET"def decode_mib_per_latent(@@ -304,6 +306,45 @@ def _free_mib(return (free - out_bytes) / 2**20, elem / 2+def host_budget_disabled() -> bool:+ return (os.environ.get(HOST_BUDGET_ENV) or "").strip().lower() in ("0", "off", "false", "no")+++def _host_free_mib(vae: Any, z: Any) -> Optional[tuple[float, float]]:+ """``_free_mib`` for a CPU / MPS decode: the planner's safe budget of the live reading (MPS: available RAM and+ Metal's working set less what Metal holds; CPU: available RAM; each less the planner reserve), after the fp32+ output accumulator. None elsewhere, under the kill switch, or with no reading."""+ kind = getattr(getattr(z, "device", None), "type", None)+ if kind not in ("cpu", "mps") or host_budget_disabled():+ return None+ from .diffusion_memory import (+ DeviceMemory,+ _available_system_memory_mib,+ _host_ram_capacity_mib,+ _safe_device_budget_mib,+ _windows_commit_headroom_mib,+ snapshot_device_memory,+ )++ if kind == "cpu":+ # An enforcing cgroup kills the process past its limit (no OOM to retry on), and Windows fails on commit.+ free = _available_system_memory_mib()+ commit = _windows_commit_headroom_mib()+ if free is not None and commit is not None:+ free = min(free, commit)+ memory = DeviceMemory("cpu", "cpu", "system_memory", free, _host_ram_capacity_mib())+ else:+ memory = snapshot_device_memory(types.SimpleNamespace(device = kind, backend = kind))+ budget = _safe_device_budget_mib(memory)+ if budget is None:+ return None+ ratio = _geometry(vae)[0]+ frames = z.shape[2] if z.dim() == 5 else 1+ out_bytes = 4 * 4 * z.shape[0] * frames * z.shape[-2] * z.shape[-1] * ratio * ratio+ elem = next(vae.decoder.parameters()).element_size()+ return budget - out_bytes / 2**20, elem / 2++def _fused(vae: Any) -> bool:return (bool(getattr(vae, "_unsloth_vae_fused_installed", 0))@@ -328,7 +369,8 @@ def decode_platform_cap(vae: Any, z: Any) -> Optional[int]:def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:- """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now; None off CUDA."""+ """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now (CPU / MPS: of the safe+ host budget, ``UNSLOTH_DIFFUSION_VAE_HOST_BUDGET=0`` = None); None on other devices or with no reading."""raw = (os.environ.get(MAX_TILE_ENV) or "").strip()if raw.isdigit() and int(raw) > 0:return int(raw) ** 2@@ -342,6 +384,9 @@ def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:stock_tile = vae.__dict__.get("_unsloth_wide_stock_tile") or (stock_tiles(vae) or (0, 0))[0]large_stock = stock_tile >= tile > TILE_LATENTSfree = _free_mib(vae, z, cached = not large_stock)+ if free is None:+ # CPU / MPS: one pool, nothing cached to stall on, so the stock-tile rule above does not apply.+ free = _host_free_mib(vae, z)if free is None:return Nonemib, scale = freediff --git a/studio/backend/tests/test_diffusion_vae_host_tile_budget.py b/studio/backend/tests/test_diffusion_vae_host_tile_budget.pynew file mode 100644index 00000000000..2141c3d3537--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_host_tile_budget.py@@ -0,0 +1,168 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU / MPS wide-tile decodes size their tiles from the live host budget instead of always taking the 32-latent+floor (64x64 latents: 9 floor tiles, 9216 decoded latents, vs one 4096-latent tile)."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_memory as dm # noqa: E402+from core.inference import diffusion_vae_tiling as vt # noqa: E402++MIB = 2**20+++class _Decoder(torch.nn.Module):+ def __init__(self, dtype):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1, dtype = dtype))+++def _vae(dtype = torch.float32, cls_name = "AutoencoderKLQwenImage"):+ vae = type(cls_name, (), {})()+ vae.decoder = _Decoder(dtype)+ vae._unsloth_wide_geometry = (8, 32, 16)+ vae._unsloth_wide_stock_tile = 32+ return vae++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(vt.MAX_TILE_ENV, raising = False)+ monkeypatch.delenv(vt.HOST_BUDGET_ENV, raising = False)+++def _memory(+ monkeypatch,+ kind,+ free_mib,+ total_mib,+ working_set = None,+ in_use = 0,+):+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (total_mib, free_mib))+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: free_mib)+ monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: total_mib)+ monkeypatch.setattr(dm, "_windows_commit_headroom_mib", lambda: None)+ monkeypatch.setattr(+ dm, "_metal_working_set_mib", lambda: (working_set, in_use) if working_set else (None, 0)+ )+++def _expected(free_mib, total_mib, z, elem, reserve):+ out = 4 * 4 * z.shape[0] * z.shape[-2] * z.shape[-1] * 64 / MIB+ per_latent = vt.decode_mib_per_latent(8, False, "AutoencoderKLQwenImage") * elem / 2+ return max(0, int(vt.FREE_FRACTION * (free_mib - reserve - out) / per_latent))+++def test_cpu_budget_engages_and_grows_the_tiles(monkeypatch):+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)+ budget = vt.decode_tile_budget(vae, z)+ # system memory keeps max(1 GiB, 10% of total) back; fp32 prices 2x the bf16 coefficient+ assert budget == _expected(48 * 1024, 64 * 1024, z, 4, max(1024, int(64 * 1024 * 0.10)))+ assert budget >= 64 * 64+ assert vt.choose_tiles(64, 64, budget) == (64, 64)+ assert vt.choose_tiles(64, 64, None) == (32, 32)+++def test_tight_cpu_memory_stays_at_the_floor(monkeypatch):+ _memory(monkeypatch, "cpu", free_mib = 1500, total_mib = 8 * 1024)+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)+ budget = vt.decode_tile_budget(vae, z)+ assert budget is not None and budget < 32 * 32+ assert vt.choose_tiles(64, 64, budget) == (32, 32)+++def test_kill_switch_restores_the_floor(monkeypatch):+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)+ monkeypatch.setenv(vt.HOST_BUDGET_ENV, "0")+ vae, z = _vae(), torch.zeros(1, 16, 1, 64, 64)+ assert vt.decode_tile_budget(vae, z) is None+ assert vt.choose_tiles(64, 64, vt.decode_tile_budget(vae, z)) == (32, 32)+++def test_explicit_max_tile_still_wins(monkeypatch):+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)+ monkeypatch.setenv(vt.MAX_TILE_ENV, "40")+ assert vt.decode_tile_budget(_vae(), torch.zeros(1, 16, 1, 64, 64)) == 1600+++def test_mps_reads_the_planners_unified_budget(monkeypatch):+ """MPS takes snapshot_device_memory's reading, as the load planner does: Metal's working set less what Metal+ holds, widened to the RAM reading only when that leaves more."""+ z = types.SimpleNamespace(device = torch.device("mps"), shape = (1, 16, 1, 64, 64), dim = lambda: 5)+ vae = _vae(torch.bfloat16)+ # 64 GB Mac, 20 GiB RAM free, 48 GiB working set of which Metal holds 30: the working set (18 GiB) wins.+ _memory(monkeypatch, "mps", 20 * 1024, 64 * 1024, working_set = 48 * 1024, in_use = 30 * 1024)+ assert vt.decode_tile_budget(vae, z) == _expected(+ 18 * 1024, 48 * 1024, z, 2, int(48 * 1024 * 0.15)+ )+ # No working-set reading: RAM less the unified reserve.+ _memory(monkeypatch, "mps", 40 * 1024, 64 * 1024)+ assert vt.decode_tile_budget(vae, z) == _expected(+ 40 * 1024, 64 * 1024, z, 2, max(2048, int(64 * 1024 * 0.20))+ )+++def test_cpu_budget_is_capped_by_the_cgroup_and_windows_commit(monkeypatch):+ """A container's enforcing limit, not host RAM: past it the process is killed, with no OOM to retry on."""+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 3 * 1024)+ monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: 4 * 1024)+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)+ capped = vt.decode_tile_budget(vae, z)+ assert capped == _expected(3 * 1024, 4 * 1024, z, 4, 1024)+ assert capped < 64 * 64 and vt.choose_tiles(64, 64, capped) != (64, 64)+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)+ monkeypatch.setattr(dm, "_windows_commit_headroom_mib", lambda: 3 * 1024)+ assert vt.decode_tile_budget(vae, z) == _expected(3 * 1024, 64 * 1024, z, 4, 64 * 1024 // 10)+++def test_no_reading_keeps_the_floor(monkeypatch):+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (None, None))+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: None)+ assert vt.decode_tile_budget(_vae(), torch.zeros(1, 16, 1, 64, 64)) is None+++def test_other_devices_never_take_the_host_reading(monkeypatch):+ called = []+ monkeypatch.setattr(dm, "snapshot_device_memory", lambda t: called.append(t))+ for kind in ("xpu", "meta"):+ z = types.SimpleNamespace(+ device = torch.device(kind), shape = (1, 16, 1, 64, 64), dim = lambda: 5+ )+ assert vt._host_free_mib(_vae(), z) is None+ assert not called+++def test_cuda_path_is_unchanged(monkeypatch):+ """A CUDA latent never reaches the host reading; its budget is the mem_get_info formula."""+ monkeypatch.setattr(vt, "_host_free_mib", lambda *a: pytest.fail("host reading on CUDA"))+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (20 * 2**30, 24 * 2**30))+ monkeypatch.setattr(dm, "cuda_reusable_cache_bytes", lambda device = None: 0)+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)+ z = types.SimpleNamespace(+ device = torch.device("cuda", 0), shape = (1, 16, 1, 64, 64), dim = lambda: 5+ )+ vae = _vae(torch.bfloat16)+ out = 4 * 4 * 64 * 64 * 64 / MIB+ assert vt.decode_tile_budget(vae, z) == int(+ vt.FREE_FRACTION+ * (20 * 1024 - out)+ / vt.decode_mib_per_latent(8, False, "AutoencoderKLQwenImage")+ )+++def test_floor_shortfall_and_untiled_release_stay_cuda_only(monkeypatch):+ """The floor is still always tried off CUDA, and the untiled-decode release never fires on CPU."""+ _memory(monkeypatch, "cpu", free_mib = 100, total_mib = 8 * 1024)+ vae, z = _vae(), torch.zeros(1, 16, 1, 64, 64)+ assert vt.floor_shortfall(vae, z) is None+ assert vt._free_mib(vae, z) is Nonediff --git a/studio/backend/tests/test_diffusion_vae_tiling.py b/studio/backend/tests/test_diffusion_vae_tiling.pyindex e642c84b9c6..0092df69e93 100644--- a/studio/backend/tests/test_diffusion_vae_tiling.py+++ b/studio/backend/tests/test_diffusion_vae_tiling.py@@ -452,6 +452,7 @@ def test_budget_comes_from_free_vram_and_the_env_caps_it(monkeypatch):z = torch.zeros(1, 4, 1, 64, 64)assert vt.decode_tile_budget(vae, z) == vt.TILE_LATENTS**2monkeypatch.delenv(vt.MAX_TILE_ENV)+ monkeypatch.setenv(vt.HOST_BUDGET_ENV, "0")assert vt.decode_tile_budget(vae, z) is Noneassert vt.choose_tiles(64, 64, None) == (32, 32)diff --git a/studio/backend/tests/test_vae_tile_geometry.py b/studio/backend/tests/test_vae_tile_geometry.pyindex 7b2bbedf727..35b8cd0270d 100644--- a/studio/backend/tests/test_vae_tile_geometry.py+++ b/studio/backend/tests/test_vae_tile_geometry.py@@ -299,14 +299,16 @@ def _latent_channels(config: dict) -> int:def _no_memory_reading():"""Probe the grid a decode takes with no free-memory reading: the smallest tiles, the same on every host.- Budgeted planners (video_ltx2_vae_tiles) widen tiles, or go untiled, when the reading says they fit; on a CPU+ Budgeted planners (video_ltx2_vae_tiles, diffusion_vae_tiling) widen tiles, or go untiled, when the reading says they fit; on a CPUhost that reading is host RAM, so an unpinned probe would see whatever the runner has free."""- mod = _module("video_vae_untiled")- if mod is None or not hasattr(mod, "_free_bytes"):- yield- return- with mock.patch.object(mod, "_free_bytes", lambda device: None):+ with contextlib.ExitStack() as stack:+ mod = _module("video_vae_untiled")+ if mod is not None and hasattr(mod, "_free_bytes"):+ stack.enter_context(mock.patch.object(mod, "_free_bytes", lambda device: None))+ image = _module("diffusion_vae_tiling")+ if image is not None and hasattr(image, "_host_free_mib"):+ stack.enter_context(mock.patch.object(image, "_host_free_mib", lambda vae, z: None))yield