Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: size CPU / MPS VAE decode tiles from free host memory (CPU 512 px decode 2.1x faster) (#17186)

Daniel Han3be02a67fa2e
diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.py
index 327605b3cc3..58d5260d5d5 100644
--- a/studio/backend/core/inference/diffusion_vae_tiling.py
+++ b/studio/backend/core/inference/diffusion_vae_tiling.py
@@ -116,6 +116,8 @@ DECODE_MIB_PER_LATENT = 1.7
DECODE_MIB_RATIO = 16
FREE_FRACTION = 0.75
MAX_TILE_ENV = "UNSLOTH_DIFFUSION_VAE_MAX_TILE"
+# "0" keeps CPU / MPS decodes at the floor tile (no memory reading), as before the host budget.
+HOST_BUDGET_ENV = "UNSLOTH_DIFFUSION_VAE_HOST_BUDGET"
def decode_mib_per_latent(
@@ -304,6 +306,45 @@ def _free_mib(
return (free - out_bytes) / 2**20, elem / 2
+def host_budget_disabled() -> bool:
+ return (os.environ.get(HOST_BUDGET_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _host_free_mib(vae: Any, z: Any) -> Optional[tuple[float, float]]:
+ """``_free_mib`` for a CPU / MPS decode: the planner's safe budget of the live reading (MPS: available RAM and
+ Metal's working set less what Metal holds; CPU: available RAM; each less the planner reserve), after the fp32
+ output accumulator. None elsewhere, under the kill switch, or with no reading."""
+ kind = getattr(getattr(z, "device", None), "type", None)
+ if kind not in ("cpu", "mps") or host_budget_disabled():
+ return None
+ from .diffusion_memory import (
+ DeviceMemory,
+ _available_system_memory_mib,
+ _host_ram_capacity_mib,
+ _safe_device_budget_mib,
+ _windows_commit_headroom_mib,
+ snapshot_device_memory,
+ )
+
+ if kind == "cpu":
+ # An enforcing cgroup kills the process past its limit (no OOM to retry on), and Windows fails on commit.
+ free = _available_system_memory_mib()
+ commit = _windows_commit_headroom_mib()
+ if free is not None and commit is not None:
+ free = min(free, commit)
+ memory = DeviceMemory("cpu", "cpu", "system_memory", free, _host_ram_capacity_mib())
+ else:
+ memory = snapshot_device_memory(types.SimpleNamespace(device = kind, backend = kind))
+ budget = _safe_device_budget_mib(memory)
+ if budget is None:
+ return None
+ ratio = _geometry(vae)[0]
+ frames = z.shape[2] if z.dim() == 5 else 1
+ out_bytes = 4 * 4 * z.shape[0] * frames * z.shape[-2] * z.shape[-1] * ratio * ratio
+ elem = next(vae.decoder.parameters()).element_size()
+ return budget - out_bytes / 2**20, elem / 2
+
+
def _fused(vae: Any) -> bool:
return (
bool(getattr(vae, "_unsloth_vae_fused_installed", 0))
@@ -328,7 +369,8 @@ def decode_platform_cap(vae: Any, z: Any) -> Optional[int]:
def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:
- """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now; None off CUDA."""
+ """Largest decode tile area (latents) that fits ``FREE_FRACTION`` of the free VRAM now (CPU / MPS: of the safe
+ host budget, ``UNSLOTH_DIFFUSION_VAE_HOST_BUDGET=0`` = None); None on other devices or with no reading."""
raw = (os.environ.get(MAX_TILE_ENV) or "").strip()
if raw.isdigit() and int(raw) > 0:
return int(raw) ** 2
@@ -342,6 +384,9 @@ def decode_tile_budget(vae: Any, z: Any) -> Optional[int]:
stock_tile = vae.__dict__.get("_unsloth_wide_stock_tile") or (stock_tiles(vae) or (0, 0))[0]
large_stock = stock_tile >= tile > TILE_LATENTS
free = _free_mib(vae, z, cached = not large_stock)
+ if free is None:
+ # CPU / MPS: one pool, nothing cached to stall on, so the stock-tile rule above does not apply.
+ free = _host_free_mib(vae, z)
if free is None:
return None
mib, scale = free
diff --git a/studio/backend/tests/test_diffusion_vae_host_tile_budget.py b/studio/backend/tests/test_diffusion_vae_host_tile_budget.py
new file mode 100644
index 00000000000..2141c3d3537
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_host_tile_budget.py
@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU / MPS wide-tile decodes size their tiles from the live host budget instead of always taking the 32-latent
+floor (64x64 latents: 9 floor tiles, 9216 decoded latents, vs one 4096-latent tile)."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_memory as dm # noqa: E402
+from core.inference import diffusion_vae_tiling as vt # noqa: E402
+
+MIB = 2**20
+
+
+class _Decoder(torch.nn.Module):
+ def __init__(self, dtype):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1, dtype = dtype))
+
+
+def _vae(dtype = torch.float32, cls_name = "AutoencoderKLQwenImage"):
+ vae = type(cls_name, (), {})()
+ vae.decoder = _Decoder(dtype)
+ vae._unsloth_wide_geometry = (8, 32, 16)
+ vae._unsloth_wide_stock_tile = 32
+ return vae
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(vt.MAX_TILE_ENV, raising = False)
+ monkeypatch.delenv(vt.HOST_BUDGET_ENV, raising = False)
+
+
+def _memory(
+ monkeypatch,
+ kind,
+ free_mib,
+ total_mib,
+ working_set = None,
+ in_use = 0,
+):
+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (total_mib, free_mib))
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: free_mib)
+ monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: total_mib)
+ monkeypatch.setattr(dm, "_windows_commit_headroom_mib", lambda: None)
+ monkeypatch.setattr(
+ dm, "_metal_working_set_mib", lambda: (working_set, in_use) if working_set else (None, 0)
+ )
+
+
+def _expected(free_mib, total_mib, z, elem, reserve):
+ out = 4 * 4 * z.shape[0] * z.shape[-2] * z.shape[-1] * 64 / MIB
+ per_latent = vt.decode_mib_per_latent(8, False, "AutoencoderKLQwenImage") * elem / 2
+ return max(0, int(vt.FREE_FRACTION * (free_mib - reserve - out) / per_latent))
+
+
+def test_cpu_budget_engages_and_grows_the_tiles(monkeypatch):
+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)
+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)
+ budget = vt.decode_tile_budget(vae, z)
+ # system memory keeps max(1 GiB, 10% of total) back; fp32 prices 2x the bf16 coefficient
+ assert budget == _expected(48 * 1024, 64 * 1024, z, 4, max(1024, int(64 * 1024 * 0.10)))
+ assert budget >= 64 * 64
+ assert vt.choose_tiles(64, 64, budget) == (64, 64)
+ assert vt.choose_tiles(64, 64, None) == (32, 32)
+
+
+def test_tight_cpu_memory_stays_at_the_floor(monkeypatch):
+ _memory(monkeypatch, "cpu", free_mib = 1500, total_mib = 8 * 1024)
+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)
+ budget = vt.decode_tile_budget(vae, z)
+ assert budget is not None and budget < 32 * 32
+ assert vt.choose_tiles(64, 64, budget) == (32, 32)
+
+
+def test_kill_switch_restores_the_floor(monkeypatch):
+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)
+ monkeypatch.setenv(vt.HOST_BUDGET_ENV, "0")
+ vae, z = _vae(), torch.zeros(1, 16, 1, 64, 64)
+ assert vt.decode_tile_budget(vae, z) is None
+ assert vt.choose_tiles(64, 64, vt.decode_tile_budget(vae, z)) == (32, 32)
+
+
+def test_explicit_max_tile_still_wins(monkeypatch):
+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)
+ monkeypatch.setenv(vt.MAX_TILE_ENV, "40")
+ assert vt.decode_tile_budget(_vae(), torch.zeros(1, 16, 1, 64, 64)) == 1600
+
+
+def test_mps_reads_the_planners_unified_budget(monkeypatch):
+ """MPS takes snapshot_device_memory's reading, as the load planner does: Metal's working set less what Metal
+ holds, widened to the RAM reading only when that leaves more."""
+ z = types.SimpleNamespace(device = torch.device("mps"), shape = (1, 16, 1, 64, 64), dim = lambda: 5)
+ vae = _vae(torch.bfloat16)
+ # 64 GB Mac, 20 GiB RAM free, 48 GiB working set of which Metal holds 30: the working set (18 GiB) wins.
+ _memory(monkeypatch, "mps", 20 * 1024, 64 * 1024, working_set = 48 * 1024, in_use = 30 * 1024)
+ assert vt.decode_tile_budget(vae, z) == _expected(
+ 18 * 1024, 48 * 1024, z, 2, int(48 * 1024 * 0.15)
+ )
+ # No working-set reading: RAM less the unified reserve.
+ _memory(monkeypatch, "mps", 40 * 1024, 64 * 1024)
+ assert vt.decode_tile_budget(vae, z) == _expected(
+ 40 * 1024, 64 * 1024, z, 2, max(2048, int(64 * 1024 * 0.20))
+ )
+
+
+def test_cpu_budget_is_capped_by_the_cgroup_and_windows_commit(monkeypatch):
+ """A container's enforcing limit, not host RAM: past it the process is killed, with no OOM to retry on."""
+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 3 * 1024)
+ monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: 4 * 1024)
+ vae, z = _vae(torch.float32), torch.zeros(1, 16, 1, 64, 64)
+ capped = vt.decode_tile_budget(vae, z)
+ assert capped == _expected(3 * 1024, 4 * 1024, z, 4, 1024)
+ assert capped < 64 * 64 and vt.choose_tiles(64, 64, capped) != (64, 64)
+ _memory(monkeypatch, "cpu", free_mib = 48 * 1024, total_mib = 64 * 1024)
+ monkeypatch.setattr(dm, "_windows_commit_headroom_mib", lambda: 3 * 1024)
+ assert vt.decode_tile_budget(vae, z) == _expected(3 * 1024, 64 * 1024, z, 4, 64 * 1024 // 10)
+
+
+def test_no_reading_keeps_the_floor(monkeypatch):
+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (None, None))
+ monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: None)
+ assert vt.decode_tile_budget(_vae(), torch.zeros(1, 16, 1, 64, 64)) is None
+
+
+def test_other_devices_never_take_the_host_reading(monkeypatch):
+ called = []
+ monkeypatch.setattr(dm, "snapshot_device_memory", lambda t: called.append(t))
+ for kind in ("xpu", "meta"):
+ z = types.SimpleNamespace(
+ device = torch.device(kind), shape = (1, 16, 1, 64, 64), dim = lambda: 5
+ )
+ assert vt._host_free_mib(_vae(), z) is None
+ assert not called
+
+
+def test_cuda_path_is_unchanged(monkeypatch):
+ """A CUDA latent never reaches the host reading; its budget is the mem_get_info formula."""
+ monkeypatch.setattr(vt, "_host_free_mib", lambda *a: pytest.fail("host reading on CUDA"))
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (20 * 2**30, 24 * 2**30))
+ monkeypatch.setattr(dm, "cuda_reusable_cache_bytes", lambda device = None: 0)
+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)
+ z = types.SimpleNamespace(
+ device = torch.device("cuda", 0), shape = (1, 16, 1, 64, 64), dim = lambda: 5
+ )
+ vae = _vae(torch.bfloat16)
+ out = 4 * 4 * 64 * 64 * 64 / MIB
+ assert vt.decode_tile_budget(vae, z) == int(
+ vt.FREE_FRACTION
+ * (20 * 1024 - out)
+ / vt.decode_mib_per_latent(8, False, "AutoencoderKLQwenImage")
+ )
+
+
+def test_floor_shortfall_and_untiled_release_stay_cuda_only(monkeypatch):
+ """The floor is still always tried off CUDA, and the untiled-decode release never fires on CPU."""
+ _memory(monkeypatch, "cpu", free_mib = 100, total_mib = 8 * 1024)
+ vae, z = _vae(), torch.zeros(1, 16, 1, 64, 64)
+ assert vt.floor_shortfall(vae, z) is None
+ assert vt._free_mib(vae, z) is None
diff --git a/studio/backend/tests/test_diffusion_vae_tiling.py b/studio/backend/tests/test_diffusion_vae_tiling.py
index e642c84b9c6..0092df69e93 100644
--- a/studio/backend/tests/test_diffusion_vae_tiling.py
+++ b/studio/backend/tests/test_diffusion_vae_tiling.py
@@ -452,6 +452,7 @@ def test_budget_comes_from_free_vram_and_the_env_caps_it(monkeypatch):
z = torch.zeros(1, 4, 1, 64, 64)
assert vt.decode_tile_budget(vae, z) == vt.TILE_LATENTS**2
monkeypatch.delenv(vt.MAX_TILE_ENV)
+ monkeypatch.setenv(vt.HOST_BUDGET_ENV, "0")
assert vt.decode_tile_budget(vae, z) is None
assert vt.choose_tiles(64, 64, None) == (32, 32)
diff --git a/studio/backend/tests/test_vae_tile_geometry.py b/studio/backend/tests/test_vae_tile_geometry.py
index 7b2bbedf727..35b8cd0270d 100644
--- a/studio/backend/tests/test_vae_tile_geometry.py
+++ b/studio/backend/tests/test_vae_tile_geometry.py
@@ -299,14 +299,16 @@ def _latent_channels(config: dict) -> int:
def _no_memory_reading():
"""Probe the grid a decode takes with no free-memory reading: the smallest tiles, the same on every host.
- Budgeted planners (video_ltx2_vae_tiles) widen tiles, or go untiled, when the reading says they fit; on a CPU
+ Budgeted planners (video_ltx2_vae_tiles, diffusion_vae_tiling) widen tiles, or go untiled, when the reading says they fit; on a CPU
host that reading is host RAM, so an unpinned probe would see whatever the runner has free.
"""
- mod = _module("video_vae_untiled")
- if mod is None or not hasattr(mod, "_free_bytes"):
- yield
- return
- with mock.patch.object(mod, "_free_bytes", lambda device: None):
+ with contextlib.ExitStack() as stack:
+ mod = _module("video_vae_untiled")
+ if mod is not None and hasattr(mod, "_free_bytes"):
+ stack.enter_context(mock.patch.object(mod, "_free_bytes", lambda device: None))
+ image = _module("diffusion_vae_tiling")
+ if image is not None and hasattr(image, "_host_free_mib"):
+ stack.enter_context(mock.patch.object(image, "_host_free_mib", lambda vae, z: None))
yield