Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MLX images: tile the VAE decode when the untiled one would not fit (#14796)

Daniel Han9a822f490bf9
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index 0812e11aac2..f2fcd582bfe 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -158,6 +158,32 @@ def compute_precision_name(chip: Optional[str] = None) -> str:
return "float16" if ("apple m1" in name or "apple m2" in name) else "bfloat16"
+# Untiled VAE decode working set above the weights (mflux Z-Image VAE, bf16, Apple M1 Metal): 1.99 GiB at 512x512 and
+# 6.64 GiB at 1024x1024, i.e. about 0.44 GiB plus 6.35 KB per output pixel. Tiled (mflux's 512 px tiles) it stays at
+# 2.0 GiB at 1024x1024; a 512x512 image is a single tile, so tiling changes nothing there.
+_VAE_DECODE_FIXED_BYTES = int(0.44 * 1024**3)
+_VAE_DECODE_BYTES_PER_PIXEL = 6350
+VAE_TILING_ENV = "UNSLOTH_DIFFUSION_MLX_VAE_TILING"
+
+
+def untiled_decode_bytes(width: int, height: int) -> int:
+ return _VAE_DECODE_FIXED_BYTES + _VAE_DECODE_BYTES_PER_PIXEL * int(width) * int(height)
+
+
+def vae_tiling_needed(width: int, height: int, free_bytes: Optional[int]) -> bool:
+ """Tile the decode only when the untiled one would not fit what the working set has left, as ComfyUI tiles only
+ after an out-of-memory decode; unified memory pages instead of failing, so the check comes first.
+ ``UNSLOTH_DIFFUSION_MLX_VAE_TILING=1`` / ``0`` forces it on / off."""
+ raw = os.environ.get(VAE_TILING_ENV, "").strip().lower()
+ if raw in ("1", "on", "true", "yes"):
+ return True
+ if raw in ("0", "off", "false", "no"):
+ return False
+ if free_bytes is None:
+ return False
+ return untiled_decode_bytes(width, height) > free_bytes
+
+
class MfluxRuntime:
"""The only code that imports mlx / mflux. Tests replace it with a fake (``backend.runtime_factory``)."""
@@ -226,6 +252,7 @@ class MfluxRuntime:
except MlxCancelled as exc:
raise StopImageGenerationException(str(exc)) from exc
+ self._plan_vae_tiling(model, width, height)
callback = _StepCallback()
model.callbacks.register(callback)
cancelled = False
@@ -256,6 +283,31 @@ class MfluxRuntime:
raise MlxCancelled("cancelled")
return result.image
+ def free_working_set_bytes(self) -> Optional[int]:
+ """Metal's recommended working set minus what MLX holds live now (the resident model), or None."""
+ import mlx.core as mx
+
+ try:
+ info = mx.device_info() if hasattr(mx, "device_info") else mx.metal.device_info()
+ budget = int(info.get("max_recommended_working_set_size") or 0)
+ except Exception: # noqa: BLE001 -- no Metal reading: leave the decode untiled
+ return None
+ if budget <= 0:
+ return None
+ return budget - int(mx.get_active_memory())
+
+ def _plan_vae_tiling(self, model: Any, width: int, height: int) -> None:
+ from mflux.models.common.vae.tiling_config import TilingConfig
+
+ tiled = TilingConfig.may_tile_implicitly(model) and vae_tiling_needed(
+ width, height, self.free_working_set_bytes()
+ )
+ model.tiling_config = TilingConfig() if tiled else None
+ self.last_vae_tiled = tiled
+
+ def vae_tiled(self) -> bool:
+ return bool(getattr(self, "last_vae_tiled", False))
+
def cache_text_encodings(
self,
model: Any,
@@ -836,6 +888,9 @@ class MlxDiffusionBackend:
if _cancel_event.is_set():
return
building = True
+ # The runtime outlives its models: the last decode's tiling belongs to the model it ran on.
+ if hasattr(runtime, "last_vae_tiled"):
+ runtime.last_vae_tiled = False
model = runtime.build(
str(out_dir),
model_config = model_config,
@@ -1098,6 +1153,9 @@ class MlxDiffusionBackend:
)
total = time.perf_counter() - started
timing = {"total_s": round(total, 3)}
+ vae_tiled = getattr(runtime, "vae_tiled", None)
+ if callable(vae_tiled):
+ timing["vae_tiled"] = bool(vae_tiled())
if len(marks) >= 2:
# Exclude the first step: it includes prompt encoding.
timing["s_per_step"] = round((marks[-1] - marks[0]) / (len(marks) - 1), 4)
@@ -1302,7 +1360,8 @@ class MlxDiffusionBackend:
"gguf_variant": extract_quant_token(state.gguf_filename),
"cpu_offload": False,
"offload_policy": "none",
- "vae_tiling": False,
+ # Decided per generation from its size and the free working set; this is the last one's.
+ "vae_tiling": bool(getattr(self._runtime, "last_vae_tiled", False)),
"memory_mode": None,
"speed_mode": None,
"speed_optims": [],
diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.py
index 6018ee4da47..e1110466614 100644
--- a/studio/backend/tests/test_mlx_diffusion_backend.py
+++ b/studio/backend/tests/test_mlx_diffusion_backend.py
@@ -564,6 +564,105 @@ def test_policy_helpers(monkeypatch):
assert mb.compute_precision_name("Apple M1") == "bfloat16"
+_GIB = 1024**3
+
+
+ "side, free_gib, tiled",
+ [
+ # 16 GB Mac (10.67 GiB working set) holding Z-Image Q4_0 + an 8-bit text encoder (8.65 GiB): 2 GiB left.
+ (1024, 2.0, True),
+ (512, 2.0, False),
+ # 64 GB Mac: the untiled 1024 decode (about 6.6 GiB) fits, so it stays untiled, bit for bit as before.
+ (1024, 38.0, False),
+ (2048, 38.0, False),
+ (2048, 20.0, True),
+ ],
+)
+def test_vae_tiling_engages_only_when_the_untiled_decode_does_not_fit(
+ monkeypatch, side, free_gib, tiled
+):
+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)
+ assert mb.vae_tiling_needed(side, side, int(free_gib * _GIB)) is tiled
+
+
+def test_vae_tiling_env_and_unknown_budget(monkeypatch):
+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)
+ assert mb.vae_tiling_needed(4096, 4096, None) is False
+ monkeypatch.setenv(mb.VAE_TILING_ENV, "1")
+ assert mb.vae_tiling_needed(256, 256, 64 * _GIB) is True
+ monkeypatch.setenv(mb.VAE_TILING_ENV, "0")
+ assert mb.vae_tiling_needed(4096, 4096, 0) is False
+
+
+def test_untiled_decode_estimate_matches_the_metal_measurements():
+ # mflux Z-Image VAE, bf16, Apple M1 (Virtual) via Metal: 1.99 GiB at 512 and 6.64 GiB at 1024 above the weights.
+ assert abs(mb.untiled_decode_bytes(512, 512) / _GIB - 1.99) < 0.05
+ assert abs(mb.untiled_decode_bytes(1024, 1024) / _GIB - 6.64) < 0.05
+
+
+ "free_gib, implicit, expected", [(2.0, True, True), (40.0, True, False), (2.0, False, False)]
+)
+def test_the_runtime_sets_mfluxs_tiling_config_per_generation(
+ monkeypatch, free_gib, implicit, expected
+):
+ pytest.importorskip("mlx.core")
+ tiling = pytest.importorskip("mflux.models.common.vae.tiling_config")
+ pytest.importorskip("mflux.utils.exceptions")
+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)
+
+ seen = {}
+
+ class _Registry:
+ def __init__(self):
+ self.in_loop = []
+
+ def register(self, callback):
+ self.in_loop.append(callback)
+
+ class _Vae:
+ supports_implicit_tiling = implicit
+
+ class _Model:
+ callbacks = _Registry()
+ vae = _Vae()
+ tiling_config = None
+
+ def generate_image(self, **kwargs):
+ seen["tiling_config"] = self.tiling_config
+ return type("R", (), {"image": "png"})()
+
+ runtime = mb.MfluxRuntime()
+ monkeypatch.setattr(runtime, "free_working_set_bytes", lambda: int(free_gib * _GIB))
+ model = _Model()
+ image = runtime.generate(
+ model,
+ prompt = "p",
+ negative_prompt = None,
+ seed = 1,
+ steps = 1,
+ width = 1024,
+ height = 1024,
+ guidance = None,
+ on_step = lambda done: None,
+ )
+ assert image == "png"
+ assert isinstance(seen["tiling_config"], tiling.TilingConfig) is expected
+ assert runtime.vae_tiled() is expected
+
+
+def test_a_new_load_does_not_report_the_previous_models_tiled_decode(env):
+ backend, runtime, _calls = env
+ assert _load(backend)["phase"] == "ready"
+ # The previous model's last generation tiled its decode.
+ runtime.last_vae_tiled = True
+ assert backend.status()["vae_tiling"] is True
+ backend.unload()
+ assert _load(backend)["phase"] == "ready"
+ assert backend.status()["vae_tiling"] is False
+
+
def test_importing_the_engine_does_not_import_mlx():
import subprocess
import sys