Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MLX images: tile the VAE decode when the untiled one would not fit (#14796)
Daniel Han9a822f490bf9
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.pyindex 0812e11aac2..f2fcd582bfe 100644--- a/studio/backend/core/inference/mlx_diffusion_backend.py+++ b/studio/backend/core/inference/mlx_diffusion_backend.py@@ -158,6 +158,32 @@ def compute_precision_name(chip: Optional[str] = None) -> str:return "float16" if ("apple m1" in name or "apple m2" in name) else "bfloat16"+# Untiled VAE decode working set above the weights (mflux Z-Image VAE, bf16, Apple M1 Metal): 1.99 GiB at 512x512 and+# 6.64 GiB at 1024x1024, i.e. about 0.44 GiB plus 6.35 KB per output pixel. Tiled (mflux's 512 px tiles) it stays at+# 2.0 GiB at 1024x1024; a 512x512 image is a single tile, so tiling changes nothing there.+_VAE_DECODE_FIXED_BYTES = int(0.44 * 1024**3)+_VAE_DECODE_BYTES_PER_PIXEL = 6350+VAE_TILING_ENV = "UNSLOTH_DIFFUSION_MLX_VAE_TILING"+++def untiled_decode_bytes(width: int, height: int) -> int:+ return _VAE_DECODE_FIXED_BYTES + _VAE_DECODE_BYTES_PER_PIXEL * int(width) * int(height)+++def vae_tiling_needed(width: int, height: int, free_bytes: Optional[int]) -> bool:+ """Tile the decode only when the untiled one would not fit what the working set has left, as ComfyUI tiles only+ after an out-of-memory decode; unified memory pages instead of failing, so the check comes first.+ ``UNSLOTH_DIFFUSION_MLX_VAE_TILING=1`` / ``0`` forces it on / off."""+ raw = os.environ.get(VAE_TILING_ENV, "").strip().lower()+ if raw in ("1", "on", "true", "yes"):+ return True+ if raw in ("0", "off", "false", "no"):+ return False+ if free_bytes is None:+ return False+ return untiled_decode_bytes(width, height) > free_bytes++class MfluxRuntime:"""The only code that imports mlx / mflux. Tests replace it with a fake (``backend.runtime_factory``)."""@@ -226,6 +252,7 @@ class MfluxRuntime:except MlxCancelled as exc:raise StopImageGenerationException(str(exc)) from exc+ self._plan_vae_tiling(model, width, height)callback = _StepCallback()model.callbacks.register(callback)cancelled = False@@ -256,6 +283,31 @@ class MfluxRuntime:raise MlxCancelled("cancelled")return result.image+ def free_working_set_bytes(self) -> Optional[int]:+ """Metal's recommended working set minus what MLX holds live now (the resident model), or None."""+ import mlx.core as mx++ try:+ info = mx.device_info() if hasattr(mx, "device_info") else mx.metal.device_info()+ budget = int(info.get("max_recommended_working_set_size") or 0)+ except Exception: # noqa: BLE001 -- no Metal reading: leave the decode untiled+ return None+ if budget <= 0:+ return None+ return budget - int(mx.get_active_memory())++ def _plan_vae_tiling(self, model: Any, width: int, height: int) -> None:+ from mflux.models.common.vae.tiling_config import TilingConfig++ tiled = TilingConfig.may_tile_implicitly(model) and vae_tiling_needed(+ width, height, self.free_working_set_bytes()+ )+ model.tiling_config = TilingConfig() if tiled else None+ self.last_vae_tiled = tiled++ def vae_tiled(self) -> bool:+ return bool(getattr(self, "last_vae_tiled", False))+def cache_text_encodings(self,model: Any,@@ -836,6 +888,9 @@ class MlxDiffusionBackend:if _cancel_event.is_set():returnbuilding = True+ # The runtime outlives its models: the last decode's tiling belongs to the model it ran on.+ if hasattr(runtime, "last_vae_tiled"):+ runtime.last_vae_tiled = Falsemodel = runtime.build(str(out_dir),model_config = model_config,@@ -1098,6 +1153,9 @@ class MlxDiffusionBackend:)total = time.perf_counter() - startedtiming = {"total_s": round(total, 3)}+ vae_tiled = getattr(runtime, "vae_tiled", None)+ if callable(vae_tiled):+ timing["vae_tiled"] = bool(vae_tiled())if len(marks) >= 2:# Exclude the first step: it includes prompt encoding.timing["s_per_step"] = round((marks[-1] - marks[0]) / (len(marks) - 1), 4)@@ -1302,7 +1360,8 @@ class MlxDiffusionBackend:"gguf_variant": extract_quant_token(state.gguf_filename),"cpu_offload": False,"offload_policy": "none",- "vae_tiling": False,+ # Decided per generation from its size and the free working set; this is the last one's.+ "vae_tiling": bool(getattr(self._runtime, "last_vae_tiled", False)),"memory_mode": None,"speed_mode": None,"speed_optims": [],diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.pyindex 6018ee4da47..e1110466614 100644--- a/studio/backend/tests/test_mlx_diffusion_backend.py+++ b/studio/backend/tests/test_mlx_diffusion_backend.py@@ -564,6 +564,105 @@ def test_policy_helpers(monkeypatch):assert mb.compute_precision_name("Apple M1") == "bfloat16"+_GIB = 1024**3+++ "side, free_gib, tiled",+ [+ # 16 GB Mac (10.67 GiB working set) holding Z-Image Q4_0 + an 8-bit text encoder (8.65 GiB): 2 GiB left.+ (1024, 2.0, True),+ (512, 2.0, False),+ # 64 GB Mac: the untiled 1024 decode (about 6.6 GiB) fits, so it stays untiled, bit for bit as before.+ (1024, 38.0, False),+ (2048, 38.0, False),+ (2048, 20.0, True),+ ],+)+def test_vae_tiling_engages_only_when_the_untiled_decode_does_not_fit(+ monkeypatch, side, free_gib, tiled+):+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)+ assert mb.vae_tiling_needed(side, side, int(free_gib * _GIB)) is tiled+++def test_vae_tiling_env_and_unknown_budget(monkeypatch):+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)+ assert mb.vae_tiling_needed(4096, 4096, None) is False+ monkeypatch.setenv(mb.VAE_TILING_ENV, "1")+ assert mb.vae_tiling_needed(256, 256, 64 * _GIB) is True+ monkeypatch.setenv(mb.VAE_TILING_ENV, "0")+ assert mb.vae_tiling_needed(4096, 4096, 0) is False+++def test_untiled_decode_estimate_matches_the_metal_measurements():+ # mflux Z-Image VAE, bf16, Apple M1 (Virtual) via Metal: 1.99 GiB at 512 and 6.64 GiB at 1024 above the weights.+ assert abs(mb.untiled_decode_bytes(512, 512) / _GIB - 1.99) < 0.05+ assert abs(mb.untiled_decode_bytes(1024, 1024) / _GIB - 6.64) < 0.05+++ "free_gib, implicit, expected", [(2.0, True, True), (40.0, True, False), (2.0, False, False)]+)+def test_the_runtime_sets_mfluxs_tiling_config_per_generation(+ monkeypatch, free_gib, implicit, expected+):+ pytest.importorskip("mlx.core")+ tiling = pytest.importorskip("mflux.models.common.vae.tiling_config")+ pytest.importorskip("mflux.utils.exceptions")+ monkeypatch.delenv(mb.VAE_TILING_ENV, raising = False)++ seen = {}++ class _Registry:+ def __init__(self):+ self.in_loop = []++ def register(self, callback):+ self.in_loop.append(callback)++ class _Vae:+ supports_implicit_tiling = implicit++ class _Model:+ callbacks = _Registry()+ vae = _Vae()+ tiling_config = None++ def generate_image(self, **kwargs):+ seen["tiling_config"] = self.tiling_config+ return type("R", (), {"image": "png"})()++ runtime = mb.MfluxRuntime()+ monkeypatch.setattr(runtime, "free_working_set_bytes", lambda: int(free_gib * _GIB))+ model = _Model()+ image = runtime.generate(+ model,+ prompt = "p",+ negative_prompt = None,+ seed = 1,+ steps = 1,+ width = 1024,+ height = 1024,+ guidance = None,+ on_step = lambda done: None,+ )+ assert image == "png"+ assert isinstance(seen["tiling_config"], tiling.TilingConfig) is expected+ assert runtime.vae_tiled() is expected+++def test_a_new_load_does_not_report_the_previous_models_tiled_decode(env):+ backend, runtime, _calls = env+ assert _load(backend)["phase"] == "ready"+ # The previous model's last generation tiled its decode.+ runtime.last_vae_tiled = True+ assert backend.status()["vae_tiling"] is True+ backend.unload()+ assert _load(backend)["phase"] == "ready"+ assert backend.status()["vae_tiling"] is False++def test_importing_the_engine_does_not_import_mlx():import subprocessimport sys