Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio (macOS): size image loads against the Metal working set, not RAM minus 20% (#13776)
oobabooga4b0e815bbf04
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 74e3079362e..ece7a4317e8 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -52,6 +52,8 @@ DEFAULT_IMAGE_WIDTH = 1024DEFAULT_IMAGE_HEIGHT = 1024# flat allowance for fixed pipeline costs (scheduler, embeddings, CUDA context, fragmentation)DEFAULT_BASE_OVERHEAD_MIB = 2048+# Reserve kind for an mps reading bounded by Metal's recommended working set (see snapshot_device_memory).+METAL_WORKING_SET = "metal_working_set"_host_memory_reclaim_warning_logged = False@@ -575,6 +577,8 @@ class DeviceMemory:memory_kind: str # "discrete_vram" | "unified_memory" | "system_memory" | "unknown"free_mib: Optional[int] = Nonetotal_mib: Optional[int] = None+ # Metal's recommended working set when it bounds free / total (mps); None everywhere else.+ metal_working_set_mib: Optional[int] = None@propertydef is_unified(self) -> bool:@@ -644,14 +648,42 @@ def snapshot_device_memory(target: Any) -> DeviceMemory:free, total = _xpu_memory()return DeviceMemory(backend, device, "discrete_vram", free, total)if device == "mps":- # Apple Silicon shares one CPU/GPU pool: system memory is the budget, offload pointless+ # One CPU/GPU pool, so offload is pointless. The device is Metal's recommended working set (what the GPU holds+ # with good performance; allocations past it succeed and page), bounded by the RAM available right now.total, free = _system_memory_mib()- return DeviceMemory(backend, device, "unified_memory", free, total)+ ram = DeviceMemory(backend, device, "unified_memory", free, total)+ working_set, in_use = _metal_working_set_mib()+ if working_set is None:+ return ram+ ws_free = None if free is None else min(int(free), max(0, working_set - in_use))+ ws_total = working_set if total is None else min(int(total), working_set)+ metal = DeviceMemory(+ backend, device, "unified_memory", ws_free, ws_total, metal_working_set_mib = working_set+ )+ # The working set only widens the RAM reading: on an idle Mac with more RAM available than the working set,+ # 15% of it can leave less than RAM minus 20%, and a load that fit under the RAM reading must keep fitting.+ if free is not None and total is not None:+ ram_budget = int(free) - _reserve_mib("unified_memory", int(total))+ if ram_budget > int(ws_free) - _reserve_mib(METAL_WORKING_SET, ws_total):+ return ram+ return metaltotal, free = _system_memory_mib()return DeviceMemory(backend, device, "system_memory", free, total)+def _metal_working_set_mib() -> tuple[Optional[int], int]:+ """(Metal's recommended working set, what this process already holds in Metal) in MiB, or (None, 0) when torch+ cannot report it (``recommended_max_memory`` arrived in torch 2.5)."""+ try:+ import torch+ working_set = int(torch.mps.recommended_max_memory()) // (1024 * 1024)+ in_use = int(torch.mps.driver_allocated_memory()) // (1024 * 1024)+ except Exception: # noqa: BLE001 - no reading: size against system RAM as before+ return None, 0+ return (working_set, in_use) if working_set > 0 else (None, 0)++def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,without flushing it.@@ -1104,6 +1136,10 @@ def calibrated_image_activation(def _reserve_mib(memory_kind: str, base: int) -> int:+ if memory_kind == METAL_WORKING_SET:+ # Only other apps' growth during the load: the working set already sits below RAM. At least as strict as+ # llama.cpp (_APPLE_UNIFIED_MEMORY_FRACTION), which keeps 15% of the lower of available RAM and working set.+ return int(base * 0.15)if memory_kind == "unified_memory":return max(2048, int(base * 0.20)) # OS + CPU share this poolif memory_kind == "system_memory":@@ -1132,6 +1168,8 @@ def _rocm_linux_apu_os_room_outside_pool(memory: DeviceMemory) -> bool:def _budget_reserve_kind(memory: DeviceMemory) -> str:+ if getattr(memory, "metal_working_set_mib", None) is not None:+ return METAL_WORKING_SETreturn "discrete_vram" if _rocm_linux_apu_os_room_outside_pool(memory) else memory.memory_kind@@ -1664,6 +1702,7 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)budget = estimates.get("safe_device_budget_mib")weights = estimates.get("model_dense_mib")overhead = estimates.get("base_overhead_mib")+ companions = estimates.get("companion_dense_mib")free = getattr(memory, "free_mib", None)except Exception: # noqa: BLE001 - malformed plan: never block the loadreturn None@@ -1673,19 +1712,39 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)if required <= int(budget):return Nonewhat = f"'{family}'" if family else "This model"- free_note = (- f"of the {int(free) / 1024:.0f} GB currently free, after reserving room for the "- "operating system"- if free is not None- else "after reserving room for the operating system"- )+ if free is None:+ free_note = "after reserving room for the operating system"+ elif getattr(memory, "metal_working_set_mib", None) is not None:+ free_note = (+ f"of the {int(free) / 1024:.0f} GB the GPU can use right now, after reserving room "+ "for other applications"+ )+ else:+ free_note = (+ f"of the {int(free) / 1024:.0f} GB currently free, after reserving room for the "+ "operating system"+ )+ # The text encoders and VAE load dense whatever quant the main model is, so past this floor no quant of it fits.+ floor = int(companions or 0) + int(overhead or 0)+ if companions and floor > int(budget):+ advice = (+ f"Its text encoders and VAE alone need about {floor / 1024:.0f} GB, so a lower quant of "+ "the main model will not fit either. Use a smaller model, or free memory by closing "+ "other applications."+ )+ else:+ advice = (+ "Use a smaller or more quantized model (for example a lower GGUF quant), or free memory "+ "by closing other applications."+ )+ # One decimal when whole GB would print the same number twice ("needs about 14 GB, only about 14 GB usable").+ digits = 1 if f"{required / 1024:.0f}" == f"{int(budget) / 1024:.0f}" else 0return (- f"{what} needs about {required / 1024:.0f} GB of memory for its weights, but only "- f"about {int(budget) / 1024:.0f} GB is usable on this device ({free_note}). This device "+ f"{what} needs about {required / 1024:.{digits}f} GB of memory for its weights, but only "+ f"about {int(budget) / 1024:.{digits}f} GB is usable on this device ({free_note}). This device ""has unified memory, so the CPU and GPU share one pool: offloading weights to the CPU ""frees nothing, and the operating system stops an oversized load outright instead of "- "reporting an out-of-memory error. Use a smaller or more quantized model (for example a "- "lower GGUF quant), or free memory by closing other applications. (Server "+ f"reporting an out-of-memory error. {advice} (Server "f"installs that know this estimate is wrong can set {UNIFIED_OVERSIZE_ENV}=1 to attempt ""the load anyway.)")diff --git a/studio/backend/tests/test_diffusion_mps_metal_budget.py b/studio/backend/tests/test_diffusion_mps_metal_budget.pynew file mode 100644index 00000000000..977a283604f--- /dev/null+++ b/studio/backend/tests/test_diffusion_mps_metal_budget.py@@ -0,0 +1,186 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Apple Silicon: the unified-memory refusal sizes against Metal's working set, not RAM minus a 20% reserve.++The machine is a 24 GiB MacBook Air with every app closed: 16.8 GiB available and a 16 GiB Metal working set (two+thirds of RAM). Weights are the Hub sizes of the unsloth GGUFs and of the bf16 Qwen3-4B text encoder and VAE every+Z-Image and FLUX.2 Klein 4B load adds. Under the RAM reading Z-Image Q4_K_M was refused with "needs about 15 GB, only+about 12 GB usable", the numbers in the report.+"""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_memory as dm+from core.inference.diffusion_memory import (+ plan_diffusion_memory,+ snapshot_device_memory,+ unified_memory_shortfall_message,+)++RAM_MIB = 24 * 1024+AVAILABLE_MIB = 17_203 # 16.8 GiB+WORKING_SET_MIB = 16 * 1024++TEXT_ENCODER_MIB = 7_672 # Qwen3-4B, bf16, shared by both families+ZIMAGE_VAE_MIB = 159+KLEIN_VAE_MIB = 160+GGUF_MIB = {+ "z-image Q2_K": 3_471,+ "z-image Q3_K_S": 3_768,+ "z-image Q4_K_M": 4_785,+ "klein Q4_K_M": 2_483,+ "klein Q6_K": 3_251,+ "klein Q8_0": 4_101,+}++MPS = types.SimpleNamespace(device = "mps", backend = "mps", supports_model_cpu_offload = False)+++def mac(monkeypatch):+ state = {"available": AVAILABLE_MIB, "working_set": (WORKING_SET_MIB, 0)}+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (RAM_MIB, state["available"]))+ monkeypatch.setattr(dm, "_metal_working_set_mib", lambda: state["working_set"])+ return state+++def _verdict(name):+ vae = ZIMAGE_VAE_MIB if name.startswith("z-image") else KLEIN_VAE_MIB+ plan = plan_diffusion_memory(+ target = MPS,+ device_memory = snapshot_device_memory(MPS),+ model_dense_mib = dm.estimate_gguf_resident_mib(GGUF_MIB[name]) + TEXT_ENCODER_MIB + vae,+ companion_dense_mib = TEXT_ENCODER_MIB + vae,+ text_encoder_dense_mib = TEXT_ENCODER_MIB,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None),+ )+ return plan, unified_memory_shortfall_message(plan, family = name.split()[0])+++def test_the_working_set_bounds_the_mps_reading(mac):+ memory = snapshot_device_memory(MPS)+ assert (memory.free_mib, memory.total_mib) == (WORKING_SET_MIB, WORKING_SET_MIB)+ assert memory.metal_working_set_mib == WORKING_SET_MIB+ # 15% of the working set held back: 13.6 GiB, not 12.+ assert dm._safe_device_budget_mib(memory) == WORKING_SET_MIB - int(WORKING_SET_MIB * 0.15)+++def test_available_ram_and_this_process_metal_still_bound_it(mac):+ mac["available"] = 10_000+ assert snapshot_device_memory(MPS).free_mib == 10_000+ mac["available"] = AVAILABLE_MIB+ mac["working_set"] = (WORKING_SET_MIB, 1_024)+ assert snapshot_device_memory(MPS).free_mib == WORKING_SET_MIB - 1_024+++def test_the_working_set_never_budgets_below_the_ram_reading(mac):+ # 2 GiB already in Metal: 14,336 - 2,457 = 11,879 would undercut RAM minus 20% (12,288), so RAM stands.+ mac["working_set"] = (WORKING_SET_MIB, 2_048)+ memory = snapshot_device_memory(MPS)+ assert (memory.free_mib, memory.metal_working_set_mib) == (AVAILABLE_MIB, None)+ assert dm._safe_device_budget_mib(memory) == AVAILABLE_MIB - int(RAM_MIB * 0.20)+++# Idle Macs with more RAM available than the working set (2/3 of RAM up to 32 GB, 3/4 above).+IDLE_MACS = [+ (24, 19_456, 16_384),+ (32, 27_648, 21_845),+ (64, 57_671, 49_152),+ (128, 115_343, 98_304),+ (192, 173_015, 147_456),+]++[email protected]("ram_gb, available, working_set", IDLE_MACS)+def test_an_idle_mac_keeps_every_load_the_ram_reading_admitted(+ mac, monkeypatch, ram_gb, available, working_set+):+ ram = ram_gb * 1024+ monkeypatch.setattr(dm, "_system_memory_mib", lambda: (ram, available))+ mac["working_set"] = (working_set, 0)+ ram_budget = available - int(ram * 0.20)+ assert dm._safe_device_budget_mib(snapshot_device_memory(MPS)) == ram_budget+ # A model needing exactly the RAM budget: admitted before the working set reading existed, so still admitted.+ plan = plan_diffusion_memory(+ target = MPS,+ device_memory = snapshot_device_memory(MPS),+ model_dense_mib = ram_budget - dm.DEFAULT_BASE_OVERHEAD_MIB,+ runtime_headroom_mib = dm.estimate_image_runtime_mib(width = None, height = None),+ )+ assert unified_memory_shortfall_message(plan) is None+++def test_without_a_working_set_reading_the_ram_reading_stands(mac):+ mac["working_set"] = (None, 0)+ memory = snapshot_device_memory(MPS)+ assert (memory.free_mib, memory.total_mib, memory.metal_working_set_mib) == (+ AVAILABLE_MIB,+ RAM_MIB,+ None,+ )+ assert dm._safe_device_budget_mib(memory) == AVAILABLE_MIB - int(RAM_MIB * 0.20)+++def test_the_ram_reading_reproduces_the_report(mac):+ mac["working_set"] = (None, 0)+ _, message = _verdict("z-image Q4_K_M")+ assert "needs about 15 GB" in message+ assert "only about 12 GB is usable" in message++[email protected]("name", ["z-image Q2_K", "z-image Q3_K_S", "klein Q4_K_M", "klein Q6_K"])+def test_models_that_fit_the_working_set_now_load(mac, name):+ mac["working_set"] = (None, 0)+ assert _verdict(name)[1] is not None # refused under the RAM reading+ mac["working_set"] = (WORKING_SET_MIB, 0)+ assert _verdict(name)[1] is None++[email protected]("name", ["z-image Q4_K_M", "klein Q8_0"])+def test_weights_past_the_working_set_budget_are_still_refused(mac, name):+ plan, message = _verdict(name)+ assert message is not None+ assert "the GPU can use right now" in message+ # A lower quant does fit here (z-image Q2_K above), so the quant advice is the right one.+ assert "lower GGUF quant" in message+++def test_close_figures_get_a_decimal_instead_of_reading_as_equal(mac):+ # 14,186 vs 13,927 MiB would both print as "about 14 GB".+ assert "needs about 13.9 GB" in _verdict("klein Q8_0")[1]+ assert "only about 13.6 GB is usable" in _verdict("klein Q8_0")[1]+ assert "needs about 15 GB" in _verdict("z-image Q4_K_M")[1]+ assert "only about 14 GB is usable" in _verdict("z-image Q4_K_M")[1]+++def test_the_encoder_floor_replaces_the_quant_advice_when_no_quant_can_fit(mac):+ # A busy Mac: 9,000 - 2,457 leaves less than the 7.8 GiB of companions + overhead.+ mac["available"] = 9_000+ _, message = _verdict("z-image Q2_K")+ assert "text encoders and VAE alone need about 10 GB" in message+ assert "lower GGUF quant" not in message+++def test_the_working_set_reading_comes_from_torch_mps(monkeypatch):+ mps = types.SimpleNamespace(+ recommended_max_memory = lambda: WORKING_SET_MIB * 1024 * 1024,+ driver_allocated_memory = lambda: 512 * 1024 * 1024,+ )+ monkeypatch.setitem(dm.sys.modules, "torch", types.SimpleNamespace(mps = mps))+ assert dm._metal_working_set_mib() == (WORKING_SET_MIB, 512)++ def unsupported():+ raise AttributeError("recommended_max_memory")++ mps.recommended_max_memory = unsupported+ assert dm._metal_working_set_mib() == (None, 0)+++def test_other_unified_devices_keep_their_reserve():+ integrated = dm.DeviceMemory("cuda", "cuda", "unified_memory", AVAILABLE_MIB, RAM_MIB)+ assert dm._safe_device_budget_mib(integrated) == AVAILABLE_MIB - int(RAM_MIB * 0.20)