Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: finite MPS memory cap and sd.cpp --mmap on macOS (#13763)
Daniel Han35c74c86aaeb
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex c1c0d2e7c1c..b18208644ad 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -641,6 +641,82 @@ def _mps_supports_bfloat16(torch: Any) -> bool:return False+MPS_HIGH_WATERMARK_ENV = "PYTORCH_MPS_HIGH_WATERMARK_RATIO"+# torch's accepted range for the high watermark ratio (MPSAllocator default_high_watermark_upper_bound).+_MPS_RATIO_MAX = 2.0+# Room left outside the cap for macOS, the window server and this process's own CPU-side memory.+_MPS_CAP_OS_RESERVE_MIN = 1536 * 1024**2+_MPS_CAP_OS_RESERVE_FRAC = 0.10+++def mps_high_watermark_ratio(+ recommended_bytes: Optional[int], total_ram_bytes: Optional[int]+) -> Optional[float]:+ """The MPS high watermark ratio (a multiple of the recommended working set) that caps Metal allocations at+ physical RAM minus an OS reserve, never below the working set Apple recommends. None when either reading is+ missing (no cap: the caller keeps the unbounded allocator).++ torch's own default is 1.7x the working set, which is above physical RAM on every Apple Silicon Mac (the working+ set is ~2/3 to 3/4 of RAM), so past RAM it swaps instead of raising; 0.0 removes the limit entirely. A finite cap+ turns that into an "MPS backend out of memory" RuntimeError that the batch backoff and VAE tile retry handle."""+ try:+ recommended = int(recommended_bytes or 0)+ total = int(total_ram_bytes or 0)+ except (TypeError, ValueError):+ return None+ if recommended <= 0 or total <= 0:+ return None+ reserve = max(_MPS_CAP_OS_RESERVE_MIN, int(total * _MPS_CAP_OS_RESERVE_FRAC))+ cap = max(recommended, total - reserve)+ return round(min(_MPS_RATIO_MAX, cap / recommended), 4)+++def _total_ram_bytes() -> Optional[int]:+ try:+ import psutil+ return int(psutil.virtual_memory().total)+ except Exception: # noqa: BLE001 -- fall back to sysconf+ pass+ try:+ return int(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES"))+ except Exception: # noqa: BLE001+ return None+++def _mps_cap_disabled() -> bool:+ if MPS_HIGH_WATERMARK_ENV in os.environ and os.environ.get(MPS_HIGH_WATERMARK_ENV) != "0.0":+ return True+ # The oversized-load opt-in asks to run past the unified-memory guard, so it keeps the unbounded allocator too.+ return os.environ.get("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", "").strip().lower() in (+ "1",+ "true",+ "yes",+ "on",+ )+++def apply_mps_memory_cap(torch: Any, logger: Any = None) -> Optional[float]:+ """Cap MPS allocations below physical RAM (``mps_high_watermark_ratio``). Returns the ratio set, or None when+ nothing was changed: a user-set PYTORCH_MPS_HIGH_WATERMARK_RATIO, the oversized-load opt-in, or torch without+ ``recommended_max_memory`` (< 2.5) / ``set_per_process_memory_fraction``. Set at runtime because the allocator+ may already exist (its env reading is one-shot), and best-effort: a failure keeps today's allocator."""+ if _mps_cap_disabled():+ return None+ try:+ mps = torch.mps+ ratio = mps_high_watermark_ratio(mps.recommended_max_memory(), _total_ram_bytes())+ if ratio is None:+ return None+ mps.set_per_process_memory_fraction(float(ratio))+ except Exception as exc: # noqa: BLE001 -- old torch / no reading: keep the unbounded allocator+ if logger is not None:+ logger.info("diffusion: MPS memory cap not set (%s)", exc)+ return None+ if logger is not None:+ logger.info("diffusion: MPS allocations capped at %.3fx the recommended working set", ratio)+ return ratio++def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:mps_available = Falsetry:@@ -654,14 +730,17 @@ def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:mps_available = Falseif mps_available:- # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, at the first MPS allocation (the probe below), so relax it- # first or the allocator caps at ~1.7x recommendedMaxWorkingSet and can OOM a model that would fit. setdefault- # respects an override.- os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.0")+ user_ratio = MPS_HIGH_WATERMARK_ENV in os.environ+ # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, when the allocator is built (the probe below, or an earlier+ # recommended_max_memory() reading). 0.0 here only lifts torch's 1.7x-of-working-set default, which sits above+ # physical RAM; apply_mps_memory_cap() then sets the finite cap. setdefault respects an override.+ os.environ.setdefault(MPS_HIGH_WATERMARK_ENV, "0.0")# Prefer bfloat16, else float32, NEVER silent float16: modern DiTs produce activations far outside fp16's# range (Z-Image MLP peaks near 9e5 -> inf -> NaN -> black image). bf16 (macOS 14+) shares fp32's exponent# range; older macOS uses fp32.dtype = torch.bfloat16 if _mps_supports_bfloat16(torch) else torch.float32+ if not user_ratio:+ apply_mps_memory_cap(torch)return DiffusionDeviceTarget(device = "mps",dtype = dtype,diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex d7a9aab6831..37f53f03b86 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1631,9 +1631,9 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)still loads, degrading to group / whole-module CPU offload and streaming from host RAM. On AppleSilicon (and integrated CUDA) the CPU and GPU share one pool, so offload moves bytes within thatpool and frees nothing: ``plan_diffusion_memory`` correctly returns ``none`` and the load then- allocates past physical memory. There is no torch OOM to catch, because ``_mps_or_cpu_target``- sets PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 to disable the MPS allocator's hard limit, so the- failure is the OS killing the process with no Python exception.+ allocates past physical memory. ``apply_mps_memory_cap`` bounds MPS allocations just below RAM,+ but that only turns the overrun into a late OOM after minutes of loading (and integrated CUDA has+ no such cap), so refusing up front is still the better failure.Weights only (``model_dense_mib`` plus the flat base overhead): the per-call runtime headroom isa coarse activation / VAE-decode estimate and the mps path already turns on VAE tiling anddiff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex 3548fda6687..86217373220 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -272,6 +272,65 @@ def metal_placement_flags(offload: Sequence[str]) -> list[str]:return [f for f in flags if f not in offload]+# 1 / 0 forces sd.cpp's --mmap on / off on macOS; unset decides from the installed build (``mac_mmap_flags``).+SD_CPP_MMAP_ENV = "UNSLOTH_DIFFUSION_SD_CPP_MMAP"+# leejet/stable-diffusion.cpp#2037 (master-907) maps mmapped weights through Metal device buffers. Older builds accept+# --mmap but hand Metal a CPU buffer it cannot address, so the flag is only safe from this build on.+METAL_MMAP_MIN_BUILD = 907+_INSTALL_RECORD = ".unsloth-sd-cpp-install.json"+++def sd_cpp_build_number(tag: Optional[str]) -> Optional[int]:+ """``N`` of a ``master-N-<sha>[-u<sha>]`` release tag, else None."""+ import re++ m = re.match(r"^master-(\d+)(?:-|$)", str(tag or "").strip())+ return int(m.group(1)) if m else None++[email protected]_cache(maxsize = 8)+def _installed_sd_cpp_tag(binary: str) -> Optional[str]:+ """The release tag Studio's installer recorded next to ``binary`` (the binary sits at or a few levels under the+ install root). None for a binary the installer did not place (``SD_CLI_PATH``, PATH, a source build)."""+ import json+ from pathlib import Path++ try:+ path = Path(binary).resolve()+ except Exception: # noqa: BLE001+ return None+ for parent in list(path.parents)[:3]:+ try:+ with open(parent / _INSTALL_RECORD, "r", encoding = "utf-8") as f:+ rec = json.load(f)+ except (OSError, ValueError):+ continue+ tag = rec.get("tag") if isinstance(rec, dict) else None+ return tag if isinstance(tag, str) and tag else None+ return None+++def mac_mmap_flags(binary: Optional[str], extra_args: Optional[Sequence[str]] = None) -> list[str]:+ """``["--mmap"]`` on macOS when ``binary`` maps weights correctly on Metal, else nothing.++ With it, GGUF / safetensors weights stay clean file-backed pages (Metal wraps them in place, the CPU backend reads+ them in place) instead of a second anonymous copy, which on unified memory is the copy that swaps: upstream measured+ a Qwen-Image 2.1 Q4_K_M peak footprint of 10.2 -> 1.4 GB on an M1 Pro, pixel-identical. Fails closed on a build it+ cannot date; ``UNSLOTH_DIFFUSION_SD_CPP_MMAP`` = 1 / 0 forces it either way."""+ import os+ import sys++ if sys.platform != "darwin" or "--mmap" in (extra_args or ()):+ return []+ forced = os.environ.get(SD_CPP_MMAP_ENV, "").strip().lower()+ if forced in ("0", "false", "no", "off"):+ return []+ if forced in ("1", "true", "yes", "on"):+ return ["--mmap"]+ build = sd_cpp_build_number(_installed_sd_cpp_tag(str(binary))) if binary else None+ return ["--mmap"] if build is not None and build >= METAL_MMAP_MIN_BUILD else []++# Everything on the CPU backend. sd.cpp prefers GPU -> integrated GPU -> CPU and only `--backend` changes which# backend EXECUTES the graph (`--offload-to-cpu` moves parameters, not compute), so this is the one flag that removes# ggml-metal entirely.@@ -483,6 +542,7 @@ def build_sd_cpp_command(if offload:cmd += offloadcmd += metal_placement_flags(offload)+ cmd += mac_mmap_flags(binary, extra_args)if verbose:cmd += ["-v"]if extra_args:@@ -575,6 +635,7 @@ def build_sd_cpp_video_command(if offload:cmd += offloadcmd += metal_placement_flags(offload)+ cmd += mac_mmap_flags(binary, extra_args)if verbose:cmd.append("-v")if extra_args:@@ -686,6 +747,7 @@ def build_sd_cpp_server_command(backend_args = [*offload, *placement, *(extra_args or ())]cmd += [f for f in native_speed_flags(native_speed, backend_args) if f not in offload]cmd += placement+ cmd += mac_mmap_flags(binary, extra_args)if verbose:cmd += ["-v"]if extra_args:diff --git a/studio/backend/tests/test_diffusion_device.py b/studio/backend/tests/test_diffusion_device.pyindex 4af4b1e574b..d10450a615d 100644--- a/studio/backend/tests/test_diffusion_device.py+++ b/studio/backend/tests/test_diffusion_device.py@@ -1021,3 +1021,95 @@ def test_a_multi_card_pick_declines_to_rank_when_ranking_is_barred(monkeypatch):assert probed == []with pytest.raises(ValueError):dd.resolve_selected_cuda_ordinal([9], allow_ranking = False)+++# ── MPS allocation cap ────────────────────────────────────────────────++GIB = 1024**3+++ "recommended_gib, ram_gib, expected_cap_gib",+ [+ (5.33, 8, 6.5), # 8 GB Mac: RAM - 1.5 GiB floor reserve+ (10.67, 16, 14.4), # RAM - 10%+ (96, 128, 115.2),+ (5.0, 5.5, 5.0), # reserve would go below the working set: never cap under it+ ],+)+def test_mps_ratio_caps_below_ram_never_below_working_set(+ recommended_gib, ram_gib, expected_cap_gib+):+ ratio = dd.mps_high_watermark_ratio(int(recommended_gib * GIB), int(ram_gib * GIB))+ cap_gib = ratio * recommended_gib+ assert cap_gib == pytest.approx(expected_cap_gib, rel = 1e-3)+ assert cap_gib < ram_gib+ assert 1.0 <= ratio <= 2.0+ assert cap_gib < 1.7 * recommended_gib # tighter than torch's own default, which is above RAM+++def test_mps_ratio_unknown_readings_mean_no_cap():+ assert dd.mps_high_watermark_ratio(None, 16 * GIB) is None+ assert dd.mps_high_watermark_ratio(10 * GIB, 0) is None+ assert dd.mps_high_watermark_ratio("x", 16 * GIB) is None+++def _mps_cap_torch(+ recorded,+ *,+ recommended = 10 * GIB,+ fraction_raises = False,+):+ torch = _make_torch(mps_available = True, mps_probe = "pass")++ def _set(fraction):+ if fraction_raises:+ raise AttributeError("old torch")+ recorded.append(fraction)++ torch.mps = types.SimpleNamespace(+ recommended_max_memory = lambda: recommended, set_per_process_memory_fraction = _set+ )+ return torch+++def test_mps_target_sets_finite_cap(monkeypatch):+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)+ recorded = []+ _install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")+ assert dd.resolve_diffusion_device_target().device == "mps"+ assert recorded == [pytest.approx(14.4 / 10, rel = 1e-3)]+++ "env",+ [+ {dd.MPS_HIGH_WATERMARK_ENV: "0.0"}, # explicit "no cap" stays no cap+ {dd.MPS_HIGH_WATERMARK_ENV: "1.2"},+ {"UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD": "1"},+ ],+)+def test_mps_cap_respects_user_override(monkeypatch, env):+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)+ for k, v in env.items():+ monkeypatch.setenv(k, v)+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)+ recorded = []+ _install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")+ assert dd.resolve_diffusion_device_target().device == "mps"+ assert recorded == []+ if dd.MPS_HIGH_WATERMARK_ENV in env:+ import os+ assert os.environ[dd.MPS_HIGH_WATERMARK_ENV] == env[dd.MPS_HIGH_WATERMARK_ENV]+++def test_mps_cap_failure_keeps_target(monkeypatch):+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)+ _install(monkeypatch, _mps_cap_torch([], fraction_raises = True), studio_device = "mlx")+ t = dd.resolve_diffusion_device_target()+ assert t.device == "mps" and t.dtype == BF16diff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.pyindex 9efcabb01a1..45d2678b0fa 100644--- a/studio/backend/tests/test_sd_cpp_args.py+++ b/studio/backend/tests/test_sd_cpp_args.py@@ -1037,3 +1037,85 @@ def test_device_backend_flags_are_appended_not_folded_into_the_policy():before = list(policy)device_backend_flags("CUDA1", policy)assert policy == before+++# ── Metal mmap (leejet/stable-diffusion.cpp#2037) ───────────────────────────+++def _managed_install(tmp_path, tag):+ import json++ root = tmp_path / "stable-diffusion.cpp"+ (root / "bin").mkdir(parents = True)+ binary = root / "bin" / "sd-cli"+ binary.write_text("")+ (root / ".unsloth-sd-cpp-install.json").write_text(+ json.dumps({"tag": tag, "accelerator": "cpu"})+ )+ _sd_cpp_args_module._installed_sd_cpp_tag.cache_clear()+ return str(binary)+++def _all_builders(binary, extra_args = None):+ files = SdCppModelFiles(+ diffusion_model = "/m/model.gguf", vae = "/m/vae.safetensors", llm = "/m/te.gguf"+ )+ yield build_sd_cpp_command(+ binary, files, SdCppGenParams(prompt = "a cat"), output_path = "/o.png", extra_args = extra_args+ )+ yield build_sd_cpp_video_command(+ binary,+ files,+ SdCppVideoGenParams(prompt = "a cat", width = 256, height = 256, num_frames = 5),+ output_path = "/o.webm",+ extra_args = extra_args,+ )+ yield build_sd_cpp_server_command(+ binary, files, host = "127.0.0.1", port = 1, extra_args = extra_args+ )+++ "tag, expected",+ [+ ("master-951-f89d9b1-uafac7a7", True), # the pinned bundle+ ("master-907-500ef5f", True), # the build that merged #2037+ ("master-906-abcdef0", False), # --mmap there hands Metal an unaddressable CPU buffer+ ("master-813-bfbef5b-u6321a69", False),+ ("b1234", False), # undatable: fail closed+ ],+)+def test_mmap_rides_every_macos_builder_from_the_metal_mmap_build(+ monkeypatch, tmp_path, tag, expected+):+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", raising = False)+ monkeypatch.setattr("sys.platform", "darwin")+ binary = _managed_install(tmp_path, tag)+ for cmd in _all_builders(binary):+ assert ("--mmap" in cmd) is expected+ assert cmd.count("--mmap") <= 1+++def test_mmap_is_macos_only_and_never_on_an_unrecorded_binary(monkeypatch, tmp_path):+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", raising = False)+ binary = _managed_install(tmp_path, "master-951-f89d9b1-uafac7a7")+ for other in ("linux", "win32"):+ monkeypatch.setattr("sys.platform", other)+ assert all("--mmap" not in cmd for cmd in _all_builders(binary))+ monkeypatch.setattr("sys.platform", "darwin")+ stray = tmp_path / "elsewhere" / "sd-cli"+ stray.parent.mkdir()+ stray.write_text("")+ assert all("--mmap" not in cmd for cmd in _all_builders(str(stray)))+++def test_mmap_env_override_and_no_duplicate(monkeypatch, tmp_path):+ monkeypatch.setattr("sys.platform", "darwin")+ new = _managed_install(tmp_path, "master-951-f89d9b1-uafac7a7")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", "0")+ assert all("--mmap" not in cmd for cmd in _all_builders(new))+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", "1")+ assert all("--mmap" in cmd for cmd in _all_builders("/usr/local/bin/sd-cli"))+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP")+ for cmd in _all_builders(new, extra_args = ["--mmap"]):+ assert cmd.count("--mmap") == 1