Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: finite MPS memory cap and sd.cpp --mmap on macOS (#13763)

Daniel Han35c74c86aaeb
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index c1c0d2e7c1c..b18208644ad 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -641,6 +641,82 @@ def _mps_supports_bfloat16(torch: Any) -> bool:
return False
+MPS_HIGH_WATERMARK_ENV = "PYTORCH_MPS_HIGH_WATERMARK_RATIO"
+# torch's accepted range for the high watermark ratio (MPSAllocator default_high_watermark_upper_bound).
+_MPS_RATIO_MAX = 2.0
+# Room left outside the cap for macOS, the window server and this process's own CPU-side memory.
+_MPS_CAP_OS_RESERVE_MIN = 1536 * 1024**2
+_MPS_CAP_OS_RESERVE_FRAC = 0.10
+
+
+def mps_high_watermark_ratio(
+ recommended_bytes: Optional[int], total_ram_bytes: Optional[int]
+) -> Optional[float]:
+ """The MPS high watermark ratio (a multiple of the recommended working set) that caps Metal allocations at
+ physical RAM minus an OS reserve, never below the working set Apple recommends. None when either reading is
+ missing (no cap: the caller keeps the unbounded allocator).
+
+ torch's own default is 1.7x the working set, which is above physical RAM on every Apple Silicon Mac (the working
+ set is ~2/3 to 3/4 of RAM), so past RAM it swaps instead of raising; 0.0 removes the limit entirely. A finite cap
+ turns that into an "MPS backend out of memory" RuntimeError that the batch backoff and VAE tile retry handle."""
+ try:
+ recommended = int(recommended_bytes or 0)
+ total = int(total_ram_bytes or 0)
+ except (TypeError, ValueError):
+ return None
+ if recommended <= 0 or total <= 0:
+ return None
+ reserve = max(_MPS_CAP_OS_RESERVE_MIN, int(total * _MPS_CAP_OS_RESERVE_FRAC))
+ cap = max(recommended, total - reserve)
+ return round(min(_MPS_RATIO_MAX, cap / recommended), 4)
+
+
+def _total_ram_bytes() -> Optional[int]:
+ try:
+ import psutil
+ return int(psutil.virtual_memory().total)
+ except Exception: # noqa: BLE001 -- fall back to sysconf
+ pass
+ try:
+ return int(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES"))
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _mps_cap_disabled() -> bool:
+ if MPS_HIGH_WATERMARK_ENV in os.environ and os.environ.get(MPS_HIGH_WATERMARK_ENV) != "0.0":
+ return True
+ # The oversized-load opt-in asks to run past the unified-memory guard, so it keeps the unbounded allocator too.
+ return os.environ.get("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
+
+
+def apply_mps_memory_cap(torch: Any, logger: Any = None) -> Optional[float]:
+ """Cap MPS allocations below physical RAM (``mps_high_watermark_ratio``). Returns the ratio set, or None when
+ nothing was changed: a user-set PYTORCH_MPS_HIGH_WATERMARK_RATIO, the oversized-load opt-in, or torch without
+ ``recommended_max_memory`` (< 2.5) / ``set_per_process_memory_fraction``. Set at runtime because the allocator
+ may already exist (its env reading is one-shot), and best-effort: a failure keeps today's allocator."""
+ if _mps_cap_disabled():
+ return None
+ try:
+ mps = torch.mps
+ ratio = mps_high_watermark_ratio(mps.recommended_max_memory(), _total_ram_bytes())
+ if ratio is None:
+ return None
+ mps.set_per_process_memory_fraction(float(ratio))
+ except Exception as exc: # noqa: BLE001 -- old torch / no reading: keep the unbounded allocator
+ if logger is not None:
+ logger.info("diffusion: MPS memory cap not set (%s)", exc)
+ return None
+ if logger is not None:
+ logger.info("diffusion: MPS allocations capped at %.3fx the recommended working set", ratio)
+ return ratio
+
+
def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:
mps_available = False
try:
@@ -654,14 +730,17 @@ def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:
mps_available = False
if mps_available:
- # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, at the first MPS allocation (the probe below), so relax it
- # first or the allocator caps at ~1.7x recommendedMaxWorkingSet and can OOM a model that would fit. setdefault
- # respects an override.
- os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.0")
+ user_ratio = MPS_HIGH_WATERMARK_ENV in os.environ
+ # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, when the allocator is built (the probe below, or an earlier
+ # recommended_max_memory() reading). 0.0 here only lifts torch's 1.7x-of-working-set default, which sits above
+ # physical RAM; apply_mps_memory_cap() then sets the finite cap. setdefault respects an override.
+ os.environ.setdefault(MPS_HIGH_WATERMARK_ENV, "0.0")
# Prefer bfloat16, else float32, NEVER silent float16: modern DiTs produce activations far outside fp16's
# range (Z-Image MLP peaks near 9e5 -> inf -> NaN -> black image). bf16 (macOS 14+) shares fp32's exponent
# range; older macOS uses fp32.
dtype = torch.bfloat16 if _mps_supports_bfloat16(torch) else torch.float32
+ if not user_ratio:
+ apply_mps_memory_cap(torch)
return DiffusionDeviceTarget(
device = "mps",
dtype = dtype,
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index d7a9aab6831..37f53f03b86 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1631,9 +1631,9 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)
still loads, degrading to group / whole-module CPU offload and streaming from host RAM. On Apple
Silicon (and integrated CUDA) the CPU and GPU share one pool, so offload moves bytes within that
pool and frees nothing: ``plan_diffusion_memory`` correctly returns ``none`` and the load then
- allocates past physical memory. There is no torch OOM to catch, because ``_mps_or_cpu_target``
- sets PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 to disable the MPS allocator's hard limit, so the
- failure is the OS killing the process with no Python exception.
+ allocates past physical memory. ``apply_mps_memory_cap`` bounds MPS allocations just below RAM,
+ but that only turns the overrun into a late OOM after minutes of loading (and integrated CUDA has
+ no such cap), so refusing up front is still the better failure.
Weights only (``model_dense_mib`` plus the flat base overhead): the per-call runtime headroom is
a coarse activation / VAE-decode estimate and the mps path already turns on VAE tiling and
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 3548fda6687..86217373220 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -272,6 +272,65 @@ def metal_placement_flags(offload: Sequence[str]) -> list[str]:
return [f for f in flags if f not in offload]
+# 1 / 0 forces sd.cpp's --mmap on / off on macOS; unset decides from the installed build (``mac_mmap_flags``).
+SD_CPP_MMAP_ENV = "UNSLOTH_DIFFUSION_SD_CPP_MMAP"
+# leejet/stable-diffusion.cpp#2037 (master-907) maps mmapped weights through Metal device buffers. Older builds accept
+# --mmap but hand Metal a CPU buffer it cannot address, so the flag is only safe from this build on.
+METAL_MMAP_MIN_BUILD = 907
+_INSTALL_RECORD = ".unsloth-sd-cpp-install.json"
+
+
+def sd_cpp_build_number(tag: Optional[str]) -> Optional[int]:
+ """``N`` of a ``master-N-<sha>[-u<sha>]`` release tag, else None."""
+ import re
+
+ m = re.match(r"^master-(\d+)(?:-|$)", str(tag or "").strip())
+ return int(m.group(1)) if m else None
+
+
[email protected]_cache(maxsize = 8)
+def _installed_sd_cpp_tag(binary: str) -> Optional[str]:
+ """The release tag Studio's installer recorded next to ``binary`` (the binary sits at or a few levels under the
+ install root). None for a binary the installer did not place (``SD_CLI_PATH``, PATH, a source build)."""
+ import json
+ from pathlib import Path
+
+ try:
+ path = Path(binary).resolve()
+ except Exception: # noqa: BLE001
+ return None
+ for parent in list(path.parents)[:3]:
+ try:
+ with open(parent / _INSTALL_RECORD, "r", encoding = "utf-8") as f:
+ rec = json.load(f)
+ except (OSError, ValueError):
+ continue
+ tag = rec.get("tag") if isinstance(rec, dict) else None
+ return tag if isinstance(tag, str) and tag else None
+ return None
+
+
+def mac_mmap_flags(binary: Optional[str], extra_args: Optional[Sequence[str]] = None) -> list[str]:
+ """``["--mmap"]`` on macOS when ``binary`` maps weights correctly on Metal, else nothing.
+
+ With it, GGUF / safetensors weights stay clean file-backed pages (Metal wraps them in place, the CPU backend reads
+ them in place) instead of a second anonymous copy, which on unified memory is the copy that swaps: upstream measured
+ a Qwen-Image 2.1 Q4_K_M peak footprint of 10.2 -> 1.4 GB on an M1 Pro, pixel-identical. Fails closed on a build it
+ cannot date; ``UNSLOTH_DIFFUSION_SD_CPP_MMAP`` = 1 / 0 forces it either way."""
+ import os
+ import sys
+
+ if sys.platform != "darwin" or "--mmap" in (extra_args or ()):
+ return []
+ forced = os.environ.get(SD_CPP_MMAP_ENV, "").strip().lower()
+ if forced in ("0", "false", "no", "off"):
+ return []
+ if forced in ("1", "true", "yes", "on"):
+ return ["--mmap"]
+ build = sd_cpp_build_number(_installed_sd_cpp_tag(str(binary))) if binary else None
+ return ["--mmap"] if build is not None and build >= METAL_MMAP_MIN_BUILD else []
+
+
# Everything on the CPU backend. sd.cpp prefers GPU -> integrated GPU -> CPU and only `--backend` changes which
# backend EXECUTES the graph (`--offload-to-cpu` moves parameters, not compute), so this is the one flag that removes
# ggml-metal entirely.
@@ -483,6 +542,7 @@ def build_sd_cpp_command(
if offload:
cmd += offload
cmd += metal_placement_flags(offload)
+ cmd += mac_mmap_flags(binary, extra_args)
if verbose:
cmd += ["-v"]
if extra_args:
@@ -575,6 +635,7 @@ def build_sd_cpp_video_command(
if offload:
cmd += offload
cmd += metal_placement_flags(offload)
+ cmd += mac_mmap_flags(binary, extra_args)
if verbose:
cmd.append("-v")
if extra_args:
@@ -686,6 +747,7 @@ def build_sd_cpp_server_command(
backend_args = [*offload, *placement, *(extra_args or ())]
cmd += [f for f in native_speed_flags(native_speed, backend_args) if f not in offload]
cmd += placement
+ cmd += mac_mmap_flags(binary, extra_args)
if verbose:
cmd += ["-v"]
if extra_args:
diff --git a/studio/backend/tests/test_diffusion_device.py b/studio/backend/tests/test_diffusion_device.py
index 4af4b1e574b..d10450a615d 100644
--- a/studio/backend/tests/test_diffusion_device.py
+++ b/studio/backend/tests/test_diffusion_device.py
@@ -1021,3 +1021,95 @@ def test_a_multi_card_pick_declines_to_rank_when_ranking_is_barred(monkeypatch):
assert probed == []
with pytest.raises(ValueError):
dd.resolve_selected_cuda_ordinal([9], allow_ranking = False)
+
+
+# ── MPS allocation cap ────────────────────────────────────────────────
+
+GIB = 1024**3
+
+
+ "recommended_gib, ram_gib, expected_cap_gib",
+ [
+ (5.33, 8, 6.5), # 8 GB Mac: RAM - 1.5 GiB floor reserve
+ (10.67, 16, 14.4), # RAM - 10%
+ (96, 128, 115.2),
+ (5.0, 5.5, 5.0), # reserve would go below the working set: never cap under it
+ ],
+)
+def test_mps_ratio_caps_below_ram_never_below_working_set(
+ recommended_gib, ram_gib, expected_cap_gib
+):
+ ratio = dd.mps_high_watermark_ratio(int(recommended_gib * GIB), int(ram_gib * GIB))
+ cap_gib = ratio * recommended_gib
+ assert cap_gib == pytest.approx(expected_cap_gib, rel = 1e-3)
+ assert cap_gib < ram_gib
+ assert 1.0 <= ratio <= 2.0
+ assert cap_gib < 1.7 * recommended_gib # tighter than torch's own default, which is above RAM
+
+
+def test_mps_ratio_unknown_readings_mean_no_cap():
+ assert dd.mps_high_watermark_ratio(None, 16 * GIB) is None
+ assert dd.mps_high_watermark_ratio(10 * GIB, 0) is None
+ assert dd.mps_high_watermark_ratio("x", 16 * GIB) is None
+
+
+def _mps_cap_torch(
+ recorded,
+ *,
+ recommended = 10 * GIB,
+ fraction_raises = False,
+):
+ torch = _make_torch(mps_available = True, mps_probe = "pass")
+
+ def _set(fraction):
+ if fraction_raises:
+ raise AttributeError("old torch")
+ recorded.append(fraction)
+
+ torch.mps = types.SimpleNamespace(
+ recommended_max_memory = lambda: recommended, set_per_process_memory_fraction = _set
+ )
+ return torch
+
+
+def test_mps_target_sets_finite_cap(monkeypatch):
+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
+ recorded = []
+ _install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")
+ assert dd.resolve_diffusion_device_target().device == "mps"
+ assert recorded == [pytest.approx(14.4 / 10, rel = 1e-3)]
+
+
+ "env",
+ [
+ {dd.MPS_HIGH_WATERMARK_ENV: "0.0"}, # explicit "no cap" stays no cap
+ {dd.MPS_HIGH_WATERMARK_ENV: "1.2"},
+ {"UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD": "1"},
+ ],
+)
+def test_mps_cap_respects_user_override(monkeypatch, env):
+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
+ for k, v in env.items():
+ monkeypatch.setenv(k, v)
+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
+ recorded = []
+ _install(monkeypatch, _mps_cap_torch(recorded), studio_device = "mlx")
+ assert dd.resolve_diffusion_device_target().device == "mps"
+ assert recorded == []
+ if dd.MPS_HIGH_WATERMARK_ENV in env:
+ import os
+ assert os.environ[dd.MPS_HIGH_WATERMARK_ENV] == env[dd.MPS_HIGH_WATERMARK_ENV]
+
+
+def test_mps_cap_failure_keeps_target(monkeypatch):
+ monkeypatch.delenv(dd.MPS_HIGH_WATERMARK_ENV, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD", raising = False)
+ monkeypatch.setattr(dd, "_total_ram_bytes", lambda: 16 * GIB)
+ _install(monkeypatch, _mps_cap_torch([], fraction_raises = True), studio_device = "mlx")
+ t = dd.resolve_diffusion_device_target()
+ assert t.device == "mps" and t.dtype == BF16
diff --git a/studio/backend/tests/test_sd_cpp_args.py b/studio/backend/tests/test_sd_cpp_args.py
index 9efcabb01a1..45d2678b0fa 100644
--- a/studio/backend/tests/test_sd_cpp_args.py
+++ b/studio/backend/tests/test_sd_cpp_args.py
@@ -1037,3 +1037,85 @@ def test_device_backend_flags_are_appended_not_folded_into_the_policy():
before = list(policy)
device_backend_flags("CUDA1", policy)
assert policy == before
+
+
+# ── Metal mmap (leejet/stable-diffusion.cpp#2037) ───────────────────────────
+
+
+def _managed_install(tmp_path, tag):
+ import json
+
+ root = tmp_path / "stable-diffusion.cpp"
+ (root / "bin").mkdir(parents = True)
+ binary = root / "bin" / "sd-cli"
+ binary.write_text("")
+ (root / ".unsloth-sd-cpp-install.json").write_text(
+ json.dumps({"tag": tag, "accelerator": "cpu"})
+ )
+ _sd_cpp_args_module._installed_sd_cpp_tag.cache_clear()
+ return str(binary)
+
+
+def _all_builders(binary, extra_args = None):
+ files = SdCppModelFiles(
+ diffusion_model = "/m/model.gguf", vae = "/m/vae.safetensors", llm = "/m/te.gguf"
+ )
+ yield build_sd_cpp_command(
+ binary, files, SdCppGenParams(prompt = "a cat"), output_path = "/o.png", extra_args = extra_args
+ )
+ yield build_sd_cpp_video_command(
+ binary,
+ files,
+ SdCppVideoGenParams(prompt = "a cat", width = 256, height = 256, num_frames = 5),
+ output_path = "/o.webm",
+ extra_args = extra_args,
+ )
+ yield build_sd_cpp_server_command(
+ binary, files, host = "127.0.0.1", port = 1, extra_args = extra_args
+ )
+
+
+ "tag, expected",
+ [
+ ("master-951-f89d9b1-uafac7a7", True), # the pinned bundle
+ ("master-907-500ef5f", True), # the build that merged #2037
+ ("master-906-abcdef0", False), # --mmap there hands Metal an unaddressable CPU buffer
+ ("master-813-bfbef5b-u6321a69", False),
+ ("b1234", False), # undatable: fail closed
+ ],
+)
+def test_mmap_rides_every_macos_builder_from_the_metal_mmap_build(
+ monkeypatch, tmp_path, tag, expected
+):
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", raising = False)
+ monkeypatch.setattr("sys.platform", "darwin")
+ binary = _managed_install(tmp_path, tag)
+ for cmd in _all_builders(binary):
+ assert ("--mmap" in cmd) is expected
+ assert cmd.count("--mmap") <= 1
+
+
+def test_mmap_is_macos_only_and_never_on_an_unrecorded_binary(monkeypatch, tmp_path):
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", raising = False)
+ binary = _managed_install(tmp_path, "master-951-f89d9b1-uafac7a7")
+ for other in ("linux", "win32"):
+ monkeypatch.setattr("sys.platform", other)
+ assert all("--mmap" not in cmd for cmd in _all_builders(binary))
+ monkeypatch.setattr("sys.platform", "darwin")
+ stray = tmp_path / "elsewhere" / "sd-cli"
+ stray.parent.mkdir()
+ stray.write_text("")
+ assert all("--mmap" not in cmd for cmd in _all_builders(str(stray)))
+
+
+def test_mmap_env_override_and_no_duplicate(monkeypatch, tmp_path):
+ monkeypatch.setattr("sys.platform", "darwin")
+ new = _managed_install(tmp_path, "master-951-f89d9b1-uafac7a7")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", "0")
+ assert all("--mmap" not in cmd for cmd in _all_builders(new))
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP", "1")
+ assert all("--mmap" in cmd for cmd in _all_builders("/usr/local/bin/sd-cli"))
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_SD_CPP_MMAP")
+ for cmd in _all_builders(new, extra_args = ["--mmap"]):
+ assert cmd.count("--mmap") == 1