Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): reject oversized Metal alongside loads before startup (#14209)

Lee Jackson2e95c7c329cd
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 218236da468..bfe401dff54 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -620,6 +620,38 @@ class GpuMemoryShortError(RuntimeError):
self.gpu_indices = gpu_indices
+def _apple_wired_headroom_bytes() -> Optional[int]:
+ """Wired GPU memory a Metal load may still take, system-wide; None if unknown."""
+ from utils.hardware import is_apple_silicon
+
+ if not is_apple_silicon():
+ return None
+ try:
+ probe = subprocess.run(
+ ["sysctl", "-n", "iogpu.wired_limit_mb"],
+ capture_output = True,
+ text = True,
+ timeout = 5,
+ encoding = "utf-8",
+ errors = "replace",
+ )
+ cap_bytes = int(probe.stdout.strip()) * 1024 * 1024
+ except Exception:
+ return None
+ # 0 means the kernel default, Metal's recommended working set.
+ if cap_bytes <= 0:
+ from utils.hardware.hardware import _metal_recommended_working_set_bytes
+ cap_bytes = _metal_recommended_working_set_bytes()
+ if cap_bytes <= 0:
+ return None
+ from utils.hardware.hardware import _read_apple_gpu_stats
+
+ in_use = _read_apple_gpu_stats().get("vram_used_bytes")
+ if in_use is None:
+ return None
+ return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))
+
+
def _gpu_plan_mib(total_bytes: int, on: list[int], gpus) -> dict[int, int]:
"""MiB this load holds per GPU: an even share, capped at what each card had free, since
--fit on offloads the rest to CPU."""
@@ -14192,34 +14224,12 @@ class LlamaCppBackend:
@staticmethod
def _apple_metal_wired_ceiling_bytes() -> int:
"""Wired GPU memory a Metal load may still take, system-wide; 0 if unknown or used up."""
- from utils.hardware import is_apple_silicon
+ return _apple_wired_headroom_bytes() or 0
- if not is_apple_silicon():
- return 0
- try:
- probe = subprocess.run(
- ["sysctl", "-n", "iogpu.wired_limit_mb"],
- capture_output = True,
- text = True,
- timeout = 5,
- encoding = "utf-8",
- errors = "replace",
- )
- cap_bytes = int(probe.stdout.strip()) * 1024 * 1024
- except Exception:
- return 0
- # 0 means the kernel default, Metal's recommended working set.
- if cap_bytes <= 0:
- from utils.hardware.hardware import _metal_recommended_working_set_bytes
- cap_bytes = _metal_recommended_working_set_bytes()
- if cap_bytes <= 0:
- return 0
- from utils.hardware.hardware import _read_apple_gpu_stats
-
- in_use = _read_apple_gpu_stats().get("vram_used_bytes")
- if in_use is None:
- return 0
- return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))
+ @staticmethod
+ def _apple_metal_wired_headroom_bytes() -> Optional[int]:
+ """The same headroom, but None when unknown, so 0 means used up."""
+ return _apple_wired_headroom_bytes()
@staticmethod
def _metal_fit_target_floor_mib() -> float:
@@ -26615,6 +26625,7 @@ class LlamaCppBackend:
# Bound before the try for the same reason as _detected_gpus.
_placement_verdict_partial = False
_ctx_capped_for_vram = False
+ _metal_alongside_shortfall: Optional[tuple[int, int]] = None
# A path that never prices the launch must not commit the previous one's plan.
self._pending_plan_mib = {}
# Set by the expert-spill raise after the fit.
@@ -28928,6 +28939,30 @@ class LlamaCppBackend:
if not _caller_owns_budget and not _paravirtual_cpu_forced:
_metal_fit_target_floor_mib = self._metal_fit_target_floor_mib()
+ if (
+ intent.refuse_partial_gpu_fit
+ and gpu_memory_mode != "manual"
+ and not _paravirtual_cpu_forced
+ # A CPU-only child takes no Metal memory; the host-RAM guard prices it.
+ and not _device_selection_is_cpu(extra_args, os.environ)
+ ):
+ # No discrete GPU row on Metal, so the partial-placement verdict
+ # cannot refuse this; CPU offload adds no unified memory.
+ budget_mib = _apple_fit_budget_mib
+ # Unknown headroom skips the wired cap, a known 0 is exhausted. The
+ # reserve matches the wired refusals: the footprint omits an unsized MTP draft.
+ wired_bytes = self._apple_metal_wired_headroom_bytes()
+ if wired_bytes is not None:
+ wired_mib = int(
+ wired_bytes // (1024 * 1024) * max(0.0, 1.0 - _flat_mtp_reserve)
+ )
+ budget_mib = min(budget_mib, wired_mib)
+ need_mib = math.ceil(
+ _apple_footprint_mib(effective_ctx, _fit_ctx_checkpoints)
+ )
+ if need_mib > budget_mib:
+ _metal_alongside_shortfall = (need_mib, budget_mib)
+
# Prefer fewer serving slots on GPU over --fit on offload: when the extra
# --parallel slots push the footprint past the pin budget, llama-server
# offloads layers to host and decode collapses ~3x (#6718). Retry the fit
@@ -29839,6 +29874,16 @@ class LlamaCppBackend:
mtp_overhead_fn = None
_mtp_kv_unsized = False
+ # Outside the placement catch, which would fall back to --fit on; the route
+ # evicts an idle resident and retries.
+ if _metal_alongside_shortfall is not None:
+ need_mib, budget_mib = _metal_alongside_shortfall
+ raise GpuMemoryShortError(
+ f"This model and its context need about {need_mib / 1024:.1f} GiB, "
+ f"but only {budget_mib / 1024:.1f} GiB remains in this Mac's "
+ "unified-memory budget. Free memory or use a smaller model.",
+ short_mib = need_mib - budget_mib,
+ )
# A hand-set context unified memory cannot hold. Raised here so the handler
# above cannot turn it back into the launch it refuses, and ahead of the
# Vulkan and APU checks, which describe hardware this branch has ruled out.
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 95b03f86a5e..649906732a1 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -562,9 +562,13 @@ def _hf_cache_is_empty(_empty_hf_hub_cache, monkeypatch):
def _no_live_metal_wired_ceiling(monkeypatch):
"""Keep Metal context verdicts off the host's live GPU memory."""
from core.inference.llama_cpp import LlamaCppBackend
+
monkeypatch.setattr(
LlamaCppBackend, "_apple_metal_wired_ceiling_bytes", staticmethod(lambda: 0)
)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_apple_metal_wired_headroom_bytes", staticmethod(lambda: None)
+ )
@pytest.fixture(autouse = True)
diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.py
index fcaf2d880de..1f26d655b56 100644
--- a/studio/backend/tests/test_metal_explicit_context_guard.py
+++ b/studio/backend/tests/test_metal_explicit_context_guard.py
@@ -123,6 +123,9 @@ def _launch(
mmproj_bytes = 0,
wired_bytes = 0,
ckpt_bytes = 0,
+ refuse_partial_gpu_fit = False,
+ launches = None,
+ wired_exhausted = False,
):
"""Drive the real load_model with no GPU enumerated (the Metal condition).
@@ -138,7 +141,8 @@ def _launch(
helper's actual return contract -- its 4096 floor, and its habit of handing the
request straight back. ``budget_bytes`` / ``weights_bytes`` / ``kv_per_token`` /
``native`` then place the model against the budget, and only matter with ``real_fit``.
- ``wired_bytes`` is the GPU wired headroom, 0 when unreadable. ``ckpt_bytes`` prices each
+ ``wired_bytes`` is the GPU wired headroom, 0 when unreadable; ``wired_exhausted`` reads
+ it as known and used up. ``ckpt_bytes`` prices each
SWA context checkpoint and advertises the flag, so ``--ctx-checkpoints`` is charged.
"""
monkeypatch.setattr(
@@ -151,6 +155,11 @@ def _launch(
"_apple_metal_wired_ceiling_bytes",
staticmethod(lambda: wired_bytes if metal else 0),
)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_apple_metal_wired_headroom_bytes",
+ staticmethod(lambda: (0 if wired_exhausted else wired_bytes or None) if metal else None),
+ )
if paravirtual:
import core.inference.llama_cpp as _llama_cpp
monkeypatch.setattr(_llama_cpp, "_metal_device_is_paravirtual", lambda: True)
@@ -208,6 +217,8 @@ def _launch(
if not cmd or str(cmd[0]) != "/fake/llama-server":
return _REAL_POPEN(cmd, **kwargs)
captured["cmd"] = list(cmd)
+ if launches is not None:
+ launches.append(list(cmd))
return type(
"Process",
(),
@@ -231,6 +242,7 @@ def _launch(
gguf_path = str(_write_gguf(tmp_path / "model.gguf")),
model_identifier = "test",
n_ctx = n_ctx,
+ refuse_partial_gpu_fit = refuse_partial_gpu_fit,
gpu_memory_mode = gpu_memory_mode,
gpu_layers = gpu_layers,
extra_args = extra_args,
@@ -1943,3 +1955,114 @@ class TestAutoGivesUpContextBeforeLayers:
"""The explicit path refuses against the wired limit instead (#10804)."""
with pytest.raises(RuntimeError, match = "24,576"):
self._ctx(tmp_path, monkeypatch, 7.5, n_ctx = 32768)
+
+
+class TestMetalAlongsideAdmission:
+ """An alongside load must report a shared-memory shortfall before spawning."""
+
+ @pytest.mark.parametrize("n_ctx", [0, 32768])
+ @pytest.mark.parametrize("budget_gib,wired_gib", [(9, 8), (32, 8)])
+ def test_weights_over_remaining_memory_never_spawn(
+ self, tmp_path, monkeypatch, n_ctx, budget_gib, wired_gib
+ ):
+ from core.inference.llama_cpp import GpuMemoryShortError
+
+ launches = []
+ with pytest.raises(GpuMemoryShortError) as caught:
+ _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = n_ctx,
+ real_fit = True,
+ weights_bytes = 16 * 1024**3,
+ budget_bytes = budget_gib * 1024**3,
+ wired_bytes = wired_gib * 1024**3,
+ cache_type_kv = "q8_0",
+ refuse_partial_gpu_fit = True,
+ launches = launches,
+ )
+ assert launches == []
+ assert caught.value.short_mib > 0
+ assert not caught.value.capped
+ # Metal and MLX residents share one pool, not discrete GPU ordinals.
+ assert caught.value.gpu_indices is None
+
+ def test_minimum_context_must_fit_too(self, tmp_path, monkeypatch):
+ from core.inference.llama_cpp import GpuMemoryShortError
+ with pytest.raises(GpuMemoryShortError):
+ _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ real_fit = True,
+ weights_bytes = 1024**3,
+ budget_bytes = 9 * 1024**3,
+ kv_per_token = 16 * 1024**2,
+ refuse_partial_gpu_fit = True,
+ )
+
+ def test_used_up_wired_headroom_refuses_a_small_model(self, tmp_path, monkeypatch):
+ from core.inference.llama_cpp import GpuMemoryShortError
+
+ launches = []
+ with pytest.raises(GpuMemoryShortError):
+ _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ real_fit = True,
+ weights_bytes = 1024**3,
+ budget_bytes = 32 * 1024**3,
+ wired_exhausted = True,
+ refuse_partial_gpu_fit = True,
+ launches = launches,
+ )
+ assert launches == []
+
+ def test_auto_context_that_fits_still_launches(self, tmp_path, monkeypatch):
+ captured = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ real_fit = True,
+ weights_bytes = 1024**3,
+ budget_bytes = 9 * 1024**3,
+ refuse_partial_gpu_fit = True,
+ )
+ assert captured["cmd"][0] == "/fake/llama-server"
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"refuse_partial_gpu_fit": False},
+ {"refuse_partial_gpu_fit": True, "gpu_memory_mode": "manual", "gpu_layers": 0},
+ {"refuse_partial_gpu_fit": True, "paravirtual": True},
+ {"refuse_partial_gpu_fit": True, "extra_args": ["--device", "none"]},
+ ],
+ )
+ def test_existing_explicit_exemptions(self, tmp_path, monkeypatch, kwargs):
+ assert _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ real_fit = True,
+ weights_bytes = 16 * 1024**3,
+ budget_bytes = 9 * 1024**3,
+ **kwargs,
+ )["cmd"]
+
+ def test_retry_rechecks_memory_after_eviction(self, tmp_path, monkeypatch):
+ from core.inference.llama_cpp import GpuMemoryShortError
+
+ backend = LlamaCppBackend()
+ args = dict(
+ n_ctx = 0,
+ real_fit = True,
+ weights_bytes = 16 * 1024**3,
+ refuse_partial_gpu_fit = True,
+ backend = backend,
+ )
+ with pytest.raises(GpuMemoryShortError):
+ _launch(tmp_path, monkeypatch, budget_bytes = 9 * 1024**3, **args)
+ assert not backend.is_active
+ assert _launch(tmp_path, monkeypatch, budget_bytes = 32 * 1024**3, **args)["cmd"]
diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.py
index d64fba0ac20..4d7671e5ff1 100644
--- a/studio/backend/tests/test_multi_model_serving.py
+++ b/studio/backend/tests/test_multi_model_serving.py
@@ -1524,3 +1524,39 @@ def test_the_replace_fallback_frees_the_active_model_before_placing_the_new_one(
inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
)
assert seen == [False] and primary.model_identifier == "org/C-GGUF"
+
+
+def test_metal_shortfall_evicts_then_rechecks_shared_memory(backends, monkeypatch, tmp_path):
+ from .test_metal_explicit_context_guard import _launch
+
+ primary, extra = backends
+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")
+ attempts, launches = [], []
+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)
+
+ async def load(request, *args, **kwargs):
+ budget_gib = 9 if extra.llama.is_active else 32
+ attempts.append(budget_gib)
+ _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ real_fit = True,
+ budget_bytes = budget_gib * 1024**3,
+ weights_bytes = 16 * 1024**3,
+ refuse_partial_gpu_fit = not request.force_alongside,
+ launches = launches,
+ )
+ llama = inf.get_llama_cpp_backend()
+ llama.model_identifier = request.model_path
+ llama.is_loaded = llama.is_active = True
+ return LoadResponse.model_construct(status = "loaded", model = request.model_path)
+
+ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)
+ response = asyncio.run(
+ inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")
+ )
+ assert attempts == [9, 32]
+ assert len(launches) == 1
+ assert response.evicted == ["org/B-GGUF:Q8_0"]
+ assert primary.is_loaded