Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): reject oversized Metal alongside loads before startup (#14209)
Lee Jackson2e95c7c329cd
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 218236da468..bfe401dff54 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -620,6 +620,38 @@ class GpuMemoryShortError(RuntimeError):self.gpu_indices = gpu_indices+def _apple_wired_headroom_bytes() -> Optional[int]:+ """Wired GPU memory a Metal load may still take, system-wide; None if unknown."""+ from utils.hardware import is_apple_silicon++ if not is_apple_silicon():+ return None+ try:+ probe = subprocess.run(+ ["sysctl", "-n", "iogpu.wired_limit_mb"],+ capture_output = True,+ text = True,+ timeout = 5,+ encoding = "utf-8",+ errors = "replace",+ )+ cap_bytes = int(probe.stdout.strip()) * 1024 * 1024+ except Exception:+ return None+ # 0 means the kernel default, Metal's recommended working set.+ if cap_bytes <= 0:+ from utils.hardware.hardware import _metal_recommended_working_set_bytes+ cap_bytes = _metal_recommended_working_set_bytes()+ if cap_bytes <= 0:+ return None+ from utils.hardware.hardware import _read_apple_gpu_stats++ in_use = _read_apple_gpu_stats().get("vram_used_bytes")+ if in_use is None:+ return None+ return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))++def _gpu_plan_mib(total_bytes: int, on: list[int], gpus) -> dict[int, int]:"""MiB this load holds per GPU: an even share, capped at what each card had free, since--fit on offloads the rest to CPU."""@@ -14192,34 +14224,12 @@ class LlamaCppBackend:@staticmethoddef _apple_metal_wired_ceiling_bytes() -> int:"""Wired GPU memory a Metal load may still take, system-wide; 0 if unknown or used up."""- from utils.hardware import is_apple_silicon+ return _apple_wired_headroom_bytes() or 0- if not is_apple_silicon():- return 0- try:- probe = subprocess.run(- ["sysctl", "-n", "iogpu.wired_limit_mb"],- capture_output = True,- text = True,- timeout = 5,- encoding = "utf-8",- errors = "replace",- )- cap_bytes = int(probe.stdout.strip()) * 1024 * 1024- except Exception:- return 0- # 0 means the kernel default, Metal's recommended working set.- if cap_bytes <= 0:- from utils.hardware.hardware import _metal_recommended_working_set_bytes- cap_bytes = _metal_recommended_working_set_bytes()- if cap_bytes <= 0:- return 0- from utils.hardware.hardware import _read_apple_gpu_stats-- in_use = _read_apple_gpu_stats().get("vram_used_bytes")- if in_use is None:- return 0- return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))+ @staticmethod+ def _apple_metal_wired_headroom_bytes() -> Optional[int]:+ """The same headroom, but None when unknown, so 0 means used up."""+ return _apple_wired_headroom_bytes()@staticmethoddef _metal_fit_target_floor_mib() -> float:@@ -26615,6 +26625,7 @@ class LlamaCppBackend:# Bound before the try for the same reason as _detected_gpus._placement_verdict_partial = False_ctx_capped_for_vram = False+ _metal_alongside_shortfall: Optional[tuple[int, int]] = None# A path that never prices the launch must not commit the previous one's plan.self._pending_plan_mib = {}# Set by the expert-spill raise after the fit.@@ -28928,6 +28939,30 @@ class LlamaCppBackend:if not _caller_owns_budget and not _paravirtual_cpu_forced:_metal_fit_target_floor_mib = self._metal_fit_target_floor_mib()+ if (+ intent.refuse_partial_gpu_fit+ and gpu_memory_mode != "manual"+ and not _paravirtual_cpu_forced+ # A CPU-only child takes no Metal memory; the host-RAM guard prices it.+ and not _device_selection_is_cpu(extra_args, os.environ)+ ):+ # No discrete GPU row on Metal, so the partial-placement verdict+ # cannot refuse this; CPU offload adds no unified memory.+ budget_mib = _apple_fit_budget_mib+ # Unknown headroom skips the wired cap, a known 0 is exhausted. The+ # reserve matches the wired refusals: the footprint omits an unsized MTP draft.+ wired_bytes = self._apple_metal_wired_headroom_bytes()+ if wired_bytes is not None:+ wired_mib = int(+ wired_bytes // (1024 * 1024) * max(0.0, 1.0 - _flat_mtp_reserve)+ )+ budget_mib = min(budget_mib, wired_mib)+ need_mib = math.ceil(+ _apple_footprint_mib(effective_ctx, _fit_ctx_checkpoints)+ )+ if need_mib > budget_mib:+ _metal_alongside_shortfall = (need_mib, budget_mib)+# Prefer fewer serving slots on GPU over --fit on offload: when the extra# --parallel slots push the footprint past the pin budget, llama-server# offloads layers to host and decode collapses ~3x (#6718). Retry the fit@@ -29839,6 +29874,16 @@ class LlamaCppBackend:mtp_overhead_fn = None_mtp_kv_unsized = False+ # Outside the placement catch, which would fall back to --fit on; the route+ # evicts an idle resident and retries.+ if _metal_alongside_shortfall is not None:+ need_mib, budget_mib = _metal_alongside_shortfall+ raise GpuMemoryShortError(+ f"This model and its context need about {need_mib / 1024:.1f} GiB, "+ f"but only {budget_mib / 1024:.1f} GiB remains in this Mac's "+ "unified-memory budget. Free memory or use a smaller model.",+ short_mib = need_mib - budget_mib,+ )# A hand-set context unified memory cannot hold. Raised here so the handler# above cannot turn it back into the launch it refuses, and ahead of the# Vulkan and APU checks, which describe hardware this branch has ruled out.diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 95b03f86a5e..649906732a1 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -562,9 +562,13 @@ def _hf_cache_is_empty(_empty_hf_hub_cache, monkeypatch):def _no_live_metal_wired_ceiling(monkeypatch):"""Keep Metal context verdicts off the host's live GPU memory."""from core.inference.llama_cpp import LlamaCppBackend+monkeypatch.setattr(LlamaCppBackend, "_apple_metal_wired_ceiling_bytes", staticmethod(lambda: 0))+ monkeypatch.setattr(+ LlamaCppBackend, "_apple_metal_wired_headroom_bytes", staticmethod(lambda: None)+ )@pytest.fixture(autouse = True)diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.pyindex fcaf2d880de..1f26d655b56 100644--- a/studio/backend/tests/test_metal_explicit_context_guard.py+++ b/studio/backend/tests/test_metal_explicit_context_guard.py@@ -123,6 +123,9 @@ def _launch(mmproj_bytes = 0,wired_bytes = 0,ckpt_bytes = 0,+ refuse_partial_gpu_fit = False,+ launches = None,+ wired_exhausted = False,):"""Drive the real load_model with no GPU enumerated (the Metal condition).@@ -138,7 +141,8 @@ def _launch(helper's actual return contract -- its 4096 floor, and its habit of handing therequest straight back. ``budget_bytes`` / ``weights_bytes`` / ``kv_per_token`` /``native`` then place the model against the budget, and only matter with ``real_fit``.- ``wired_bytes`` is the GPU wired headroom, 0 when unreadable. ``ckpt_bytes`` prices each+ ``wired_bytes`` is the GPU wired headroom, 0 when unreadable; ``wired_exhausted`` reads+ it as known and used up. ``ckpt_bytes`` prices eachSWA context checkpoint and advertises the flag, so ``--ctx-checkpoints`` is charged."""monkeypatch.setattr(@@ -151,6 +155,11 @@ def _launch("_apple_metal_wired_ceiling_bytes",staticmethod(lambda: wired_bytes if metal else 0),)+ monkeypatch.setattr(+ LlamaCppBackend,+ "_apple_metal_wired_headroom_bytes",+ staticmethod(lambda: (0 if wired_exhausted else wired_bytes or None) if metal else None),+ )if paravirtual:import core.inference.llama_cpp as _llama_cppmonkeypatch.setattr(_llama_cpp, "_metal_device_is_paravirtual", lambda: True)@@ -208,6 +217,8 @@ def _launch(if not cmd or str(cmd[0]) != "/fake/llama-server":return _REAL_POPEN(cmd, **kwargs)captured["cmd"] = list(cmd)+ if launches is not None:+ launches.append(list(cmd))return type("Process",(),@@ -231,6 +242,7 @@ def _launch(gguf_path = str(_write_gguf(tmp_path / "model.gguf")),model_identifier = "test",n_ctx = n_ctx,+ refuse_partial_gpu_fit = refuse_partial_gpu_fit,gpu_memory_mode = gpu_memory_mode,gpu_layers = gpu_layers,extra_args = extra_args,@@ -1943,3 +1955,114 @@ class TestAutoGivesUpContextBeforeLayers:"""The explicit path refuses against the wired limit instead (#10804)."""with pytest.raises(RuntimeError, match = "24,576"):self._ctx(tmp_path, monkeypatch, 7.5, n_ctx = 32768)+++class TestMetalAlongsideAdmission:+ """An alongside load must report a shared-memory shortfall before spawning."""++ @pytest.mark.parametrize("n_ctx", [0, 32768])+ @pytest.mark.parametrize("budget_gib,wired_gib", [(9, 8), (32, 8)])+ def test_weights_over_remaining_memory_never_spawn(+ self, tmp_path, monkeypatch, n_ctx, budget_gib, wired_gib+ ):+ from core.inference.llama_cpp import GpuMemoryShortError++ launches = []+ with pytest.raises(GpuMemoryShortError) as caught:+ _launch(+ tmp_path,+ monkeypatch,+ n_ctx = n_ctx,+ real_fit = True,+ weights_bytes = 16 * 1024**3,+ budget_bytes = budget_gib * 1024**3,+ wired_bytes = wired_gib * 1024**3,+ cache_type_kv = "q8_0",+ refuse_partial_gpu_fit = True,+ launches = launches,+ )+ assert launches == []+ assert caught.value.short_mib > 0+ assert not caught.value.capped+ # Metal and MLX residents share one pool, not discrete GPU ordinals.+ assert caught.value.gpu_indices is None++ def test_minimum_context_must_fit_too(self, tmp_path, monkeypatch):+ from core.inference.llama_cpp import GpuMemoryShortError+ with pytest.raises(GpuMemoryShortError):+ _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ real_fit = True,+ weights_bytes = 1024**3,+ budget_bytes = 9 * 1024**3,+ kv_per_token = 16 * 1024**2,+ refuse_partial_gpu_fit = True,+ )++ def test_used_up_wired_headroom_refuses_a_small_model(self, tmp_path, monkeypatch):+ from core.inference.llama_cpp import GpuMemoryShortError++ launches = []+ with pytest.raises(GpuMemoryShortError):+ _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ real_fit = True,+ weights_bytes = 1024**3,+ budget_bytes = 32 * 1024**3,+ wired_exhausted = True,+ refuse_partial_gpu_fit = True,+ launches = launches,+ )+ assert launches == []++ def test_auto_context_that_fits_still_launches(self, tmp_path, monkeypatch):+ captured = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ real_fit = True,+ weights_bytes = 1024**3,+ budget_bytes = 9 * 1024**3,+ refuse_partial_gpu_fit = True,+ )+ assert captured["cmd"][0] == "/fake/llama-server"++ @pytest.mark.parametrize(+ "kwargs",+ [+ {"refuse_partial_gpu_fit": False},+ {"refuse_partial_gpu_fit": True, "gpu_memory_mode": "manual", "gpu_layers": 0},+ {"refuse_partial_gpu_fit": True, "paravirtual": True},+ {"refuse_partial_gpu_fit": True, "extra_args": ["--device", "none"]},+ ],+ )+ def test_existing_explicit_exemptions(self, tmp_path, monkeypatch, kwargs):+ assert _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ real_fit = True,+ weights_bytes = 16 * 1024**3,+ budget_bytes = 9 * 1024**3,+ **kwargs,+ )["cmd"]++ def test_retry_rechecks_memory_after_eviction(self, tmp_path, monkeypatch):+ from core.inference.llama_cpp import GpuMemoryShortError++ backend = LlamaCppBackend()+ args = dict(+ n_ctx = 0,+ real_fit = True,+ weights_bytes = 16 * 1024**3,+ refuse_partial_gpu_fit = True,+ backend = backend,+ )+ with pytest.raises(GpuMemoryShortError):+ _launch(tmp_path, monkeypatch, budget_bytes = 9 * 1024**3, **args)+ assert not backend.is_active+ assert _launch(tmp_path, monkeypatch, budget_bytes = 32 * 1024**3, **args)["cmd"]diff --git a/studio/backend/tests/test_multi_model_serving.py b/studio/backend/tests/test_multi_model_serving.pyindex d64fba0ac20..4d7671e5ff1 100644--- a/studio/backend/tests/test_multi_model_serving.py+++ b/studio/backend/tests/test_multi_model_serving.py@@ -1524,3 +1524,39 @@ def test_the_replace_fallback_frees_the_active_model_before_placing_the_new_one(inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s"))assert seen == [False] and primary.model_identifier == "org/C-GGUF"+++def test_metal_shortfall_evicts_then_rechecks_shared_memory(backends, monkeypatch, tmp_path):+ from .test_metal_explicit_context_guard import _launch++ primary, extra = backends+ extra.request = LoadRequest(model_path = "org/B-GGUF", gguf_variant = "Q8_0")+ attempts, launches = [], []+ monkeypatch.setattr(inf, "release_chat_gpu_claim", lambda: None)++ async def load(request, *args, **kwargs):+ budget_gib = 9 if extra.llama.is_active else 32+ attempts.append(budget_gib)+ _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ real_fit = True,+ budget_bytes = budget_gib * 1024**3,+ weights_bytes = 16 * 1024**3,+ refuse_partial_gpu_fit = not request.force_alongside,+ launches = launches,+ )+ llama = inf.get_llama_cpp_backend()+ llama.model_identifier = request.model_path+ llama.is_loaded = llama.is_active = True+ return LoadResponse.model_construct(status = "loaded", model = request.model_path)++ monkeypatch.setattr(inf, "_run_tracked_load_model_impl", load)+ response = asyncio.run(+ inf.load_model_gated(LoadRequest(model_path = "org/C-GGUF", alongside = True), None, "s")+ )+ assert attempts == [9, 32]+ assert len(launches) == 1+ assert response.evicted == ["org/B-GGUF:Q8_0"]+ assert primary.is_loaded