Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: refuse a hand-set Metal context only past the GPU wired limit (#10804)
Long Yixinge8c5187b22d5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex dede00d3d92..ddc125afdbd 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3775,6 +3775,9 @@ _CTX_FIT_VRAM_FRACTION = 0.97# 0.85 MLX uses in mlx_inference.py (_configure_memory_limits); not kept in sync._APPLE_UNIFIED_MEMORY_FRACTION = 0.85+# The 5% of wired headroom held back absorbs error in the footprint estimate.+_APPLE_WIRED_CEILING_FRACTION = 0.95+# _fit_context_to_vram's floor: the shortest context the search will settle for, so a# value equal to it is that floor rather than a measurement. The Metal branch re-prices# from _FIT_FLOOR_MIN_CTX (the search's 256 alignment step) before trusting it.@@ -12372,6 +12375,45 @@ class LlamaCppBackend:)return unmapped+ @staticmethod+ def _apple_metal_wired_ceiling_bytes() -> int:+ """Wired GPU memory a Metal load may still take, system-wide; 0 if unknown or used up."""+ from utils.hardware import is_apple_silicon++ if not is_apple_silicon():+ return 0+ try:+ probe = subprocess.run(+ ["sysctl", "-n", "iogpu.wired_limit_mb"],+ capture_output = True,+ text = True,+ timeout = 5,+ encoding = "utf-8",+ errors = "replace",+ )+ cap_bytes = int(probe.stdout.strip()) * 1024 * 1024+ except Exception:+ return 0+ # 0 means the kernel default, Metal's recommended working set.+ if cap_bytes <= 0:+ try:+ import mlx.core as mx+ from utils.hardware.hardware import _mlx_device_info+ if mx.metal.is_available():+ cap_bytes = int(+ _mlx_device_info(mx).get("max_recommended_working_set_size") or 0+ )+ except Exception:+ cap_bytes = 0+ if cap_bytes <= 0:+ return 0+ from utils.hardware.hardware import _read_apple_gpu_stats++ in_use = _read_apple_gpu_stats().get("vram_used_bytes")+ if in_use is None:+ return 0+ return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))+@staticmethoddef _rocm_arch_gate_keep(binary: Optional[str], torch_mod, for_llama_server: bool@@ -15031,6 +15073,7 @@ class LlamaCppBackend:cache_type_kv: Optional[str] = None,*,nothing_fits: bool = False,+ wired_limit: bool = False,) -> Optional[str]:"""Refusal when a hand-set context exceeds what unified memory holds (else None).@@ -15064,6 +15107,9 @@ class LlamaCppBackend:branch falls back to when KV cannot be sized is a guess, and refusing against itwould block contexts that load fine today.+ ``wired_limit`` marks a ceiling priced under the GPU wired headroom, which Auto does not+ size from, so the advice leaves Auto out.+UNSLOTH_ALLOW_METAL_CTX_OVERCOMMIT=1 abstains, matching the host-offload opt-out,though the failure mode it re-enables is the whole machine rather than one app."""@@ -15092,6 +15138,7 @@ class LlamaCppBackend:if (cache_type_kv or "f16").strip().lower() in ("f16", "fp16", "")else "")+ auto_advice = "" if wired_limit else "leave it on Auto, "if nothing_fits:return ("No context fits in this Mac's unified memory with this model. The weights "@@ -15109,11 +15156,26 @@ class LlamaCppBackend:"tokens. The GPU and the rest of the system share one pool here, so there is ""nothing to offload to, and the load would bring the machine down instead of "f"reporting an error. Lower the context to {max_available_ctx:,} or less, "- "leave it on Auto, or use a more quantized GGUF."+ f"{auto_advice}or use a more quantized GGUF."f"{kv_hint} Set "f"{LlamaCppBackend.METAL_CTX_OVERCOMMIT_ENV}=1 to load it anyway.")+ @staticmethod+ def _metal_context_pressure_message(+ requested_ctx: int, need_mib: float, free_budget_mib: float+ ) -> str:+ # need up, free down, so the printed pair cannot round into a tie+ need_gb = math.ceil(need_mib / 1024)+ free_gb = math.floor(max(0.0, free_budget_mib) / 1024)+ return (+ f"A context of {requested_ctx:,} tokens needs about {need_gb} GB of unified "+ f"memory, more than the {free_gb} GB Studio budgets from the memory free right "+ "now. It fits under the GPU's wired memory limit, so it is loading anyway, but "+ "macOS may have to compress or swap other apps to make room and generation may "+ "slow down. Lower the context or free memory to avoid that."+ )+@staticmethoddef _cuda_context_overcommit_notice(requested_ctx: int,@@ -23970,6 +24032,7 @@ class LlamaCppBackend:# the handler's own log line on purpose: test_tp_vision_regression# string-searches this function's source for it to check ordering.)_metal_ctx_refusal: Optional[str] = None+ _metal_ctx_warning: Optional[str] = None# A discounted Metal ceiling must launch the measured full-offload placement._metal_full_offload_pinned = False# The fit and launch must use the same Model Memory snapshot.@@ -25667,10 +25730,7 @@ class LlamaCppBackend:elif _apple_budget_mib > 0 and effective_ctx > 0:# No GPU on Metal: the branches above are skipped and the context# stays at native, over-committing unified memory (#5118, #6529).- # Cap with the same fit math; Auto shrinks to the cap, an explicit- # request above it is refused. "--fit on" stays a backstop but not- # one this can lean on: llama.cpp sizes its reduction from- # ggml-metal's free-memory report, blind to Unsloth's own footprint+ # Not "--fit on" alone: llama.cpp sizes from ggml-metal's free-memory report, blind to Unsloth's own footprint# and the wired limit. See _metal_context_overcommit_message.native_ctx_for_cap = self._context_length or effective_ctx# This arm's floor, which is _FIT_MIN_CTX only while the child's@@ -25725,10 +25785,14 @@ class LlamaCppBackend:_apple_model_size_fit / (1024**3),)- def _apple_ctx_fit(target: int, min_ctx: int) -> int:+ def _apple_ctx_fit(+ target: int,+ min_ctx: int,+ budget_mib: Optional[int] = None,+ ) -> int:return self._fit_context_to_vram(target,- _apple_fit_budget_mib,+ _apple_fit_budget_mib if budget_mib is None else budget_mib,_apple_model_size_fit,cache_type_kv,min_ctx = min_ctx,@@ -25746,10 +25810,10 @@ class LlamaCppBackend:ctx_checkpoints = _fit_ctx_checkpoints,)- def _apple_footprint_mib(ctx: int) -> float:+ def _apple_footprint_mib(ctx: int, ctx_checkpoints: int = 0) -> float:return (_apple_model_size_fit- + _kv_bytes(ctx)+ + _kv_bytes(ctx, ctx_checkpoints)+ _mtp_bytes(ctx)+ _cc_bytes(ctx)) / (1024 * 1024)@@ -25824,7 +25888,7 @@ class LlamaCppBackend:effective_ctx, cache_type_kv)elif (- (_apple_measured_ceiling is not None or _apple_nothing_fits)+ self._can_estimate_kv()and not _caller_owns_budgetand not _paravirtual_cpu_forced):@@ -25887,12 +25951,46 @@ class LlamaCppBackend:max_available_ctx = max(max_available_ctx, _apple_measured_ceiling)- _metal_ctx_refusal = self._metal_context_overcommit_message(- effective_ctx,- _apple_measured_ceiling or 0,- cache_type_kv,- nothing_fits = _apple_nothing_fits,+ # Past free memory macOS swaps; past the wired limit the machine panics.+ _apple_wired_mib = int(+ self._apple_metal_wired_ceiling_bytes()+ // (1024 * 1024)+ * max(0.0, 1.0 - _flat_mtp_reserve))+ if (+ _apple_wired_mib <= 0+ or _apple_model_size_fit / (1024 * 1024) > _apple_wired_mib+ ):+ _metal_ctx_refusal = self._metal_context_overcommit_message(+ effective_ctx,+ _apple_measured_ceiling or 0,+ cache_type_kv,+ nothing_fits = _apple_nothing_fits,+ )+ else:+ # Priced like the fit (SWA checkpoints included) so admission matches the refusal ceiling.+ _requested_mib = _apple_footprint_mib(+ effective_ctx, _fit_ctx_checkpoints+ )+ if _requested_mib > _apple_wired_mib:+ _wired_ctx = _apple_ctx_fit(+ effective_ctx, _FIT_FLOOR_MIN_CTX, _apple_wired_mib+ )+ _wired_fits = (+ _apple_footprint_mib(_wired_ctx, _fit_ctx_checkpoints)+ <= _apple_wired_mib+ )+ _metal_ctx_refusal = self._metal_context_overcommit_message(+ effective_ctx,+ _wired_ctx if _wired_fits else 0,+ cache_type_kv,+ nothing_fits = not _wired_fits,+ wired_limit = True,+ )+ elif _requested_mib > _apple_fit_budget_mib:+ _metal_ctx_warning = self._metal_context_pressure_message(+ effective_ctx, _requested_mib, _apple_fit_budget_mib+ )# Unmeasured floors still rely on llama.cpp's fitter._metal_full_offload_pinned = bool(_apple_host_embd and _apple_measured_ceiling is not None@@ -26433,6 +26531,7 @@ class LlamaCppBackend:# Vulkan and APU checks, which describe hardware this branch has ruled out.if _metal_ctx_refusal:raise RuntimeError(_metal_ctx_refusal)+ self._record_load_warning(_metal_ctx_warning)# An unenumerated explicit Vulkan ordinal can't be pinned; fail loudly# instead of fitting onto an unselected device. Clear the raw selectiondiff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 528ac6a9a69..ad270a128cf 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -424,6 +424,15 @@ def _hf_cache_is_empty(_empty_hf_hub_cache, monkeypatch):monkeypatch.setattr(constants, "HF_HUB_CACHE", _empty_hf_hub_cache)[email protected](autouse = True)+def _no_live_metal_wired_ceiling(monkeypatch):+ """Keep Metal context verdicts off the host's live GPU memory."""+ from core.inference.llama_cpp import LlamaCppBackend+ monkeypatch.setattr(+ LlamaCppBackend, "_apple_metal_wired_ceiling_bytes", staticmethod(lambda: 0)+ )++@pytest.fixture(autouse = True)def _no_leftover_generation_account(monkeypatch):"""Clear the media-generation owner, which is a process global no route ever resets.diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.pyindex 341ff2c90eb..7856491e45e 100644--- a/studio/backend/tests/test_llama_cpp_context_fit.py+++ b/studio/backend/tests/test_llama_cpp_context_fit.py@@ -78,6 +78,7 @@ except ImportError:from core.inference.llama_cpp import (_AUTO_OFFLOAD_CTX,_APPLE_UNIFIED_MEMORY_FRACTION,+ _APPLE_WIRED_CEILING_FRACTION,_CTX_FIT_VRAM_FRACTION,LlamaCppBackend,classify_gpu_offload_lines,@@ -827,6 +828,89 @@ class TestAppleUnifiedMemoryBudget:assert LlamaCppBackend._apple_metal_memory_budget_bytes() == 0+# Unpatched: conftest pins the probe to 0.+_REAL_WIRED_CEILING = LlamaCppBackend.__dict__["_apple_metal_wired_ceiling_bytes"].__func__+++def _install_wired_probes(monkeypatch, *, sysctl_mb, working_set, in_use):+ """``sysctl_mb`` or ``in_use`` of None is an unreadable probe."""+ from core.inference import llama_cpp as _llama_cpp+ from utils.hardware import hardware as _hardware++ _force_apple(monkeypatch)+ _install_fake_mlx(monkeypatch, working_set)+ sysctl_out = "" if sysctl_mb is None else f"{sysctl_mb}\n"+ monkeypatch.setattr(+ _llama_cpp.subprocess,+ "run",+ lambda *a, **k: _types.SimpleNamespace(stdout = sysctl_out),+ )+ monkeypatch.setattr(+ _hardware,+ "_read_apple_gpu_stats",+ lambda: {} if in_use is None else {"vram_used_bytes": in_use},+ )+++class TestAppleWiredCeiling:+ def test_zero_off_apple_silicon(self, monkeypatch):+ import platform as _platform++ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)+ monkeypatch.setattr(_platform, "system", lambda: "Linux")+ monkeypatch.setattr(_platform, "machine", lambda: "x86_64")+ assert _REAL_WIRED_CEILING() == 0++ def test_the_working_set_is_the_limit_by_default(self, monkeypatch):+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)+ assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)++ def test_older_mlx_reads_the_legacy_alias(self, monkeypatch):+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)+ mlx_core = sys.modules["mlx.core"]+ mlx_core.metal.device_info = mlx_core.device_info+ monkeypatch.delattr(mlx_core, "device_info")+ assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)++ def test_a_set_sysctl_limit_wins(self, monkeypatch):+ _install_wired_probes(+ monkeypatch, sysctl_mb = 56 * 1024, working_set = 48 * GIB, in_use = 2 * GIB+ )+ assert _REAL_WIRED_CEILING() == int(54 * GIB * _APPLE_WIRED_CEILING_FRACTION)++ def test_unread_gpu_memory_is_unresolved_not_zero(self, monkeypatch):+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = None)+ assert _REAL_WIRED_CEILING() == 0++ def test_unread_sysctl_is_unresolved_not_the_default(self, monkeypatch):+ _install_wired_probes(monkeypatch, sysctl_mb = None, working_set = 48 * GIB, in_use = 2 * GIB)+ assert _REAL_WIRED_CEILING() == 0++ def test_no_readable_limit_is_unresolved(self, monkeypatch):+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 0, in_use = 2 * GIB)+ assert _REAL_WIRED_CEILING() == 0++ @pytest.mark.parametrize(+ "counter,expected",+ [+ ("", 0),+ ('"In use system memory"=2147483648,', int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)),+ ],+ )+ def test_ioreg_resolves_only_with_the_in_use_counter(self, monkeypatch, counter, expected):+ _force_apple(monkeypatch)+ _install_fake_mlx(monkeypatch, 48 * GIB)+ block = f'"PerformanceStatistics" = {{"Device Utilization %"=3,{counter}"Alloc system memory"=1}}'++ def fake_run(cmd, **kwargs):+ if cmd[0] == "ioreg":+ return _types.SimpleNamespace(stdout = block.encode())+ return _types.SimpleNamespace(stdout = "0\n")++ monkeypatch.setattr("subprocess.run", fake_run)+ assert _REAL_WIRED_CEILING() == expected++class TestAppleContextCap:"""The real ``_fit_context_to_vram`` against the reporter's M3 Pro case."""diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.pyindex 9e489c96ee5..f2affa033c6 100644--- a/studio/backend/tests/test_metal_explicit_context_guard.py+++ b/studio/backend/tests/test_metal_explicit_context_guard.py@@ -121,6 +121,8 @@ def _launch(kv_per_token = 1024,native = NATIVE,mmproj_bytes = 0,+ wired_bytes = 0,+ ckpt_bytes = 0,):"""Drive the real load_model with no GPU enumerated (the Metal condition).@@ -136,12 +138,19 @@ def _launch(helper's actual return contract -- its 4096 floor, and its habit of handing therequest straight back. ``budget_bytes`` / ``weights_bytes`` / ``kv_per_token`` /``native`` then place the model against the budget, and only matter with ``real_fit``.+ ``wired_bytes`` is the GPU wired headroom, 0 when unreadable. ``ckpt_bytes`` prices each+ SWA context checkpoint and advertises the flag, so ``--ctx-checkpoints`` is charged."""monkeypatch.setattr(LlamaCppBackend,"_apple_metal_memory_budget_bytes",staticmethod(lambda: budget_bytes if metal else 0),)+ monkeypatch.setattr(+ LlamaCppBackend,+ "_apple_metal_wired_ceiling_bytes",+ staticmethod(lambda: wired_bytes if metal else 0),+ )if paravirtual:import core.inference.llama_cpp as _llama_cppmonkeypatch.setattr(_llama_cpp, "_metal_device_is_paravirtual", lambda: True)@@ -150,7 +159,22 @@ def _launch(backend._get_gpu_free_memory = lambda _binary = None, **_kw: []backend._read_gguf_metadata = lambda _path: Nonebackend._can_estimate_kv = lambda: can_estimate_kv- backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * kv_per_token+ backend._estimate_kv_cache_bytes = (+ lambda ctx, *a, ctx_checkpoints = 0, **k: int(ctx) * kv_per_token+ + ctx_checkpoints * ckpt_bytes+ )+ if ckpt_bytes:+ _probe = LlamaCppBackend.probe_server_capabilities.__func__+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(+ lambda cls, binary = None: {+ **_probe(cls, binary),+ "ctx_checkpoints_flag": "--ctx-checkpoints",+ }+ ),+ )backend._compute_buffer_ctx_bytes = lambda *a, **k: 0if not real_fit:backend._fit_context_to_vram = lambda target, *a, **k: min(int(target), CEILING)@@ -953,6 +977,187 @@ class TestACpuPinnedProjectorOnUnifiedMemory:assert _ctx_values(captured["cmd"])[-1] == "40960"+_WIRED = 16 * 1024**3+++class TestTheGpuWiredLimitDecidesTheRefusal:+ """#9942: a 38 GiB load refused against ~32 GB free ran fine with the opt-out."""++ _ROOMY = dict(real_fit = True, budget_bytes = _BUDGET, weights_bytes = 1024**3, kv_per_token = _FAT_KV)++ def test_a_context_over_free_memory_but_under_the_limit_loads(self, tmp_path, monkeypatch):+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 8192, wired_bytes = _WIRED, **self._ROOMY)["cmd"]+ assert _ctx_values(cmd)[-1] == "8192"++ def test_it_loads_with_a_warning(self, tmp_path, monkeypatch):+ backend = _launch(tmp_path, monkeypatch, n_ctx = 8192, wired_bytes = _WIRED, **self._ROOMY)[+ "backend"+ ]+ assert "8,192" in backend.last_load_warning+ assert "loading anyway" in backend.last_load_warning++ def test_the_warning_rounds_need_up_and_free_down(self):+ message = LlamaCppBackend._metal_context_pressure_message(8192, 9 * 1024 + 1, 9 * 1024 - 1)+ assert "about 10 GB" in message and "the 8 GB" in message++ def test_a_context_under_free_memory_carries_no_warning(self, tmp_path, monkeypatch):+ backend = _launch(tmp_path, monkeypatch, n_ctx = 2048, wired_bytes = _WIRED, **self._ROOMY)[+ "backend"+ ]+ assert backend.last_load_warning is None++ def test_an_unreadable_limit_keeps_the_free_memory_refusal(self, tmp_path, monkeypatch):+ with pytest.raises(RuntimeError, match = "unified"):+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **self._ROOMY)++ def test_a_context_past_the_limit_is_refused(self, tmp_path, monkeypatch):+ with pytest.raises(RuntimeError, match = "unified"):+ _launch(tmp_path, monkeypatch, n_ctx = 32768, wired_bytes = _WIRED, **self._ROOMY)++ def test_the_refusal_names_what_the_limit_holds(self, tmp_path, monkeypatch):+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = 32768, wired_bytes = _WIRED, **self._ROOMY)+ named = _named_ceiling(str(excinfo.value))+ assert named > 8192+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, wired_bytes = _WIRED, **self._ROOMY)["cmd"]+ assert _ctx_values(cmd)[-1] == str(named)+ with pytest.raises(RuntimeError, match = "unified"):+ _launch(tmp_path, monkeypatch, n_ctx = named + 1024, wired_bytes = _WIRED, **self._ROOMY)++ def test_the_refusal_can_name_a_context_under_the_fit_floor(self, tmp_path, monkeypatch):+ tight = dict(self._ROOMY, budget_bytes = 8 * 1024**3)+ with pytest.raises(RuntimeError) as free_only:+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **tight)+ tight["wired_bytes"] = _BUDGET+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **tight)+ named = _named_ceiling(str(excinfo.value))+ assert _named_ceiling(str(free_only.value)) < named < 8192+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, **tight)["cmd"]+ assert _ctx_values(cmd)[-1] == str(named)++ def test_the_limit_can_refuse_the_context_auto_picks(self, tmp_path, monkeypatch):+ squeezed = dict(self._ROOMY, wired_bytes = 8 * 1024**3)+ auto_cmd = _launch(tmp_path, monkeypatch, n_ctx = 0, **squeezed)["cmd"]+ auto_ctx = int(_ctx_values(auto_cmd)[-1])+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = auto_ctx, **squeezed)+ assert _named_ceiling(str(excinfo.value)) < auto_ctx+ assert "leave it on Auto" not in str(excinfo.value)++ def test_auto_still_sizes_from_free_memory(self, tmp_path, monkeypatch):+ with_limit = _launch(tmp_path, monkeypatch, n_ctx = 0, wired_bytes = _WIRED, **self._ROOMY)+ without = _launch(tmp_path, monkeypatch, n_ctx = 0, **self._ROOMY)+ assert _ctx_values(with_limit["cmd"])[-1] == _ctx_values(without["cmd"])[-1]++ def test_weights_over_free_memory_are_priced_against_the_limit(self, tmp_path, monkeypatch):+ heavy = dict(+ real_fit = True,+ budget_bytes = _BUDGET,+ weights_bytes = 10 * 1024**3,+ kv_per_token = _FAT_KV,+ wired_bytes = 32 * 1024**3,+ )+ with pytest.raises(RuntimeError, match = "unified"):+ _launch(tmp_path, monkeypatch, n_ctx = 65536, **heavy)+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 8192, **heavy)["cmd"]+ assert _ctx_values(cmd)[-1] == "8192"++ def test_weights_alone_past_the_limit_are_not_refused(self, tmp_path, monkeypatch):+ cmd = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 32768,+ wired_bytes = _WIRED,+ real_fit = True,+ budget_bytes = _BUDGET,+ weights_bytes = 20 * 1024**3,+ kv_per_token = _FAT_KV,+ )["cmd"]+ assert _ctx_values(cmd)[-1] == "32768"++ def test_weights_past_the_limit_keep_the_free_memory_refusal(self, tmp_path, monkeypatch):+ """The headroom holds the GGUF but not the fitted weights, compute reserve included."""+ squeezed = dict(+ real_fit = True,+ budget_bytes = _BUDGET,+ weights_bytes = _TIGHT_WEIGHTS,+ kv_per_token = _FAT_KV,+ wired_bytes = _TIGHT_WEIGHTS + 1024**3,+ )+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = 32768, **squeezed)+ assert _named_ceiling(str(excinfo.value)) == _TIGHT_CEILING+ assert "leave it on Auto" in str(excinfo.value)+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 512, **squeezed)["cmd"]+ assert _ctx_values(cmd)[-1] == "512"++ def test_swa_checkpoints_count_against_the_limit(self, tmp_path, monkeypatch):+ ckpt = dict(+ real_fit = True,+ budget_bytes = 20 * 1024**3,+ weights_bytes = 1024**3,+ kv_per_token = 256 * 1024,+ ckpt_bytes = 1024**3,+ extra_args = ["--ctx-checkpoints", "8"],+ wired_bytes = 24 * 1024**3,+ )+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = 56000, **ckpt)+ named = _named_ceiling(str(excinfo.value))+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, **ckpt)["cmd"]+ assert _ctx_values(cmd)[-1] == str(named)+ with pytest.raises(RuntimeError, match = "unified"):+ _launch(tmp_path, monkeypatch, n_ctx = named + 1024, **ckpt)++ def test_swa_checkpoints_count_at_the_fit_floor(self, tmp_path, monkeypatch):+ swa = dict(+ real_fit = True,+ budget_bytes = 20 * 1024**3,+ weights_bytes = 1024**3,+ kv_per_token = 256 * 1024,+ extra_args = ["--ctx-checkpoints", "1"],+ wired_bytes = 24 * 1024**3,+ )+ with pytest.raises(RuntimeError) as excinfo:+ _launch(tmp_path, monkeypatch, n_ctx = 16384, ckpt_bytes = 17 * 1024**3, **swa)+ assert _named_ceiling(str(excinfo.value)) < 8192+ with pytest.raises(RuntimeError, match = "No context fits"):+ _launch(tmp_path, monkeypatch, n_ctx = 1024, ckpt_bytes = 18 * 1024**3, **swa)++ def test_nothing_fitting_free_memory_loads_under_the_limit(self, tmp_path, monkeypatch):+ cmd = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 4096,+ wired_bytes = _WIRED,+ **TestWhenNothingFitsAtAll.NOTHING_FITS,+ )["cmd"]+ assert _ctx_values(cmd)[-1] == "4096"++ def test_nothing_fitting_the_limit_is_refused(self, tmp_path, monkeypatch):+ with pytest.raises(RuntimeError, match = "No context fits"):+ _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 4096,+ wired_bytes = _BUDGET,+ **TestWhenNothingFitsAtAll.NOTHING_FITS,+ )++ def test_a_fixed_manual_layer_count_is_still_exempt(self, tmp_path, monkeypatch):+ cmd = _launch(+ tmp_path,+ monkeypatch,+ n_ctx = 32768,+ wired_bytes = _WIRED,+ gpu_memory_mode = "manual",+ gpu_layers = 20,+ **self._ROOMY,+ )["cmd"]+ assert _ctx_values(cmd)[-1] == "32768"++_GIB = 1024**3_MIB = 1024**2diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 073538a31be..2590b3cbc51 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -2588,7 +2588,7 @@ def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:def _read_apple_gpu_stats() -> Dict[str, Any]:- """macOS IORegistry AGX live stats (utilization_pct, vram_used_bytes, system-wide), or {} on failure. No sudo needed."""+ """macOS IORegistry AGX live stats (utilization_pct, and vram_used_bytes when reported; system-wide), or {} on failure. No sudo needed."""try:result = subprocess.run(["ioreg", "-r", "-c", "AGXAccelerator"],@@ -2607,10 +2607,10 @@ def _read_apple_gpu_stats() -> Dict[str, Any]:pairs = re.findall(r'"([^"]+)"=(\d+)', stats_str)stats = {k: int(v) for k, v in pairs}- return {- "utilization_pct": stats.get("Device Utilization %", 0),- "vram_used_bytes": stats.get("In use system memory", 0),- }+ out = {"utilization_pct": stats.get("Device Utilization %", 0)}+ if "In use system memory" in stats:+ out["vram_used_bytes"] = stats["In use system memory"]+ return out# ── CPU frequency on Apple Silicon ──────────────────────────────────────────diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsxindex 7960e3e310b..ca771dd0a05 100644--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx@@ -1156,11 +1156,11 @@ export function ChatSettingsPanel({<p className="text-ui-11 text-amber-500">{isUnifiedMemory ? (<>- Context length exceeds what fits in unified memory (- {maxContextLength?.toLocaleString()} tokens). The GPU- and the rest of the system share one pool here, so there- is nothing to offload to. Lower the context, leave it on- Auto, or set the KV cache to q8_0.+ Context length is above Studio's free-memory estimate+ ({maxContextLength?.toLocaleString()} tokens). It may+ still load, but macOS may have to compress or swap other+ apps and generation may slow down. Lower the context,+ leave it on Auto, or set the KV cache to q8_0.</>) : (<>diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 4755d1581ed..8bec2615fee 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -3303,10 +3303,10 @@ export function ModelConfigPage({<p className="text-ui-11 text-amber-500">{isAppleUnifiedMemory ? (<>- Exceeds what fits in unified memory (- {loadedMaxContextLength.toLocaleString()} tokens). The- GPU and the rest of the system share one pool here, so- there is nothing to offload to.+ Above Studio's free-memory estimate (+ {loadedMaxContextLength.toLocaleString()} tokens). It+ may still load, but macOS may have to compress or swap+ other apps and generation may slow down.</>) : (<>