Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: refuse a hand-set Metal context only past the GPU wired limit (#10804)

Long Yixinge8c5187b22d5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index dede00d3d92..ddc125afdbd 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3775,6 +3775,9 @@ _CTX_FIT_VRAM_FRACTION = 0.97
# 0.85 MLX uses in mlx_inference.py (_configure_memory_limits); not kept in sync.
_APPLE_UNIFIED_MEMORY_FRACTION = 0.85
+# The 5% of wired headroom held back absorbs error in the footprint estimate.
+_APPLE_WIRED_CEILING_FRACTION = 0.95
+
# _fit_context_to_vram's floor: the shortest context the search will settle for, so a
# value equal to it is that floor rather than a measurement. The Metal branch re-prices
# from _FIT_FLOOR_MIN_CTX (the search's 256 alignment step) before trusting it.
@@ -12372,6 +12375,45 @@ class LlamaCppBackend:
)
return unmapped
+ @staticmethod
+ def _apple_metal_wired_ceiling_bytes() -> int:
+ """Wired GPU memory a Metal load may still take, system-wide; 0 if unknown or used up."""
+ from utils.hardware import is_apple_silicon
+
+ if not is_apple_silicon():
+ return 0
+ try:
+ probe = subprocess.run(
+ ["sysctl", "-n", "iogpu.wired_limit_mb"],
+ capture_output = True,
+ text = True,
+ timeout = 5,
+ encoding = "utf-8",
+ errors = "replace",
+ )
+ cap_bytes = int(probe.stdout.strip()) * 1024 * 1024
+ except Exception:
+ return 0
+ # 0 means the kernel default, Metal's recommended working set.
+ if cap_bytes <= 0:
+ try:
+ import mlx.core as mx
+ from utils.hardware.hardware import _mlx_device_info
+ if mx.metal.is_available():
+ cap_bytes = int(
+ _mlx_device_info(mx).get("max_recommended_working_set_size") or 0
+ )
+ except Exception:
+ cap_bytes = 0
+ if cap_bytes <= 0:
+ return 0
+ from utils.hardware.hardware import _read_apple_gpu_stats
+
+ in_use = _read_apple_gpu_stats().get("vram_used_bytes")
+ if in_use is None:
+ return 0
+ return max(0, int((cap_bytes - in_use) * _APPLE_WIRED_CEILING_FRACTION))
+
@staticmethod
def _rocm_arch_gate_keep(
binary: Optional[str], torch_mod, for_llama_server: bool
@@ -15031,6 +15073,7 @@ class LlamaCppBackend:
cache_type_kv: Optional[str] = None,
*,
nothing_fits: bool = False,
+ wired_limit: bool = False,
) -> Optional[str]:
"""Refusal when a hand-set context exceeds what unified memory holds (else None).
@@ -15064,6 +15107,9 @@ class LlamaCppBackend:
branch falls back to when KV cannot be sized is a guess, and refusing against it
would block contexts that load fine today.
+ ``wired_limit`` marks a ceiling priced under the GPU wired headroom, which Auto does not
+ size from, so the advice leaves Auto out.
+
UNSLOTH_ALLOW_METAL_CTX_OVERCOMMIT=1 abstains, matching the host-offload opt-out,
though the failure mode it re-enables is the whole machine rather than one app.
"""
@@ -15092,6 +15138,7 @@ class LlamaCppBackend:
if (cache_type_kv or "f16").strip().lower() in ("f16", "fp16", "")
else ""
)
+ auto_advice = "" if wired_limit else "leave it on Auto, "
if nothing_fits:
return (
"No context fits in this Mac's unified memory with this model. The weights "
@@ -15109,11 +15156,26 @@ class LlamaCppBackend:
"tokens. The GPU and the rest of the system share one pool here, so there is "
"nothing to offload to, and the load would bring the machine down instead of "
f"reporting an error. Lower the context to {max_available_ctx:,} or less, "
- "leave it on Auto, or use a more quantized GGUF."
+ f"{auto_advice}or use a more quantized GGUF."
f"{kv_hint} Set "
f"{LlamaCppBackend.METAL_CTX_OVERCOMMIT_ENV}=1 to load it anyway."
)
+ @staticmethod
+ def _metal_context_pressure_message(
+ requested_ctx: int, need_mib: float, free_budget_mib: float
+ ) -> str:
+ # need up, free down, so the printed pair cannot round into a tie
+ need_gb = math.ceil(need_mib / 1024)
+ free_gb = math.floor(max(0.0, free_budget_mib) / 1024)
+ return (
+ f"A context of {requested_ctx:,} tokens needs about {need_gb} GB of unified "
+ f"memory, more than the {free_gb} GB Studio budgets from the memory free right "
+ "now. It fits under the GPU's wired memory limit, so it is loading anyway, but "
+ "macOS may have to compress or swap other apps to make room and generation may "
+ "slow down. Lower the context or free memory to avoid that."
+ )
+
@staticmethod
def _cuda_context_overcommit_notice(
requested_ctx: int,
@@ -23970,6 +24032,7 @@ class LlamaCppBackend:
# the handler's own log line on purpose: test_tp_vision_regression
# string-searches this function's source for it to check ordering.)
_metal_ctx_refusal: Optional[str] = None
+ _metal_ctx_warning: Optional[str] = None
# A discounted Metal ceiling must launch the measured full-offload placement.
_metal_full_offload_pinned = False
# The fit and launch must use the same Model Memory snapshot.
@@ -25667,10 +25730,7 @@ class LlamaCppBackend:
elif _apple_budget_mib > 0 and effective_ctx > 0:
# No GPU on Metal: the branches above are skipped and the context
# stays at native, over-committing unified memory (#5118, #6529).
- # Cap with the same fit math; Auto shrinks to the cap, an explicit
- # request above it is refused. "--fit on" stays a backstop but not
- # one this can lean on: llama.cpp sizes its reduction from
- # ggml-metal's free-memory report, blind to Unsloth's own footprint
+ # Not "--fit on" alone: llama.cpp sizes from ggml-metal's free-memory report, blind to Unsloth's own footprint
# and the wired limit. See _metal_context_overcommit_message.
native_ctx_for_cap = self._context_length or effective_ctx
# This arm's floor, which is _FIT_MIN_CTX only while the child's
@@ -25725,10 +25785,14 @@ class LlamaCppBackend:
_apple_model_size_fit / (1024**3),
)
- def _apple_ctx_fit(target: int, min_ctx: int) -> int:
+ def _apple_ctx_fit(
+ target: int,
+ min_ctx: int,
+ budget_mib: Optional[int] = None,
+ ) -> int:
return self._fit_context_to_vram(
target,
- _apple_fit_budget_mib,
+ _apple_fit_budget_mib if budget_mib is None else budget_mib,
_apple_model_size_fit,
cache_type_kv,
min_ctx = min_ctx,
@@ -25746,10 +25810,10 @@ class LlamaCppBackend:
ctx_checkpoints = _fit_ctx_checkpoints,
)
- def _apple_footprint_mib(ctx: int) -> float:
+ def _apple_footprint_mib(ctx: int, ctx_checkpoints: int = 0) -> float:
return (
_apple_model_size_fit
- + _kv_bytes(ctx)
+ + _kv_bytes(ctx, ctx_checkpoints)
+ _mtp_bytes(ctx)
+ _cc_bytes(ctx)
) / (1024 * 1024)
@@ -25824,7 +25888,7 @@ class LlamaCppBackend:
effective_ctx, cache_type_kv
)
elif (
- (_apple_measured_ceiling is not None or _apple_nothing_fits)
+ self._can_estimate_kv()
and not _caller_owns_budget
and not _paravirtual_cpu_forced
):
@@ -25887,12 +25951,46 @@ class LlamaCppBackend:
max_available_ctx = max(
max_available_ctx, _apple_measured_ceiling
)
- _metal_ctx_refusal = self._metal_context_overcommit_message(
- effective_ctx,
- _apple_measured_ceiling or 0,
- cache_type_kv,
- nothing_fits = _apple_nothing_fits,
+ # Past free memory macOS swaps; past the wired limit the machine panics.
+ _apple_wired_mib = int(
+ self._apple_metal_wired_ceiling_bytes()
+ // (1024 * 1024)
+ * max(0.0, 1.0 - _flat_mtp_reserve)
)
+ if (
+ _apple_wired_mib <= 0
+ or _apple_model_size_fit / (1024 * 1024) > _apple_wired_mib
+ ):
+ _metal_ctx_refusal = self._metal_context_overcommit_message(
+ effective_ctx,
+ _apple_measured_ceiling or 0,
+ cache_type_kv,
+ nothing_fits = _apple_nothing_fits,
+ )
+ else:
+ # Priced like the fit (SWA checkpoints included) so admission matches the refusal ceiling.
+ _requested_mib = _apple_footprint_mib(
+ effective_ctx, _fit_ctx_checkpoints
+ )
+ if _requested_mib > _apple_wired_mib:
+ _wired_ctx = _apple_ctx_fit(
+ effective_ctx, _FIT_FLOOR_MIN_CTX, _apple_wired_mib
+ )
+ _wired_fits = (
+ _apple_footprint_mib(_wired_ctx, _fit_ctx_checkpoints)
+ <= _apple_wired_mib
+ )
+ _metal_ctx_refusal = self._metal_context_overcommit_message(
+ effective_ctx,
+ _wired_ctx if _wired_fits else 0,
+ cache_type_kv,
+ nothing_fits = not _wired_fits,
+ wired_limit = True,
+ )
+ elif _requested_mib > _apple_fit_budget_mib:
+ _metal_ctx_warning = self._metal_context_pressure_message(
+ effective_ctx, _requested_mib, _apple_fit_budget_mib
+ )
# Unmeasured floors still rely on llama.cpp's fitter.
_metal_full_offload_pinned = bool(
_apple_host_embd and _apple_measured_ceiling is not None
@@ -26433,6 +26531,7 @@ class LlamaCppBackend:
# Vulkan and APU checks, which describe hardware this branch has ruled out.
if _metal_ctx_refusal:
raise RuntimeError(_metal_ctx_refusal)
+ self._record_load_warning(_metal_ctx_warning)
# An unenumerated explicit Vulkan ordinal can't be pinned; fail loudly
# instead of fitting onto an unselected device. Clear the raw selection
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 528ac6a9a69..ad270a128cf 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -424,6 +424,15 @@ def _hf_cache_is_empty(_empty_hf_hub_cache, monkeypatch):
monkeypatch.setattr(constants, "HF_HUB_CACHE", _empty_hf_hub_cache)
[email protected](autouse = True)
+def _no_live_metal_wired_ceiling(monkeypatch):
+ """Keep Metal context verdicts off the host's live GPU memory."""
+ from core.inference.llama_cpp import LlamaCppBackend
+ monkeypatch.setattr(
+ LlamaCppBackend, "_apple_metal_wired_ceiling_bytes", staticmethod(lambda: 0)
+ )
+
+
@pytest.fixture(autouse = True)
def _no_leftover_generation_account(monkeypatch):
"""Clear the media-generation owner, which is a process global no route ever resets.
diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py
index 341ff2c90eb..7856491e45e 100644
--- a/studio/backend/tests/test_llama_cpp_context_fit.py
+++ b/studio/backend/tests/test_llama_cpp_context_fit.py
@@ -78,6 +78,7 @@ except ImportError:
from core.inference.llama_cpp import (
_AUTO_OFFLOAD_CTX,
_APPLE_UNIFIED_MEMORY_FRACTION,
+ _APPLE_WIRED_CEILING_FRACTION,
_CTX_FIT_VRAM_FRACTION,
LlamaCppBackend,
classify_gpu_offload_lines,
@@ -827,6 +828,89 @@ class TestAppleUnifiedMemoryBudget:
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == 0
+# Unpatched: conftest pins the probe to 0.
+_REAL_WIRED_CEILING = LlamaCppBackend.__dict__["_apple_metal_wired_ceiling_bytes"].__func__
+
+
+def _install_wired_probes(monkeypatch, *, sysctl_mb, working_set, in_use):
+ """``sysctl_mb`` or ``in_use`` of None is an unreadable probe."""
+ from core.inference import llama_cpp as _llama_cpp
+ from utils.hardware import hardware as _hardware
+
+ _force_apple(monkeypatch)
+ _install_fake_mlx(monkeypatch, working_set)
+ sysctl_out = "" if sysctl_mb is None else f"{sysctl_mb}\n"
+ monkeypatch.setattr(
+ _llama_cpp.subprocess,
+ "run",
+ lambda *a, **k: _types.SimpleNamespace(stdout = sysctl_out),
+ )
+ monkeypatch.setattr(
+ _hardware,
+ "_read_apple_gpu_stats",
+ lambda: {} if in_use is None else {"vram_used_bytes": in_use},
+ )
+
+
+class TestAppleWiredCeiling:
+ def test_zero_off_apple_silicon(self, monkeypatch):
+ import platform as _platform
+
+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
+ monkeypatch.setattr(_platform, "system", lambda: "Linux")
+ monkeypatch.setattr(_platform, "machine", lambda: "x86_64")
+ assert _REAL_WIRED_CEILING() == 0
+
+ def test_the_working_set_is_the_limit_by_default(self, monkeypatch):
+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
+ assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)
+
+ def test_older_mlx_reads_the_legacy_alias(self, monkeypatch):
+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
+ mlx_core = sys.modules["mlx.core"]
+ mlx_core.metal.device_info = mlx_core.device_info
+ monkeypatch.delattr(mlx_core, "device_info")
+ assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)
+
+ def test_a_set_sysctl_limit_wins(self, monkeypatch):
+ _install_wired_probes(
+ monkeypatch, sysctl_mb = 56 * 1024, working_set = 48 * GIB, in_use = 2 * GIB
+ )
+ assert _REAL_WIRED_CEILING() == int(54 * GIB * _APPLE_WIRED_CEILING_FRACTION)
+
+ def test_unread_gpu_memory_is_unresolved_not_zero(self, monkeypatch):
+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = None)
+ assert _REAL_WIRED_CEILING() == 0
+
+ def test_unread_sysctl_is_unresolved_not_the_default(self, monkeypatch):
+ _install_wired_probes(monkeypatch, sysctl_mb = None, working_set = 48 * GIB, in_use = 2 * GIB)
+ assert _REAL_WIRED_CEILING() == 0
+
+ def test_no_readable_limit_is_unresolved(self, monkeypatch):
+ _install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 0, in_use = 2 * GIB)
+ assert _REAL_WIRED_CEILING() == 0
+
+ @pytest.mark.parametrize(
+ "counter,expected",
+ [
+ ("", 0),
+ ('"In use system memory"=2147483648,', int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)),
+ ],
+ )
+ def test_ioreg_resolves_only_with_the_in_use_counter(self, monkeypatch, counter, expected):
+ _force_apple(monkeypatch)
+ _install_fake_mlx(monkeypatch, 48 * GIB)
+ block = f'"PerformanceStatistics" = {{"Device Utilization %"=3,{counter}"Alloc system memory"=1}}'
+
+ def fake_run(cmd, **kwargs):
+ if cmd[0] == "ioreg":
+ return _types.SimpleNamespace(stdout = block.encode())
+ return _types.SimpleNamespace(stdout = "0\n")
+
+ monkeypatch.setattr("subprocess.run", fake_run)
+ assert _REAL_WIRED_CEILING() == expected
+
+
class TestAppleContextCap:
"""The real ``_fit_context_to_vram`` against the reporter's M3 Pro case."""
diff --git a/studio/backend/tests/test_metal_explicit_context_guard.py b/studio/backend/tests/test_metal_explicit_context_guard.py
index 9e489c96ee5..f2affa033c6 100644
--- a/studio/backend/tests/test_metal_explicit_context_guard.py
+++ b/studio/backend/tests/test_metal_explicit_context_guard.py
@@ -121,6 +121,8 @@ def _launch(
kv_per_token = 1024,
native = NATIVE,
mmproj_bytes = 0,
+ wired_bytes = 0,
+ ckpt_bytes = 0,
):
"""Drive the real load_model with no GPU enumerated (the Metal condition).
@@ -136,12 +138,19 @@ def _launch(
helper's actual return contract -- its 4096 floor, and its habit of handing the
request straight back. ``budget_bytes`` / ``weights_bytes`` / ``kv_per_token`` /
``native`` then place the model against the budget, and only matter with ``real_fit``.
+ ``wired_bytes`` is the GPU wired headroom, 0 when unreadable. ``ckpt_bytes`` prices each
+ SWA context checkpoint and advertises the flag, so ``--ctx-checkpoints`` is charged.
"""
monkeypatch.setattr(
LlamaCppBackend,
"_apple_metal_memory_budget_bytes",
staticmethod(lambda: budget_bytes if metal else 0),
)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_apple_metal_wired_ceiling_bytes",
+ staticmethod(lambda: wired_bytes if metal else 0),
+ )
if paravirtual:
import core.inference.llama_cpp as _llama_cpp
monkeypatch.setattr(_llama_cpp, "_metal_device_is_paravirtual", lambda: True)
@@ -150,7 +159,22 @@ def _launch(
backend._get_gpu_free_memory = lambda _binary = None, **_kw: []
backend._read_gguf_metadata = lambda _path: None
backend._can_estimate_kv = lambda: can_estimate_kv
- backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * kv_per_token
+ backend._estimate_kv_cache_bytes = (
+ lambda ctx, *a, ctx_checkpoints = 0, **k: int(ctx) * kv_per_token
+ + ctx_checkpoints * ckpt_bytes
+ )
+ if ckpt_bytes:
+ _probe = LlamaCppBackend.probe_server_capabilities.__func__
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(
+ lambda cls, binary = None: {
+ **_probe(cls, binary),
+ "ctx_checkpoints_flag": "--ctx-checkpoints",
+ }
+ ),
+ )
backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
if not real_fit:
backend._fit_context_to_vram = lambda target, *a, **k: min(int(target), CEILING)
@@ -953,6 +977,187 @@ class TestACpuPinnedProjectorOnUnifiedMemory:
assert _ctx_values(captured["cmd"])[-1] == "40960"
+_WIRED = 16 * 1024**3
+
+
+class TestTheGpuWiredLimitDecidesTheRefusal:
+ """#9942: a 38 GiB load refused against ~32 GB free ran fine with the opt-out."""
+
+ _ROOMY = dict(real_fit = True, budget_bytes = _BUDGET, weights_bytes = 1024**3, kv_per_token = _FAT_KV)
+
+ def test_a_context_over_free_memory_but_under_the_limit_loads(self, tmp_path, monkeypatch):
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 8192, wired_bytes = _WIRED, **self._ROOMY)["cmd"]
+ assert _ctx_values(cmd)[-1] == "8192"
+
+ def test_it_loads_with_a_warning(self, tmp_path, monkeypatch):
+ backend = _launch(tmp_path, monkeypatch, n_ctx = 8192, wired_bytes = _WIRED, **self._ROOMY)[
+ "backend"
+ ]
+ assert "8,192" in backend.last_load_warning
+ assert "loading anyway" in backend.last_load_warning
+
+ def test_the_warning_rounds_need_up_and_free_down(self):
+ message = LlamaCppBackend._metal_context_pressure_message(8192, 9 * 1024 + 1, 9 * 1024 - 1)
+ assert "about 10 GB" in message and "the 8 GB" in message
+
+ def test_a_context_under_free_memory_carries_no_warning(self, tmp_path, monkeypatch):
+ backend = _launch(tmp_path, monkeypatch, n_ctx = 2048, wired_bytes = _WIRED, **self._ROOMY)[
+ "backend"
+ ]
+ assert backend.last_load_warning is None
+
+ def test_an_unreadable_limit_keeps_the_free_memory_refusal(self, tmp_path, monkeypatch):
+ with pytest.raises(RuntimeError, match = "unified"):
+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **self._ROOMY)
+
+ def test_a_context_past_the_limit_is_refused(self, tmp_path, monkeypatch):
+ with pytest.raises(RuntimeError, match = "unified"):
+ _launch(tmp_path, monkeypatch, n_ctx = 32768, wired_bytes = _WIRED, **self._ROOMY)
+
+ def test_the_refusal_names_what_the_limit_holds(self, tmp_path, monkeypatch):
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = 32768, wired_bytes = _WIRED, **self._ROOMY)
+ named = _named_ceiling(str(excinfo.value))
+ assert named > 8192
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, wired_bytes = _WIRED, **self._ROOMY)["cmd"]
+ assert _ctx_values(cmd)[-1] == str(named)
+ with pytest.raises(RuntimeError, match = "unified"):
+ _launch(tmp_path, monkeypatch, n_ctx = named + 1024, wired_bytes = _WIRED, **self._ROOMY)
+
+ def test_the_refusal_can_name_a_context_under_the_fit_floor(self, tmp_path, monkeypatch):
+ tight = dict(self._ROOMY, budget_bytes = 8 * 1024**3)
+ with pytest.raises(RuntimeError) as free_only:
+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **tight)
+ tight["wired_bytes"] = _BUDGET
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = 8192, **tight)
+ named = _named_ceiling(str(excinfo.value))
+ assert _named_ceiling(str(free_only.value)) < named < 8192
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, **tight)["cmd"]
+ assert _ctx_values(cmd)[-1] == str(named)
+
+ def test_the_limit_can_refuse_the_context_auto_picks(self, tmp_path, monkeypatch):
+ squeezed = dict(self._ROOMY, wired_bytes = 8 * 1024**3)
+ auto_cmd = _launch(tmp_path, monkeypatch, n_ctx = 0, **squeezed)["cmd"]
+ auto_ctx = int(_ctx_values(auto_cmd)[-1])
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = auto_ctx, **squeezed)
+ assert _named_ceiling(str(excinfo.value)) < auto_ctx
+ assert "leave it on Auto" not in str(excinfo.value)
+
+ def test_auto_still_sizes_from_free_memory(self, tmp_path, monkeypatch):
+ with_limit = _launch(tmp_path, monkeypatch, n_ctx = 0, wired_bytes = _WIRED, **self._ROOMY)
+ without = _launch(tmp_path, monkeypatch, n_ctx = 0, **self._ROOMY)
+ assert _ctx_values(with_limit["cmd"])[-1] == _ctx_values(without["cmd"])[-1]
+
+ def test_weights_over_free_memory_are_priced_against_the_limit(self, tmp_path, monkeypatch):
+ heavy = dict(
+ real_fit = True,
+ budget_bytes = _BUDGET,
+ weights_bytes = 10 * 1024**3,
+ kv_per_token = _FAT_KV,
+ wired_bytes = 32 * 1024**3,
+ )
+ with pytest.raises(RuntimeError, match = "unified"):
+ _launch(tmp_path, monkeypatch, n_ctx = 65536, **heavy)
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 8192, **heavy)["cmd"]
+ assert _ctx_values(cmd)[-1] == "8192"
+
+ def test_weights_alone_past_the_limit_are_not_refused(self, tmp_path, monkeypatch):
+ cmd = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 32768,
+ wired_bytes = _WIRED,
+ real_fit = True,
+ budget_bytes = _BUDGET,
+ weights_bytes = 20 * 1024**3,
+ kv_per_token = _FAT_KV,
+ )["cmd"]
+ assert _ctx_values(cmd)[-1] == "32768"
+
+ def test_weights_past_the_limit_keep_the_free_memory_refusal(self, tmp_path, monkeypatch):
+ """The headroom holds the GGUF but not the fitted weights, compute reserve included."""
+ squeezed = dict(
+ real_fit = True,
+ budget_bytes = _BUDGET,
+ weights_bytes = _TIGHT_WEIGHTS,
+ kv_per_token = _FAT_KV,
+ wired_bytes = _TIGHT_WEIGHTS + 1024**3,
+ )
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = 32768, **squeezed)
+ assert _named_ceiling(str(excinfo.value)) == _TIGHT_CEILING
+ assert "leave it on Auto" in str(excinfo.value)
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = 512, **squeezed)["cmd"]
+ assert _ctx_values(cmd)[-1] == "512"
+
+ def test_swa_checkpoints_count_against_the_limit(self, tmp_path, monkeypatch):
+ ckpt = dict(
+ real_fit = True,
+ budget_bytes = 20 * 1024**3,
+ weights_bytes = 1024**3,
+ kv_per_token = 256 * 1024,
+ ckpt_bytes = 1024**3,
+ extra_args = ["--ctx-checkpoints", "8"],
+ wired_bytes = 24 * 1024**3,
+ )
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = 56000, **ckpt)
+ named = _named_ceiling(str(excinfo.value))
+ cmd = _launch(tmp_path, monkeypatch, n_ctx = named, **ckpt)["cmd"]
+ assert _ctx_values(cmd)[-1] == str(named)
+ with pytest.raises(RuntimeError, match = "unified"):
+ _launch(tmp_path, monkeypatch, n_ctx = named + 1024, **ckpt)
+
+ def test_swa_checkpoints_count_at_the_fit_floor(self, tmp_path, monkeypatch):
+ swa = dict(
+ real_fit = True,
+ budget_bytes = 20 * 1024**3,
+ weights_bytes = 1024**3,
+ kv_per_token = 256 * 1024,
+ extra_args = ["--ctx-checkpoints", "1"],
+ wired_bytes = 24 * 1024**3,
+ )
+ with pytest.raises(RuntimeError) as excinfo:
+ _launch(tmp_path, monkeypatch, n_ctx = 16384, ckpt_bytes = 17 * 1024**3, **swa)
+ assert _named_ceiling(str(excinfo.value)) < 8192
+ with pytest.raises(RuntimeError, match = "No context fits"):
+ _launch(tmp_path, monkeypatch, n_ctx = 1024, ckpt_bytes = 18 * 1024**3, **swa)
+
+ def test_nothing_fitting_free_memory_loads_under_the_limit(self, tmp_path, monkeypatch):
+ cmd = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 4096,
+ wired_bytes = _WIRED,
+ **TestWhenNothingFitsAtAll.NOTHING_FITS,
+ )["cmd"]
+ assert _ctx_values(cmd)[-1] == "4096"
+
+ def test_nothing_fitting_the_limit_is_refused(self, tmp_path, monkeypatch):
+ with pytest.raises(RuntimeError, match = "No context fits"):
+ _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 4096,
+ wired_bytes = _BUDGET,
+ **TestWhenNothingFitsAtAll.NOTHING_FITS,
+ )
+
+ def test_a_fixed_manual_layer_count_is_still_exempt(self, tmp_path, monkeypatch):
+ cmd = _launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 32768,
+ wired_bytes = _WIRED,
+ gpu_memory_mode = "manual",
+ gpu_layers = 20,
+ **self._ROOMY,
+ )["cmd"]
+ assert _ctx_values(cmd)[-1] == "32768"
+
+
_GIB = 1024**3
_MIB = 1024**2
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 073538a31be..2590b3cbc51 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -2588,7 +2588,7 @@ def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:
def _read_apple_gpu_stats() -> Dict[str, Any]:
- """macOS IORegistry AGX live stats (utilization_pct, vram_used_bytes, system-wide), or {} on failure. No sudo needed."""
+ """macOS IORegistry AGX live stats (utilization_pct, and vram_used_bytes when reported; system-wide), or {} on failure. No sudo needed."""
try:
result = subprocess.run(
["ioreg", "-r", "-c", "AGXAccelerator"],
@@ -2607,10 +2607,10 @@ def _read_apple_gpu_stats() -> Dict[str, Any]:
pairs = re.findall(r'"([^"]+)"=(\d+)', stats_str)
stats = {k: int(v) for k, v in pairs}
- return {
- "utilization_pct": stats.get("Device Utilization %", 0),
- "vram_used_bytes": stats.get("In use system memory", 0),
- }
+ out = {"utilization_pct": stats.get("Device Utilization %", 0)}
+ if "In use system memory" in stats:
+ out["vram_used_bytes"] = stats["In use system memory"]
+ return out
# ── CPU frequency on Apple Silicon ──────────────────────────────────────────
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
index 7960e3e310b..ca771dd0a05 100644
--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx
+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
@@ -1156,11 +1156,11 @@ export function ChatSettingsPanel({
<p className="text-ui-11 text-amber-500">
{isUnifiedMemory ? (
<>
- Context length exceeds what fits in unified memory (
- {maxContextLength?.toLocaleString()} tokens). The GPU
- and the rest of the system share one pool here, so there
- is nothing to offload to. Lower the context, leave it on
- Auto, or set the KV cache to q8_0.
+ Context length is above Studio&apos;s free-memory estimate
+ ({maxContextLength?.toLocaleString()} tokens). It may
+ still load, but macOS may have to compress or swap other
+ apps and generation may slow down. Lower the context,
+ leave it on Auto, or set the KV cache to q8_0.
</>
) : (
<>
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 4755d1581ed..8bec2615fee 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -3303,10 +3303,10 @@ export function ModelConfigPage({
<p className="text-ui-11 text-amber-500">
{isAppleUnifiedMemory ? (
<>
- Exceeds what fits in unified memory (
- {loadedMaxContextLength.toLocaleString()} tokens). The
- GPU and the rest of the system share one pool here, so
- there is nothing to offload to.
+ Above Studio&apos;s free-memory estimate (
+ {loadedMaxContextLength.toLocaleString()} tokens). It
+ may still load, but macOS may have to compress or swap
+ other apps and generation may slow down.
</>
) : (
<>