Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep more VRAM headroom on Windows CUDA, and say when a hand-set context does not fit (#11368)
Daniel Handac0dd0ed660
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 729fa714c3f..dede00d3d92 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3848,8 +3848,13 @@ _VRAM_FLOOR_RESERVE_MIB = 512.0# ... default: 1024"): what the fallback fitter leaves when we pass nothing._LLAMA_FIT_TARGET_DEFAULT_MIB = 1024.0+# Windows + discrete NVIDIA: since driver 536.40 WDDM serves VRAM overflow from host RAM+# instead of failing (unslothai/unsloth#11349), so match llama.cpp's own fit margin.+_WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB = _LLAMA_FIT_TARGET_DEFAULT_MIB+_WINDOWS_SYSMEM_FALLBACK_MAX_FRACTION = 0.125-def _vram_reserve_floor_mib(total_mib: float) -> float:++def _vram_reserve_floor_mib(total_mib: float, *, sysmem_fallback: bool = False) -> float:"""Smallest margin a card keeps: 512 MiB, or the default's own reserve if smaller.Capping it at the default keeps the budget monotonic. A flat 512 MiB would@@ -3859,8 +3864,20 @@ def _vram_reserve_floor_mib(total_mib: float) -> float:context, the opposite of what the control promises. Capped, the reserve neverrises with the fraction, and at or below the default the percentage term alwayswins, so an unset budget is untouched on every card size.++ ``sysmem_fallback`` raises the floor (flat in ``frac``, so still monotonic) on hosts+ whose driver spills an overshoot into system RAM; it overrides a 100% budget on purpose."""- return min(_VRAM_FLOOR_RESERVE_MIB, (1.0 - _CTX_FIT_VRAM_FRACTION) * total_mib)+ floor = min(_VRAM_FLOOR_RESERVE_MIB, (1.0 - _CTX_FIT_VRAM_FRACTION) * total_mib)+ if not sysmem_fallback:+ return floor+ return max(+ floor,+ min(+ _WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB,+ _WINDOWS_SYSMEM_FALLBACK_MAX_FRACTION * total_mib,+ ),+ )def _vram_usable_mib(@@ -3869,6 +3886,7 @@ def _vram_usable_mib(frac: float,*,pooled: bool = False,+ sysmem_fallback: bool = False,) -> float:"""Free MiB one card offers a load at ``frac``, unclamped.@@ -3877,13 +3895,21 @@ def _vram_usable_mib(is one device, including a probe reporting free memory with no total, whichstill has to keep a margin: there the free reading is the only scale available,and it agrees with the known-total form at the default.++ ``sysmem_fallback``: see ``LlamaCppBackend._sysmem_fallback_risk()``; ``pooled`` ignores it."""if total_mib and total_mib > 0:- reserve = max((1.0 - frac) * total_mib, _vram_reserve_floor_mib(total_mib))+ reserve = max(+ (1.0 - frac) * total_mib,+ _vram_reserve_floor_mib(total_mib, sysmem_fallback = sysmem_fallback),+ )return free_mib - reserveif pooled:return free_mib * frac- return min(free_mib * frac, free_mib - _vram_reserve_floor_mib(free_mib))+ return min(+ free_mib * frac,+ free_mib - _vram_reserve_floor_mib(free_mib, sysmem_fallback = sysmem_fallback),+ )def _active_vram_fraction() -> float:@@ -9977,6 +10003,38 @@ class LlamaCppBackend:backends = LlamaCppBackend._installed_ggml_backends(binary)return "vulkan" in backends and not backends.intersection({"cuda", "hip"})+ _SYSMEM_FALLBACK_RISK: dict[Optional[str], bool] = {}++ @staticmethod+ def _sysmem_fallback_risk(binary: Optional[str] = None) -> bool:+ """True when an over-budget allocation would be absorbed by host RAM, not refused.++ Windows + CUDA build only. Discreteness needs no probe: integrated CUDA parts are+ Linux-only, and get_device_properties would pin a ~700 MiB CUDA context.+ """+ if sys.platform != "win32":+ return False+ if binary is None:+ try:+ binary = LlamaCppBackend._find_llama_server_binary()+ except Exception: # noqa: BLE001 - classify with whatever the lib lookup resolves+ binary = None+ # Keyed by resolved path and build revision: runtimes switch, updates swap in place.+ key = LlamaCppBackend._binary_revision(binary) or binary+ cached = LlamaCppBackend._SYSMEM_FALLBACK_RISK.get(key)+ if cached is not None:+ return cached+ try:+ backends = LlamaCppBackend._installed_ggml_backends(binary)+ except Exception as e: # noqa: BLE001 - an unreadable lib dir is not a load failure+ logger.debug("sysmem-fallback classification failed, keeping the base budget: %s", e)+ return False # not cached: a transient read error must not pin False forever+ if not backends:+ return False # an unreadable dir scans empty; same rule, not cached+ risk = "cuda" in backends+ LlamaCppBackend._SYSMEM_FALLBACK_RISK[key] = risk+ return risk+@staticmethoddef _active_gpu_visibility_mask() -> Optional[str]:"""The raw visibility mask in force (HIP, then ROCR, then CUDA on ROCm; CUDA@@ -15056,6 +15114,43 @@ class LlamaCppBackend:f"{LlamaCppBackend.METAL_CTX_OVERCOMMIT_ENV}=1 to load it anyway.")+ @staticmethod+ def _cuda_context_overcommit_notice(+ requested_ctx: int,+ max_available_ctx: int,+ cache_type_kv: Optional[str] = None,+ *,+ quantised_ctx_fits: bool = False,+ ) -> Optional[str]:+ """Advisory when a hand-set context exceeds what a discrete GPU holds (else None).++ Metal refuses this (shared wired pool panics the machine); a discrete GPU offloads+ layers to the CPU instead, so only warn.+ ``quantised_ctx_fits`` is the caller's q8_0 pricing verdict, not a guess.+ """+ if requested_ctx <= 0 or max_available_ctx <= 0:+ return None+ if requested_ctx <= max_available_ctx:+ return None+ kv_hint = ""+ if quantised_ctx_fits and (cache_type_kv or "f16").strip().lower() in (+ "f16",+ "fp16",+ "",+ ):+ kv_hint = " Setting the KV cache to q8_0 makes this context fit without shortening it."+ # Fires only after a --fit on or spill-plan hand-off: planned CPU offload on every OS.+ cause = (+ "The GPU cannot hold it, so layers will be moved to the CPU and generation "+ f"will be slower. Lower the context to {max_available_ctx:,} or less, or "+ "leave it on Auto."+ )+ return (+ f"A context of {requested_ctx:,} tokens does not fit in this GPU's memory with "+ f"this model. The largest that fits is {max_available_ctx:,} tokens. "+ f"{cause}{kv_hint}"+ )+@staticmethoddef _unmeasured_context_notice(requested_ctx: int, cache_type_kv: Optional[str] = None@@ -15566,9 +15661,13 @@ class LlamaCppBackend:# Per-GPU usable budget: free minus the reserve when total is known, else# the legacy free*frac (also covers a total-0 two-column probe).+ _sysmem_fallback = LlamaCppBackend._sysmem_fallback_risk()+def _usable(idx: int, free_mib: int) -> float:t = total_by_idx.get(idx, 0) if total_by_idx else 0- usable = _vram_usable_mib(free_mib, t, usable_fraction)+ usable = _vram_usable_mib(+ free_mib, t, usable_fraction, sysmem_fallback = _sysmem_fallback+ )return max(0.0, usable) if t > 0 else usable# Rank by usable budget (free - reserve), not raw free: a more-used large@@ -17014,7 +17113,13 @@ class LlamaCppBackend:flat_mtp = mtp_engaged and mtp_overhead_fn is Nonebudget_frac = _active_vram_fraction() - (_MTP_VRAM_RESERVE_FRAC if flat_mtp else 0.0)# Absolute reserve off total when known, else fraction-of-free; clamp >=0.- budget_mib = _vram_usable_mib(available_mib, total_mib or 0, budget_frac, pooled = pooled)+ budget_mib = _vram_usable_mib(+ available_mib,+ total_mib or 0,+ budget_frac,+ pooled = pooled,+ sysmem_fallback = LlamaCppBackend._sysmem_fallback_risk(),+ )if total_mib is not None and total_mib > 0:budget_mib = max(0.0, budget_mib)budget_bytes = budget_mib * 1024 * 1024@@ -20913,6 +21018,8 @@ class LlamaCppBackend:# direct callers (tests) with no load to inherit from, and once, not per GPU._tp_frac = vram_fraction if vram_fraction is not None else _active_vram_fraction()+ _tp_sysmem_fallback = LlamaCppBackend._sysmem_fallback_risk()+def _usable(idx: int, free_mib: int) -> float:# Through the shared helper, so the floor reserve the ranking and the# layer path apply is charged here too. Tensor mode has no --fit valve,@@ -20920,7 +21027,10 @@ class LlamaCppBackend:# offloading. Clamped on both branches, unlike _select_gpus: this pool# is summed, and one negative card must not fund another.t = total_by_idx.get(idx, 0) if total_by_idx else 0- return max(0.0, _vram_usable_mib(free_mib, t, _tp_frac))+ return max(+ 0.0,+ _vram_usable_mib(free_mib, t, _tp_frac, sysmem_fallback = _tp_sysmem_fallback),+ )# Drop GPUs whose usable budget can't hold the per-device compute-graph# buffer; they'd OOM in tensor mode. Admitting on raw free would let a@@ -21115,10 +21225,15 @@ class LlamaCppBackend:return False_tp_frac = vram_fraction if vram_fraction is not None else _active_vram_fraction()+ # Must match the planner's floor, or this passes splits the planner rejected.+ _tp_sysmem_fallback = LlamaCppBackend._sysmem_fallback_risk()def _usable(idx: int, free_mib: int) -> float:t = total_by_idx.get(idx, 0) if total_by_idx else 0- return max(0.0, _vram_usable_mib(free_mib, t, _tp_frac))+ return max(+ 0.0,+ _vram_usable_mib(free_mib, t, _tp_frac, sysmem_fallback = _tp_sysmem_fallback),+ )free_by_idx = {idx: free for idx, free in gpus}# Fail closed on a device the survey does not cover. ``gpus`` is the set@@ -23810,6 +23925,8 @@ class LlamaCppBackend:# the slot from a memory warning on the one path that also raises the# launched context. Flushed past those two, appending if one spoke._unmeasured_ctx_notice: Optional[str] = None+ _cuda_ctx_notice: Optional[str] = None+ _ctx_cap_fits = Falsetotal_by_idx: dict[int, int] = {}_gpu_mem: list[tuple[int, int, int]] = []model_size = None # set in the fit try; used by the APU RAM guard@@ -24002,11 +24119,18 @@ class LlamaCppBackend:)_fit_target_delta_mib = (_CTX_FIT_VRAM_FRACTION - _vram_frac) * _scale+ _sysmem_fallback = self._sysmem_fallback_risk(binary)+def _gpu_usable(g, frac = _vram_frac):# Callers pass the ACTIVE fraction so the ranking matches the# budget the fit then tests, else mixed totals mis-order.idx, free = g- return _vram_usable_mib(free, total_by_idx.get(idx, 0), frac)+ return _vram_usable_mib(+ free,+ total_by_idx.get(idx, 0),+ frac,+ sysmem_fallback = _sysmem_fallback,+ )def _pool_budget_mib(subset, frac):# Sum each GPU's own usable budget. Pooling free and total@@ -24396,6 +24520,7 @@ class LlamaCppBackend:ctx: int,n_gpus: int = 1,slots: int = 0,+ cache_type: Optional[str] = None,) -> int:# Context-linear compute-buffer growth (flash-attn KQ mask +# attention scratch); the flat _compute_buffer_pipeline folded@@ -24415,17 +24540,25 @@ class LlamaCppBackend:return max(1, n_gpus) * self._compute_buffer_ctx_bytes(ctx,_ubatch_for_slots(slots) if slots else _effective_ubatch,- _scratch_cache_type_kv,+ cache_type or _scratch_cache_type_kv,layer_split = n_gpus > 1 and not _pipeline_parallel_off,flash_attn = planned_flash_attn,n_parallel = slots or n_parallel,)- def _cc_split_extra(ctx: int, slots: int = 0) -> int:+ def _cc_split_extra(+ ctx: int,+ slots: int = 0,+ cache_type: Optional[str] = None,+ ) -> int:# Per-device step from the single-device rate to the split one,# for the paths that must select GPUs before they know the# count. 0 when llama.cpp declines pipeline parallelism.- return max(0, _cc_bytes(ctx, 2, slots) // 2 - _cc_bytes(ctx, 1, slots))+ return max(+ 0,+ _cc_bytes(ctx, 2, slots, cache_type) // 2+ - _cc_bytes(ctx, 1, slots, cache_type),+ )# Layer-split compute buffer (one lump; tensor mode reserves it# per device in _plan_tensor_parallel). Context-independent, so@@ -25282,6 +25415,7 @@ class LlamaCppBackend:best_cap = max(best_cap, capped)if best_cap > 0:max_available_ctx = best_cap+ _ctx_cap_fits = Trueelse:# Weights exceed 90% of every GPU subset, so no# context fits. Anchor the UI "safe zone" at the@@ -25320,6 +25454,69 @@ class LlamaCppBackend:split_extra_bytes = _cc_split_extra(effective_ctx),)# No silent shrink: effective_ctx stays == requested_ctx.+ # use_fit = the pin failed and --fit on will offload; say why.+ # Else max_available_ctx is the Auto anchor, which does not fit either.+ # -nkvo keeps the cache on the host, so the priced overflow is not real.+ if (+ use_fit+ and _ctx_cap_fits+ and not _cuda_ctx_notice+ and _kv_offload_from_args(extra_args)+ and not _args_place_tensors_on_cpu(extra_args)+ and not _env_places_tensors_on_cpu()+ and not _device_selection_is_cpu(extra_args, os.environ)+ and not _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)+ and not _env_fixes_gpu_layers()+ # A trailing user --fit off disables the fitter that would offload.+ and (+ fit_is_enabled_in(extra_args)+ or not _extra_args_set_any_flag(extra_args, {"-fit", "--fit"})+ )+ # At the native ceiling the true max above it was never searched.+ and max_available_ctx < native_ctx_for_cap+ ):+ _q8_fits = False+ if planned_flash_attn and (+ cache_type_kv or "f16"+ ).strip().lower() in (+ "f16",+ "fp16",+ "",+ ):+ _q8_kv = self._estimate_kv_cache_bytes(+ effective_ctx,+ "q8_0",+ n_parallel = n_parallel,+ swa_full = swa_full,+ kv_unified = planned_kv_unified,+ n_ubatch = _effective_ubatch,+ flash_attn = planned_flash_attn,+ )+ if _q8_kv > 0:+ _, _q8_use_fit = self._select_gpus_split_aware(+ model_size_fit+ + _q8_kv+ + _mtp_bytes(effective_ctx)+ + _cc_bytes(effective_ctx, cache_type = "q8_0"),+ gpus,+ usable_fraction = _pin_fraction,+ total_by_idx = total_by_idx,+ per_device_overhead_bytes = (+ _pipeline_overhead_bytes+ + _cc_bytes(effective_ctx, cache_type = "q8_0")+ ),+ min_gpus = _layer_min_gpus,+ split_extra_bytes = _cc_split_extra(+ effective_ctx, cache_type = "q8_0"+ ),+ )+ _q8_fits = not _q8_use_fit+ _cuda_ctx_notice = self._cuda_context_overcommit_notice(+ effective_ctx,+ max_available_ctx,+ cache_type_kv,+ quantised_ctx_fits = _q8_fits,+ )else:# Auto context: prefer fewer GPUs, cap to fit. Same# headroom threshold as _select_gpus (#5106). Rank by the@@ -29383,6 +29580,12 @@ class LlamaCppBackend:else:self._record_load_warning(_unmeasured_ctx_notice)+ if _cuda_ctx_notice and use_fit: # slot reduction may have re-pinned it+ if self._last_load_warning:+ self._amend_load_warning(" " + _cuda_ctx_notice)+ else:+ self._record_load_warning(_cuda_ctx_notice)+# LoadRequest carries this flag, so a stale rollback, an API caller or# a swapped-out runtime can ask for a replay that never happened. Hold# it to the crash path's bar (managed auto Vulkan, no explicitdiff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.pyindex 727082ab887..62632c127b8 100644--- a/studio/backend/tests/test_compute_buffer.py+++ b/studio/backend/tests/test_compute_buffer.py@@ -1389,24 +1389,29 @@ class TestSplitRateRecheckAfterSelection:if keyword.arg in ("split_extra_bytes", "split_extra_for_slots")]# Projector floor pin, explicit-context pin, reduced-slot retry, per-candidate- # re-fit. A fifth has to come here and say which context it prices at.+ # re-fit, overcommit notice's q8_0 what-if. A sixth has to say which context it prices at.## Four, not the three the counting version asserted. It counted two spellings,# `_cc_split_extra(effective_ctx)` and `_cc_split_extra(ctx),`, and the# projector-floor site spells its context `_mm_floor_ctx`, so it was invisible# to the check that claimed to cover every call site. It has been wired# correctly the whole time; nothing was holding it there.- assert len(wired) == 4, wired+ assert len(wired) == 5, wired# Each passes the step at a context of its own, so none is exempt and none# hardcodes one: `_cc_split_extra(4096)` would not match.# The reduced-slot search re-prices the step for each candidate slot count.for expression in wired:assert re.fullmatch(- r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)", expression+ r"_cc_split_extra\(\w+\)|lambda s: _cc_split_extra\(\w+, s\)"+ r"|_cc_split_extra\(\w+, cache_type='q8_0'\)",+ expression,), expressionassert "gpu_indices,use_fit=self._select_gpus_split_aware(" in load# The step rides _cc_bytes' pipelining gate, so it is 0 when llama.cpp declines.- assert "returnmax(0,_cc_bytes(ctx,2,slots)//2-_cc_bytes(ctx,1,slots))" in load+ assert (+ "returnmax(0,_cc_bytes(ctx,2,slots,cache_type)//2-_cc_bytes(ctx,1,slots,cache_type),)"+ in load+ )slots = "".join(inspect.getsource(LlamaCppBackend._slots_that_fit_on_gpu).split())assert "self._select_gpus_split_aware(" in slotsassert (diff --git a/studio/backend/tests/test_cuda_context_overcommit_notice.py b/studio/backend/tests/test_cuda_context_overcommit_notice.pynew file mode 100644index 00000000000..5a8aad60505--- /dev/null+++ b/studio/backend/tests/test_cuda_context_overcommit_notice.py@@ -0,0 +1,303 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A hand-set context a discrete GPU cannot hold must say so (issues #11349, #11336)."""++import logging+import sys+import types as _types+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++_loggers_stub = _types.ModuleType("loggers")+_loggers_stub.get_logger = lambda name: logging.getLogger(name)+_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]+sys.modules.setdefault("loggers", _loggers_stub)++_structlog_stub = _types.ModuleType("structlog")+_structlog_stub.get_logger = lambda *a, **k: logging.getLogger("stub")+sys.modules.setdefault("structlog", _structlog_stub)++from core.inference.llama_cpp import LlamaCppBackend # noqa: E402++from test_kv_cache_estimation import _backend_from_gguf # noqa: E402++GIB = 1024**3+MIB = 1024**2++QWEN3VL_8B = {+ "block_count": 36,+ "attention.head_count": 32,+ "attention.head_count_kv": 8,+ "attention.key_length": 128,+ "attention.value_length": 128,+ "embedding_length": 4096,+ "context_length": 262144,+}++REPORTED_CTX = 65536+MEASURED_KV_MIB = 9216.0 # the single allocation llama-server asks for+WEIGHTS_BYTES = int(4.80 * GIB) # UD-Q4_K_XL on disk+MMPROJ_BYTES = int(1.40 * GIB) # mmproj-F16, loaded because -ngl 99 puts it on GPU+CARD_16GIB_MIB = 16.0 * 1024++[email protected](scope = "module")+def backend():+ return _backend_from_gguf("qwen3vl", QWEN3VL_8B)+++class TestKVEstimateAgainstHardware:+ """The estimator must reproduce what llama-server actually allocated."""++ def test_kv_cache_estimate_matches_measured_allocation(self, backend):+ kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ assert kv / MIB == pytest.approx(+ MEASURED_KV_MIB, abs = 0.5+ ), f"expected the measured {MEASURED_KV_MIB} MiB allocation, got {kv / MIB:.1f} MiB"+ assert kv / GIB == pytest.approx(9.0, abs = 0.01)++ def test_kv_scales_linearly_with_context(self, backend):+ """Halving the context halves the cache: the term that dominates the budget."""+ full = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ half = backend._estimate_kv_cache_bytes(REPORTED_CTX // 2, None)+ assert half / GIB == pytest.approx(full / GIB / 2, rel = 0.01)++ def test_q8_0_cache_is_substantially_smaller(self, backend):+ """q8_0 is the lever the planner never pulls; it must be worth suggesting."""+ f16 = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ q8 = backend._estimate_kv_cache_bytes(REPORTED_CTX, "q8_0")+ assert q8 < f16+ assert q8 / f16 == pytest.approx(0.53, abs = 0.03)+ assert (f16 - q8) / GIB > 4.0, "q8_0 must free more than 4 GiB to be worth offering"+++class TestSixteenGibBudget:+ """The reported configuration must be recognised as not fitting a 16 GiB card."""++ def test_reported_config_exceeds_a_16gib_card(self, backend):+ kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ footprint = (+ WEIGHTS_BYTES+ + MMPROJ_BYTES+ + int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))+ + backend._CUDA_CONTEXT_RESERVE_BYTES+ + kv+ )+ footprint_mib = footprint / MIB+ assert (+ footprint_mib > CARD_16GIB_MIB+ ), f"footprint {footprint_mib:.0f} MiB should exceed a {CARD_16GIB_MIB:.0f} MiB card"+ assert 0 < footprint_mib - CARD_16GIB_MIB < 200++ def test_the_budget_rejects_it_by_a_clear_margin(self, backend):+ """Against the 0.97 budget rather than the raw card, the verdict is unambiguous."""+ kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ footprint_mib = (+ WEIGHTS_BYTES+ + MMPROJ_BYTES+ + int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))+ + backend._CUDA_CONTEXT_RESERVE_BYTES+ + kv+ ) / MIB+ budget_mib = CARD_16GIB_MIB * 0.97+ assert footprint_mib > budget_mib+ assert footprint_mib - budget_mib > 500++ def test_q8_0_brings_the_same_config_under_the_card(self, backend):+ """The suggestion the notice makes has to be true."""+ kv_q8 = backend._estimate_kv_cache_bytes(REPORTED_CTX, "q8_0")+ footprint = (+ WEIGHTS_BYTES+ + MMPROJ_BYTES+ + int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))+ + backend._CUDA_CONTEXT_RESERVE_BYTES+ + kv_q8+ )+ assert footprint / MIB < CARD_16GIB_MIB++ def test_text_only_at_16gib_fits_which_is_why_it_was_hard_to_see(self, backend):+ """Without the mmproj the same context fits, matching the Linux run that passed."""+ kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)+ footprint = WEIGHTS_BYTES + backend._CUDA_CONTEXT_RESERVE_BYTES + kv+ assert footprint / MIB < CARD_16GIB_MIB+++class TestOvercommitNotice:+ """The advisory itself: when it speaks, when it stays quiet, and what it says."""++ def test_silent_when_the_context_fits(self):+ assert LlamaCppBackend._cuda_context_overcommit_notice(8192, 65536, None) is None++ def test_silent_without_a_measured_ceiling(self):+ """No ceiling means no fit maths ran; _unmeasured_context_notice owns that case."""+ assert LlamaCppBackend._cuda_context_overcommit_notice(65536, 0, None) is None+ assert LlamaCppBackend._cuda_context_overcommit_notice(0, 65536, None) is None++ def test_silent_at_exactly_the_ceiling(self):+ assert LlamaCppBackend._cuda_context_overcommit_notice(32768, 32768, None) is None++ def test_reports_both_numbers(self):+ msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)+ assert msg is not None+ assert "65,536" in msg and "32,768" in msg++ def test_describes_the_planned_cpu_offload(self):+ msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)+ assert "moved to the CPU" in msg+ assert "Sysmem" not in msg++ def test_q8_hint_only_when_it_actually_fixes_it(self):+ """A suggestion that would not help is worse than no suggestion."""+ helped = LlamaCppBackend._cuda_context_overcommit_notice(+ 65536, 32768, None, quantised_ctx_fits = True+ )+ assert "q8_0" in helped++ not_helped = LlamaCppBackend._cuda_context_overcommit_notice(+ 65536, 32768, None, quantised_ctx_fits = False+ )+ assert "q8_0" not in not_helped++ def test_no_q8_hint_when_the_cache_is_already_quantised(self):+ msg = LlamaCppBackend._cuda_context_overcommit_notice(+ 65536, 32768, "q8_0", quantised_ctx_fits = True+ )+ assert "q8_0" not in msg++ def test_never_raises_and_always_returns_text(self):+ msg = LlamaCppBackend._cuda_context_overcommit_notice(+ 65536, 4096, None, quantised_ctx_fits = True+ )+ assert isinstance(msg, str) and msg.strip()++ def test_is_an_advisory_not_a_refusal(self):+ """Unlike Metal, a discrete GPU spills, so this warns rather than refuses."""+ msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)+ for word in ("cannot load", "refused", "aborted", "will not start"):+ assert word not in msg.lower()+++def _launch_explicit_ctx(tmp_path, monkeypatch, model_gb, n_ctx, **load_kwargs):+ from test_llama_cpp_placement import _backend as _placement_backend, _launch++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))+ backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])+ backend._get_gguf_size_bytes = lambda _path: int(model_gb * GIB)+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024+ backend._estimate_compute_buffer_bytes = lambda **k: 1+ cmd = _launch(backend, gguf, n_ctx = n_ctx, **load_kwargs)["cmd"]+ assert cmd[cmd.index("--fit") + 1] == "on"+ return backend.last_load_warning or ""+++def test_the_notice_names_a_context_that_fits(tmp_path, monkeypatch):+ warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)+ assert "does not fit in this GPU's memory" in warning+++def test_no_context_is_offered_when_the_weights_alone_overflow(tmp_path, monkeypatch):+ """40 GB of weights on a 24 GB card: no context fits, so none may be offered."""+ warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 40, n_ctx = 32768)+ assert "The largest that fits is" not in warning+++def test_a_windows_cuda_build_is_not_told_the_driver_spills(tmp_path, monkeypatch):+ """--fit on and spill plans place host tensors on purpose; WDDM advice cannot undo that."""+ monkeypatch.setattr(LlamaCppBackend, "_sysmem_fallback_risk", staticmethod(lambda b = None: True))+ warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)+ assert "moved to the CPU" in warning+ assert "NVIDIA Control Panel" not in warning+++ "q8_scratch_gib, extra_args, offered",+ [(0, [], True), (8, [], False), (0, ["--flash-attn", "off"], False)],+)+def test_the_q8_hint_prices_q8_compute_scratch(+ tmp_path, monkeypatch, q8_scratch_gib, extra_args, offered+):+ """q8_0 adds dequant scratch; the hint must hold with it, not with the f16 figure."""+ from test_llama_cpp_placement import _backend as _placement_backend, _launch++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))+ backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])+ backend._get_gguf_size_bytes = lambda _path: 14 * GIB+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = (+ lambda ctx, cache = None, *a, **k: int(ctx)+ * (32 if str(cache).startswith("q8") else 64)+ * 1024+ )+ backend._estimate_compute_buffer_bytes = lambda **k: 1+ backend._compute_buffer_ctx_bytes = lambda ctx, ub, cache_type = None, **k: (+ q8_scratch_gib * GIB if str(cache_type).startswith("q8") else 0+ )+ _launch(backend, gguf, n_ctx = 196608, extra_args = extra_args)+ warning = backend.last_load_warning or ""+ assert "does not fit in this GPU's memory" in warning+ assert ("q8_0" in warning) is offered+++def test_no_notice_when_the_kv_cache_stays_on_the_host(tmp_path, monkeypatch):+ """-nkvo keeps the cache off the GPU, so the priced KV overflow does not happen."""+ warning = _launch_explicit_ctx(+ tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072, extra_args = ["-nkvo"]+ )+ assert "does not fit in this GPU's memory" not in warning+++ "extra_args", [["--device", "none"], ["-ot", "exps=CPU"], ["--gpu-layers", "10"]]+)+def test_no_notice_for_a_deliberate_cpu_placement(tmp_path, monkeypatch, extra_args):+ warning = _launch_explicit_ctx(+ tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072, extra_args = extra_args+ )+ assert "does not fit in this GPU's memory" not in warning+++def test_no_notice_when_an_inherited_env_fixes_the_layer_count(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_N_GPU_LAYERS", "20")+ warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)+ assert "does not fit in this GPU's memory" not in warning+++def test_no_largest_context_claimed_at_the_native_ceiling(tmp_path, monkeypatch):+ """Past the native window the sweep stops at native, so the true maximum is unknown."""+ monkeypatch.setattr(LlamaCppBackend, "_context_length", 32768, raising = False)+ from test_llama_cpp_placement import _backend as _placement_backend, _launch++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))+ backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])+ backend._read_gguf_metadata = lambda _path: setattr(backend, "_context_length", 32768)+ backend._get_gguf_size_bytes = lambda _path: 14 * GIB+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 96 * 1024+ backend._estimate_compute_buffer_bytes = lambda **k: 1+ _launch(backend, gguf, n_ctx = 131072)+ assert "The largest that fits is 32,768" not in (backend.last_load_warning or "")++[email protected]("extra_args, warned", [(["--fit", "off"], False), (["--fit=on"], True)])+def test_a_user_fit_override_decides_the_notice(tmp_path, monkeypatch, extra_args, warned):+ from test_llama_cpp_placement import _backend as _placement_backend, _launch++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))+ backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])+ backend._get_gguf_size_bytes = lambda _path: 18 * GIB+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024+ backend._estimate_compute_buffer_bytes = lambda **k: 1+ _launch(backend, gguf, n_ctx = 131072, extra_args = extra_args)+ assert ("does not fit in this GPU's memory" in (backend.last_load_warning or "")) is warneddiff --git a/studio/backend/tests/test_slot_reduction_context_refit.py b/studio/backend/tests/test_slot_reduction_context_refit.pyindex ee7aff84860..f9da922fd14 100644--- a/studio/backend/tests/test_slot_reduction_context_refit.py+++ b/studio/backend/tests/test_slot_reduction_context_refit.py@@ -142,6 +142,7 @@ def _plan("fit": flag("--fit", "off"),"spec": flag("--spec-type", "-"),"ceiling": backend._max_context_length,+ "warning": backend.last_load_warning or "","devices": (launched["env"] or {}).get("CUDA_VISIBLE_DEVICES"),}@@ -308,6 +309,8 @@ class TestWhatMustNotMove:assert (got["ctx"], got["slots"], got["fit"]) == (32768, 2, "off")# The measured ceiling still follows the final slot count.assert got["ceiling"] == 36_096+ # Re-pinned after the reduction, so no "does not fit" notice may survive.+ assert "does not fit in this GPU" not in got["warning"]def test_an_explicit_context_that_forces_offload_is_unchanged(self, tmp_path):"""An explicit context that requires offload is unchanged."""diff --git a/studio/backend/tests/test_sysmem_fallback_vram_headroom.py b/studio/backend/tests/test_sysmem_fallback_vram_headroom.pynew file mode 100644index 00000000000..bf74612d4d1--- /dev/null+++ b/studio/backend/tests/test_sysmem_fallback_vram_headroom.py@@ -0,0 +1,327 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Windows sysmem-fallback headroom in the VRAM budget (unslothai/unsloth#11349)."""++import sys++import pytest++from core.inference.llama_cpp import (+ _CTX_FIT_VRAM_FRACTION,+ _LLAMA_FIT_TARGET_DEFAULT_MIB,+ _VRAM_FLOOR_RESERVE_MIB,+ _WINDOWS_SYSMEM_FALLBACK_MAX_FRACTION,+ _WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB,+ LlamaCppBackend,+ _vram_reserve_floor_mib,+ _vram_usable_mib,+)++GIB = 1024.0+CARD_SIZES_MIB = [4 * GIB, 6 * GIB, 8 * GIB, 12 * GIB, 16 * GIB, 24 * GIB, 48 * GIB]+FRACTIONS = [0.80, 0.90, 0.95, _CTX_FIT_VRAM_FRACTION, 0.971, 0.98, 0.99, 1.0]+++class TestFlagDefaultsToTodaysBehaviour:+ """The flag must be opt-in, so an untaught caller cannot change platform behaviour."""++ @pytest.mark.parametrize("total", CARD_SIZES_MIB)+ @pytest.mark.parametrize("frac", FRACTIONS)+ def test_default_matches_explicit_false(self, total, frac):+ free = total * 0.94+ assert _vram_usable_mib(free, total, frac) == _vram_usable_mib(+ free, total, frac, sysmem_fallback = False+ )++ @pytest.mark.parametrize("total", CARD_SIZES_MIB)+ def test_floor_default_matches_explicit_false(self, total):+ assert _vram_reserve_floor_mib(total) == _vram_reserve_floor_mib(+ total, sysmem_fallback = False+ )++ def test_unknown_total_branch_also_defaults_off(self):+ assert _vram_usable_mib(9000, 0, 0.97) == _vram_usable_mib(+ 9000, 0, 0.97, sysmem_fallback = False+ )+++class TestRaisedFloor:+ @pytest.mark.parametrize("total", CARD_SIZES_MIB)+ def test_never_reserves_less_than_before(self, total):+ assert _vram_reserve_floor_mib(total, sysmem_fallback = True) >= _vram_reserve_floor_mib(+ total+ )++ @pytest.mark.parametrize("total", CARD_SIZES_MIB)+ def test_never_exceeds_an_eighth_of_the_card(self, total):+ floor = _vram_reserve_floor_mib(total, sysmem_fallback = True)+ assert floor <= _WINDOWS_SYSMEM_FALLBACK_MAX_FRACTION * total + 1e-9++ @pytest.mark.parametrize("total", [8 * GIB, 12 * GIB, 16 * GIB, 24 * GIB, 48 * GIB])+ def test_adopts_llama_cpp_own_fit_target_above_8gib(self, total):+ assert _vram_reserve_floor_mib(total, sysmem_fallback = True) == pytest.approx(+ _LLAMA_FIT_TARGET_DEFAULT_MIB+ )++ def test_small_cards_stay_proportionate(self):+ assert _vram_reserve_floor_mib(4 * GIB, sysmem_fallback = True) == pytest.approx(512.0)++ def test_reserve_constant_is_llama_cpp_fit_target(self):+ assert _WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB == _LLAMA_FIT_TARGET_DEFAULT_MIB+ assert _WINDOWS_SYSMEM_FALLBACK_RESERVE_MIB > _VRAM_FLOOR_RESERVE_MIB+++class TestMonotonicity:+ """Raising the VRAM-budget slider must never hand back LESS context."""++ @pytest.mark.parametrize("sysmem_fallback", [False, True])+ @pytest.mark.parametrize("total", CARD_SIZES_MIB)+ def test_budget_never_decreases_as_fraction_rises(self, total, sysmem_fallback):+ budgets = [+ _vram_usable_mib(total, total, f, sysmem_fallback = sysmem_fallback) for f in FRACTIONS+ ]+ for lower, higher in zip(budgets, budgets[1:]):+ assert higher >= lower - 1e-9++ def test_docstring_8gib_example_unchanged_on_linux(self):+ t = 8 * GIB+ assert _vram_usable_mib(t, t, 0.97) == pytest.approx(7946.2, abs = 0.5)+ assert _vram_usable_mib(t, t, 0.971) == pytest.approx(7946.2, abs = 0.5)++ def test_docstring_8gib_example_is_flat_under_fallback(self):+ t = 8 * GIB+ at_default = _vram_usable_mib(t, t, 0.97, sysmem_fallback = True)+ past_default = _vram_usable_mib(t, t, 0.971, sysmem_fallback = True)+ assert at_default == pytest.approx(7168.0)+ assert past_default == pytest.approx(at_default)++ def test_a_lowered_slider_can_still_reserve_more_than_the_floor(self):+ t = 8 * GIB+ assert _vram_usable_mib(t, t, 0.80, sysmem_fallback = True) == pytest.approx(t - 0.20 * t)+++class TestPooledIsNotChargedTwice:+ @pytest.mark.parametrize("frac", [0.90, _CTX_FIT_VRAM_FRACTION, 1.0])+ def test_pooled_ignores_the_flag(self, frac):+ assert _vram_usable_mib(9000, 0, frac, pooled = True, sysmem_fallback = True) == (+ _vram_usable_mib(9000, 0, frac, pooled = True)+ )+++class TestRiskClassifier:+ @pytest.fixture(autouse = True)+ def _clear_cache(self):+ LlamaCppBackend._SYSMEM_FALLBACK_RISK.clear()+ yield+ LlamaCppBackend._SYSMEM_FALLBACK_RISK.clear()++ def test_false_off_windows(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: frozenset({"cuda", "base"})),+ )+ assert LlamaCppBackend._sysmem_fallback_risk() is False++ def test_false_on_darwin(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "darwin")+ assert LlamaCppBackend._sysmem_fallback_risk() is False++ def test_true_on_windows_cuda(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: frozenset({"cuda", "base", "cpu"})),+ )+ assert LlamaCppBackend._sysmem_fallback_risk() is True++ @pytest.mark.parametrize(+ "backends",+ [+ frozenset({"hip", "base"}), # ROCm: different vendor, different mechanism+ frozenset({"vulkan", "base"}), # Vulkan does not allocate through CUDA+ frozenset({"cpu", "base"}), # no GPU backend at all+ frozenset(), # unreadable lib dir+ ],+ )+ def test_false_on_windows_without_cuda(self, monkeypatch, backends):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: backends),+ )+ assert LlamaCppBackend._sysmem_fallback_risk() is False++ def test_multi_backend_build_with_cuda_counts(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: frozenset({"cuda", "hip", "vulkan"})),+ )+ assert LlamaCppBackend._sysmem_fallback_risk() is True++ def test_classification_failure_keeps_todays_budget(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")++ def _boom(binary = None):+ raise OSError("lib dir vanished mid-load")++ monkeypatch.setattr(LlamaCppBackend, "_installed_ggml_backends", staticmethod(_boom))+ assert LlamaCppBackend._sysmem_fallback_risk() is False++ def test_read_error_is_not_cached(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ calls = {"n": 0}++ def _flaky(binary = None):+ calls["n"] += 1+ if calls["n"] == 1:+ raise OSError("transient")+ return frozenset({"cuda"})++ monkeypatch.setattr(LlamaCppBackend, "_installed_ggml_backends", staticmethod(_flaky))+ assert LlamaCppBackend._sysmem_fallback_risk() is False+ assert LlamaCppBackend._sysmem_fallback_risk() is True++ def test_cache_is_keyed_per_binary(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ table = {+ "C:\\cuda-build\\llama-server.exe": frozenset({"cuda"}),+ "C:\\vulkan-build\\llama-server.exe": frozenset({"vulkan"}),+ }+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: table[binary]),+ )+ assert LlamaCppBackend._sysmem_fallback_risk("C:\\cuda-build\\llama-server.exe") is True+ assert LlamaCppBackend._sysmem_fallback_risk("C:\\vulkan-build\\llama-server.exe") is False++ def test_second_call_does_not_relist_the_directory(self, monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ calls = {"n": 0}++ def _counted(binary = None):+ calls["n"] += 1+ return frozenset({"cuda"})++ monkeypatch.setattr(LlamaCppBackend, "_installed_ggml_backends", staticmethod(_counted))+ for _ in range(5):+ assert LlamaCppBackend._sysmem_fallback_risk() is True+ assert calls["n"] == 1+++class TestQwen3VL8BOn16GiBCard:+ """The issue-11349 shape, and the near-miss band the fix actually changes."""++ WEIGHTS_MIB = 4.80 * GIB+ MMPROJ_MIB = 1.40 * GIB+ CUDA_CTX_MIB = 320.0+ MMPROJ_SAFETY_MIB = 1.40 * GIB * 0.4+ COMPUTE_MIB = 400.0+ TOTAL_MIB = 16 * GIB+ FREE_MIB = 15400.0 # ~1 GiB held by the Windows desktop++ @staticmethod+ def kv_mib(ctx):+ return 36 * ctx * 8 * (128 + 128) * 2 / (1024 * 1024)++ def footprint(self, ctx):+ return (+ self.WEIGHTS_MIB+ + self.MMPROJ_MIB+ + self.CUDA_CTX_MIB+ + self.MMPROJ_SAFETY_MIB+ + self.COMPUTE_MIB+ + self.kv_mib(ctx)+ )++ def test_kv_matches_the_measured_allocation(self):+ assert self.kv_mib(65536) == pytest.approx(9216.0)++ def test_65536_was_already_rejected_and_still_is(self):+ fp = self.footprint(65536)+ assert fp > _vram_usable_mib(self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION)+ assert fp > _vram_usable_mib(+ self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION, sysmem_fallback = True+ )++ def test_near_miss_context_flips_from_pin_to_fit(self):+ fp = self.footprint(49152)+ before = _vram_usable_mib(self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION)+ after = _vram_usable_mib(+ self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION, sysmem_fallback = True+ )+ assert fp <= before, "precondition: today this pins a full offload"+ assert fp > after, "after the fix it is handed to --fit instead"++ def test_comfortable_context_is_untouched(self):+ fp = self.footprint(32768)+ assert fp <= _vram_usable_mib(+ self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION, sysmem_fallback = True+ )++ def test_flip_band_is_bounded(self):+ before = _vram_usable_mib(self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION)+ after = _vram_usable_mib(+ self.FREE_MIB, self.TOTAL_MIB, _CTX_FIT_VRAM_FRACTION, sysmem_fallback = True+ )+ assert before - after == pytest.approx(+ _LLAMA_FIT_TARGET_DEFAULT_MIB+ - min(_VRAM_FLOOR_RESERVE_MIB, (1 - _CTX_FIT_VRAM_FRACTION) * self.TOTAL_MIB)+ )+++def test_fallback_risk_is_cached_per_resolved_binary(monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(LlamaCppBackend, "_SYSMEM_FALLBACK_RISK", {})+ libs = {"/vulkan/llama-server": {"vulkan"}, "/cuda/llama-server": {"cuda"}}+ selected = ["/vulkan/llama-server"]+ monkeypatch.setattr(+ LlamaCppBackend,+ "_find_llama_server_binary",+ staticmethod(lambda include_denied = False: selected[0]),+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: frozenset(libs[binary or selected[0]])),+ )+ assert LlamaCppBackend._sysmem_fallback_risk() is False+ selected[0] = "/cuda/llama-server"+ assert LlamaCppBackend._sysmem_fallback_risk() is True+++def test_fallback_risk_follows_an_in_place_binary_update(monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(LlamaCppBackend, "_SYSMEM_FALLBACK_RISK", {})+ build = ["vulkan"]+ monkeypatch.setattr(+ LlamaCppBackend, "_binary_revision", staticmethod(lambda binary: (binary, build[0]))+ )+ monkeypatch.setattr(+ LlamaCppBackend,+ "_installed_ggml_backends",+ staticmethod(lambda binary = None: frozenset({build[0]})),+ )+ assert LlamaCppBackend._sysmem_fallback_risk("/llama-server") is False+ build[0] = "cuda"+ assert LlamaCppBackend._sysmem_fallback_risk("/llama-server") is True+++def test_an_empty_backend_scan_is_not_cached(monkeypatch):+ """An unreadable lib dir scans empty; that must not pin False for the revision."""+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setattr(LlamaCppBackend, "_SYSMEM_FALLBACK_RISK", {})+ monkeypatch.setattr(LlamaCppBackend, "_binary_revision", staticmethod(lambda b: (b, 1)))+ scans = [frozenset(), frozenset({"cuda"})]+ monkeypatch.setattr(+ LlamaCppBackend, "_installed_ggml_backends", staticmethod(lambda b = None: scans.pop(0))+ )+ assert LlamaCppBackend._sysmem_fallback_risk("/llama-server") is False+ assert LlamaCppBackend._sysmem_fallback_risk("/llama-server") is True