Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: name the /v1/models VRAM-fit context estimate (#12599)
Aziz Muminov1b5e516297fb
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 779463eea31..0b831df1205 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4556,6 +4556,32 @@ _CPU_PLACEMENT_PRESENCE_FLAGS = (_MOE_OFFLOAD_FLAGS - _CPU_MOE_COUNT_FLAGS) | fr{"-ot", "--override-tensor"})_CPU_PLACEMENT_FLAGS = _MOE_OFFLOAD_FLAGS | frozenset({"-ot", "--override-tensor"})+# Pass-through that re-places what a fit priced: the fitter, split, remote (RPC) devices,+# draft tensors moved onto a card of their own, or an unpriced vocoder GGUF.+_VRAM_FIT_VOIDING_FLAGS = (+ frozenset(+ {+ "-fit",+ "--fit",+ "--rpc",+ "-otd",+ "--override-tensor-draft",+ "--spec-draft-override-tensor",+ "-mv",+ "--model-vocoder",+ }+ )+ | _FIT_CONTROL_FLAGS+ | _TENSOR_SPLIT_FLAGS+ | _SPLIT_MODE_FLAGS+)+_VRAM_FIT_VOIDING_ENV_VARS = (+ "LLAMA_ARG_FIT",+ "LLAMA_ARG_RPC",+ *_FIT_CONTROL_ENV_VARS,+ "LLAMA_ARG_TENSOR_SPLIT",+ "LLAMA_ARG_SPLIT_MODE",+)_THREAD_OVERRIDE_FLAGS = frozenset({"-t", "--threads"})_GENERATION_CPU_AFFINITY_FLAGS = frozenset({"-C", "--cpu-mask", "-Cr", "--cpu-range"})_TENSOR_SPLIT_FLAGS = frozenset({"--tensor-split", "-ts"})@@ -6435,6 +6461,65 @@ def _subsequence_index(tokens: List[str], run: List[str], hint: int) -> int:_CPU_DEVICE_VALUES = frozenset({"cpu", "none"})+def _pass_through_overrides_placement(+ extra_args: Optional[Iterable[str]],+ source_env: Mapping[str, str],+ credited: int,+ *,+ is_vulkan_backend: bool = False,+ gpu_indices: Optional[Iterable[int]] = None,+) -> bool:+ """Whether pass-through args or env run a placement other than the one a fit priced.++ ``extra_args`` / ``source_env`` are what the child really gets (a ``gpu_ids`` pin+ strips its device flags and env twins first); ``credited`` is how many devices the+ fit charged. See ``LlamaCppBackend._fit_derived_load_mode`` for each term.+ """+ # A --device naming FEWER devices than this fit charges leaves the credit+ # paying for cards the child never opens. llama.cpp REPLACES the list on every+ # occurrence (parse_device_list) and reads LLAMA_ARG_DEVICE before argv, so the+ # last of the two wins. Counted, not name-matched: CUDA/ROCm ordinals are+ # assigned after a visibility mask this launch has not written yet.+ _dev_value = _extra_args_main_device(extra_args)+ if _dev_value is None:+ _dev_value = source_env.get("LLAMA_ARG_DEVICE")+ _dev_names = [d.strip() for d in str(_dev_value or "").split(",") if d.strip()]+ _credited = credited+ gpu_indices = list(gpu_indices) if gpu_indices is not None else None+ # DISTINCT names: llama.cpp appends every entry without deduplicating and+ # keeps the list verbatim, so "CUDA0,CUDA0" is ONE physical pool listed twice,+ # and counting it as two leaves the credit paying for a card that is not there.+ _device_narrows = bool(_dev_names) and len(set(_dev_names)) < _credited+ # Vulkan is the one backend where names ARE comparable (_vulkan_device_pin ->+ # "Vulkan<i>"), so hold a surviving --device to an exact match. That catches+ # what a count cannot: a same-sized list naming DIFFERENT cards, and a longer+ # one adding devices this footprint never charged for.+ _vulkan_pin_replaced = bool(+ is_vulkan_backend+ and gpu_indices is not None+ and _dev_names+ and {name.strip().lower() for name in _dev_names} != {f"vulkan{idx}" for idx in gpu_indices}+ )+ return bool(+ _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)+ # And its env twin, the ONLY layer count the fitting path leaves: it+ # emits no -ngl, only Manual mode clears LLAMA_ARG_N_GPU_LAYERS, and the+ # fitter will not lower a count it did not set, so an inherited one stands+ # and loads a prefix out of host RAM.+ or _env_fixes_gpu_layers(source_env)+ # With the env twin: only an explicit gpu_ids pin clears LLAMA_ARG_DEVICE,+ # so an inherited "none" puts the whole load in host RAM.+ or _device_selection_is_cpu(extra_args, source_env)+ or _device_narrows+ or _vulkan_pin_replaced+ # Pass-through and last-wins: --split-mode none, or a --tensor-split that+ # zeroes a device, leaves the pooled credit paying for cards held empty.+ or split_policy_starves_devices(extra_args, _credited, source_env)+ or _args_place_tensors_on_cpu(extra_args)+ or _env_places_tensors_on_cpu(source_env)+ )++def _device_selection_is_cpu(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None) -> bool:@@ -7869,6 +7954,8 @@ class LlamaCppBackend:self._context_length: Optional[int] = Noneself._effective_context_length: Optional[int] = Noneself._max_context_length: Optional[int] = None+ # Ceiling a fit priced inside the GPU / Metal budget; None for an anchor or floor.+ self._vram_fit_context_length: Optional[int] = Noneself._effective_parallel_slots: int = 1# --parallel the last load asked for, before any fit-time reduction.self._requested_n_parallel: int = 1@@ -8429,6 +8516,19 @@ class LlamaCppBackend:"""return self._max_context_length or self._context_length+ @property+ def vram_fit_context_length(self) -> Optional[int]:+ """Largest context the load-time fit priced inside GPU / Metal memory, else None."""+ # A child wholly on CPU (--device none, Vulkan crash fallback, zero layers) holds no VRAM.+ if (+ self._gpu_offload_active is False+ or self._cpu_fallback_reason+ or self._arch_gate_forced_cpu+ or (self._gpu_memory_mode == "manual" and self._gpu_layers == 0)+ ):+ return None+ return self._vram_fit_context_length+@propertydef native_context_length(self) -> Optional[int]:"""Return the model's native context length from GGUF metadata."""@@ -15573,48 +15673,13 @@ class LlamaCppBackend:return Nonerows = list(gpus or ())source_env = os.environ if env is None else env- # A --device naming FEWER devices than this fit charges leaves the credit- # paying for cards the child never opens. llama.cpp REPLACES the list on every- # occurrence (parse_device_list) and reads LLAMA_ARG_DEVICE before argv, so the- # last of the two wins. Counted, not name-matched: CUDA/ROCm ordinals are- # assigned after a visibility mask this launch has not written yet.- _dev_value = _extra_args_main_device(extra_args)- if _dev_value is None:- _dev_value = source_env.get("LLAMA_ARG_DEVICE")- _dev_names = [d.strip() for d in str(_dev_value or "").split(",") if d.strip()]_credited = len(list(gpu_indices)) if gpu_indices is not None else len(rows)- # DISTINCT names: llama.cpp appends every entry without deduplicating and- # keeps the list verbatim, so "CUDA0,CUDA0" is ONE physical pool listed twice,- # and counting it as two leaves the credit paying for a card that is not there.- _device_narrows = bool(_dev_names) and len(set(_dev_names)) < _credited- # Vulkan is the one backend where names ARE comparable (_vulkan_device_pin ->- # "Vulkan<i>"), so hold a surviving --device to an exact match. That catches- # what a count cannot: a same-sized list naming DIFFERENT cards, and a longer- # one adding devices this footprint never charged for.- _vulkan_pin_replaced = bool(- is_vulkan_backend- and gpu_indices is not None- and _dev_names- and {name.strip().lower() for name in _dev_names}- != {f"vulkan{idx}" for idx in gpu_indices}- )- if (- _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)- # And its env twin, the ONLY layer count the fitting path leaves: it- # emits no -ngl, only Manual mode clears LLAMA_ARG_N_GPU_LAYERS, and the- # fitter will not lower a count it did not set, so an inherited one stands- # and loads a prefix out of host RAM.- or _env_fixes_gpu_layers(source_env)- # With the env twin: only an explicit gpu_ids pin clears LLAMA_ARG_DEVICE,- # so an inherited "none" puts the whole load in host RAM.- or _device_selection_is_cpu(extra_args, source_env)- or _device_narrows- or _vulkan_pin_replaced- # Pass-through and last-wins: --split-mode none, or a --tensor-split that- # zeroes a device, leaves the pooled credit paying for cards held empty.- or split_policy_starves_devices(extra_args, _credited, source_env)- or _args_place_tensors_on_cpu(extra_args)- or _env_places_tensors_on_cpu(source_env)+ if _pass_through_overrides_placement(+ extra_args,+ source_env,+ _credited,+ is_vulkan_backend = is_vulkan_backend,+ gpu_indices = gpu_indices,):logger.info("Load mode: pass-through arguments override the placement this fit "@@ -19191,6 +19256,7 @@ class LlamaCppBackend:# Provisional until the server reports the budget it resolved (auto-size picks it from VRAM).self._effective_context_length = maxtok or self._context_lengthself._max_context_length = self._context_length or maxtok or None+ self._vram_fit_context_length = Nonehealthy = self._wait_for_health(timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = cancelled)if healthy:@@ -25008,6 +25074,8 @@ class LlamaCppBackend:_unmeasured_ctx_notice: Optional[str] = None_cuda_ctx_notice: Optional[str] = None_ctx_cap_fits = False+ _vram_fit_ctx: Optional[int] = None+ _flat_mtp_engages = Falsetotal_by_idx: dict[int, int] = {}_gpu_mem: list[tuple[int, int, int]] = []model_size = None # set in the fit try; used by the APU RAM guard@@ -26537,6 +26605,8 @@ class LlamaCppBackend:if best_cap > 0:max_available_ctx = best_cap_ctx_cap_fits = True+ # A forced GPU floor plans subsets this sweep did not hold to it.+ _vram_fit_ctx = best_cap if _layer_min_gpus <= 1 else Noneelse:# Weights exceed 90% of every GPU subset, so no# context fits. Anchor the UI "safe zone" at the@@ -27074,6 +27144,44 @@ class LlamaCppBackend:_metal_ctx_warning = self._metal_context_pressure_message(effective_ctx, _requested_mib, _apple_fit_budget_mib)+ # The paravirtual pin loads on CPU, so no Metal budget held anything.+ _vram_fit_ctx = None if _paravirtual_cpu_forced else _apple_measured_ceiling+ # The measured ceiling was priced without SWA checkpoints; re-fit with them.+ if _vram_fit_ctx and (+ _apple_footprint_mib(_vram_fit_ctx, _fit_ctx_checkpoints)+ > _apple_fit_budget_mib+ ):+ _ckpt_cap = _apple_ctx_fit(_vram_fit_ctx, _FIT_FLOOR_MIN_CTX)+ _vram_fit_ctx = (+ _ckpt_cap+ if _apple_footprint_mib(_ckpt_cap, _fit_ctx_checkpoints)+ <= _apple_fit_budget_mib+ else None+ )+ # Past the wired limit Metal refuses, so it is the harder ceiling when readable.+ _fit_wired_mib = (+ int(+ self._apple_metal_wired_ceiling_bytes()+ // (1024 * 1024)+ * max(0.0, 1.0 - _flat_mtp_reserve)+ )+ if _vram_fit_ctx+ else 0+ )+ if (+ _fit_wired_mib > 0+ and _apple_footprint_mib(_vram_fit_ctx, _fit_ctx_checkpoints)+ > _fit_wired_mib+ ):+ _wired_cap = _apple_ctx_fit(+ _vram_fit_ctx, _FIT_FLOOR_MIN_CTX, _fit_wired_mib+ )+ _vram_fit_ctx = (+ _wired_cap+ if _apple_footprint_mib(_wired_cap, _fit_ctx_checkpoints)+ <= _fit_wired_mib+ else None+ )# Unmeasured floors still rely on llama.cpp's fitter._metal_full_offload_pinned = bool(_apple_host_embd and _apple_measured_ceiling is not None@@ -27250,6 +27358,7 @@ class LlamaCppBackend:# the launched plan cannot serve, which is the same ceiling /# selection inversion #9492 removed, pointing the other way.max_available_ctx = _ceiling[0]+ _vram_fit_ctx = _ceiling[0]# MoE experts in host RAM: a larger micro-batch amortises the expert# copies of prompt processing (see _MOE_SPILL_N_UBATCH). The placement@@ -27678,6 +27787,8 @@ class LlamaCppBackend:# land after the dict exists but before the later terms are added# to it, which would plan against a footprint missing them._spill_inputs = None+ # Nor is a ceiling the fallback's own fitter may spill past.+ _vram_fit_ctx = Nonetp_tensor_split = Noneeffective_ctx = requested_ctx # fall back to original@@ -30563,6 +30674,7 @@ class LlamaCppBackend:run_cmd = _revertedself._record_memory_state(run_cmd, env)_did_fit_retry = True+ self._vram_fit_context_length = Nonecontinueif (not _did_fit_retry@@ -30638,6 +30750,8 @@ class LlamaCppBackend:run_cmd = _runself._record_memory_state(_run, env)_did_fit_retry = True+ # The fit that vouched for the ceiling is what just failed.+ self._vram_fit_context_length = Nonecontinueif (not _did_fit_retry@@ -30938,6 +31052,52 @@ class LlamaCppBackend:self._max_context_length = (max_available_ctx if max_available_ctx > 0 else self._effective_context_length)+ # The fit priced Unsloth's own placement and the base weights alone: a+ # pass-through placement override, adapter or inherited projector is not.+ if _vram_fit_ctx is not None:+ _child_extras = (+ self._strip_device_extra_args(extra_args)+ if _gpu_ids_own_device_flags+ else extra_args+ )+ _child_env = dict(os.environ)+ if gpu_memory_mode == "manual":+ self._clear_manual_placement_env(_child_env)+ if _gpu_ids_own_device_flags:+ self._clear_device_placement_env(_child_env)+ if (+ _pass_through_overrides_placement(+ _child_extras,+ _child_env,+ len(gpu_indices) if gpu_indices is not None else len(gpus),+ is_vulkan_backend = is_vulkan_backend,+ gpu_indices = gpu_indices,+ )+ or _sidecar_adapter_bytes(extra_args) != 0+ # A drafter the planner could only cover with a flat cushion.+ or _flat_mtp_engages+ or _child_env.get("LLAMA_ARG_MMPROJ")+ or _child_env.get("LLAMA_ARG_MMPROJ_URL")+ or (+ not launch_mmproj_path+ and extra_args_mmproj_auto(_child_extras, _child_env)+ )+ # A hand-set fitter or split ratio re-places what the fit priced.+ or _extra_args_set_any_flag(_child_extras, _VRAM_FIT_VOIDING_FLAGS)+ or any(_child_env.get(name) for name in _VRAM_FIT_VOIDING_ENV_VARS)+ # Any surviving --device: the cap sweep may have pooled cards it leaves out.+ or _extra_args_main_device(_child_extras) is not None+ # And any companion (projector, drafter) pinned to a card of its own.+ or any(+ _extra_args_device(_child_extras, flags) is not None+ for flags in _COMPANION_DEVICE_FLAG_GROUPS+ )+ or _child_env.get("LLAMA_ARG_DEVICE")+ # --no-kv-offload keeps the whole cache in host RAM at any context.+ or not _kv_offload_from_args(_child_extras, _child_env)+ ):+ _vram_fit_ctx = None+ self._vram_fit_context_length = _vram_fit_ctx# Past the host-RAM and offload advisories, so it can say the ceiling is# the request without costing the user a memory warning it does not@@ -31042,6 +31202,9 @@ class LlamaCppBackend:healthy = Falsetry:healthy = _spawn_and_wait(cmd)+ if not healthy:+ # Every recovery below relaunches a plan the fit never priced.+ self._vram_fit_context_length = Nonefinally:# Only once a child is up, where is_active takes over. An# unconditional clear undid the re-arm and left the outer rungs (arch@@ -33233,6 +33396,7 @@ class LlamaCppBackend:self._context_length = Noneself._effective_context_length = Noneself._max_context_length = None+ self._vram_fit_context_length = Noneself._reset_effective_parallel_slots()self._slot_save_dir = Noneself._slot_save_binary = None@@ -41297,6 +41461,10 @@ class LlamaCppBackend:from utils.utils import hf_env_offlinemodel_repo_path = resolve_bicodec_repo_path(local_files_only = hf_env_offline())+ # The codec holds memory the load-time fit never priced: VRAM on CUDA, and on a+ # unified-memory host the very pool the fit sized even when it lands on the CPU.+ # Cleared first, as the model is already listed while the codec allocates.+ self._vram_fit_context_length = NoneLlamaCppBackend._codec_mgr.load_codec(audio_type, device, model_repo_path = model_repo_path)logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 9b59515f6c7..8385e704fd1 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -35692,6 +35692,10 @@ def _slot_model_objects() -> list[dict]:_max_ctx = _positive_int_or_none(getattr(llama_backend, "max_context_length", None))if _max_ctx is not None:entry["max_context_length"] = _max_ctx+ # Not the running window (context_length): omitted when no fit vouched for one (#12571).+ _fit_ctx = _positive_int_or_none(getattr(llama_backend, "vram_fit_context_length", None))+ if _fit_ctx is not None:+ entry["vram_fit_context_length"] = _fit_ctx_native_ctx = _positive_int_or_none(getattr(llama_backend, "native_context_length", None))if _native_ctx is not None:entry["native_context_length"] = _native_ctxdiff --git a/studio/backend/tests/test_context_overflow_truncation.py b/studio/backend/tests/test_context_overflow_truncation.pyindex c7fbad6cda3..a6209583b92 100644--- a/studio/backend/tests/test_context_overflow_truncation.py+++ b/studio/backend/tests/test_context_overflow_truncation.py@@ -1051,6 +1051,39 @@ def test_v1_models_exposes_real_context_window(monkeypatch):assert entry["max_context_length"] == 262144+class _FakeSpillingLlamaBackend(_FakeLlamaBackend):+ # #12571: a 100k load on a Mac whose Metal budget only fits 8k without spilling.+ context_length = 100000+ max_context_length = 8192+ vram_fit_context_length = 8192+ native_context_length = 262144+++class _FakeNothingFitsLlamaBackend(_FakeSpillingLlamaBackend):+ # Weights over every budget: max_context_length is the Auto anchor, not a fit.+ vram_fit_context_length = None+++def test_v1_models_names_the_vram_fit_estimate(monkeypatch):+ monkeypatch.setattr(routes_mod, "get_llama_cpp_backend", lambda: _FakeSpillingLlamaBackend())+ monkeypatch.setattr(routes_mod, "get_inference_backend", lambda: _FakeEmptyBackend())+ entry = _openai_model_objects()[0]+ # The window the server runs at stays the one to size requests by.+ assert entry["context_length"] == 100000+ assert entry["vram_fit_context_length"] == 8192+ # The old name still answers, with the same estimate.+ assert entry["max_context_length"] == 8192+ assert entry["native_context_length"] == 262144+++def test_v1_models_claims_no_vram_fit_when_nothing_fits(monkeypatch):+ monkeypatch.setattr(routes_mod, "get_llama_cpp_backend", lambda: _FakeNothingFitsLlamaBackend())+ monkeypatch.setattr(routes_mod, "get_inference_backend", lambda: _FakeEmptyBackend())+ entry = _openai_model_objects()[0]+ assert "vram_fit_context_length" not in entry+ assert entry["max_context_length"] == 8192++def _conversation_with_big_reasoning(trace_chars: int = 40000) -> list[dict]:messages = [{"role": "system", "content": "sys"}]for index in range(8):diff --git a/studio/backend/tests/test_vram_fit_context_length.py b/studio/backend/tests/test_vram_fit_context_length.pynew file mode 100644index 00000000000..1d73defe34b--- /dev/null+++ b/studio/backend/tests/test_vram_fit_context_length.py@@ -0,0 +1,357 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""``vram_fit_context_length`` names only a ceiling a fit priced inside the budget (#12571).++``max_context_length`` also carries anchors and floors nobody measured (the Auto offload+context, the Metal fit floor, native when no fit ran), so mirroring it would advertise a+no-spill capacity the machine does not have.+"""++from __future__ import annotations++import importlib.util+import sys+from pathlib import Path++import pytest++_TESTS_DIR = Path(__file__).resolve().parent+_BACKEND_DIR = str(_TESTS_DIR.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)+if str(_TESTS_DIR) not in sys.path:+ sys.path.insert(0, str(_TESTS_DIR))+++def _load(module_name: str, file_name: str):+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)+ module = importlib.util.module_from_spec(spec)+ # The matrix module declares dataclasses, which look their module up by name.+ sys.modules[module_name] = module+ spec.loader.exec_module(module)+ return module+++_matrix = _load("_matrix_for_vram_fit_ctx", "test_auto_offload_ctx_platform_matrix.py")+_metal = _load("_metal_for_vram_fit_ctx", "test_metal_explicit_context_guard.py")++from core.inference.llama_cpp import _AUTO_OFFLOAD_CTX, _FIT_MIN_CTX # noqa: E402++GB = 1024**3+_REAL_POPEN = __import__("subprocess").Popen+++def _loaded(tmp_path, monkeypatch, platform, accelerator, **kwargs):+ backend, gguf = _matrix.cell_backend(tmp_path, monkeypatch, platform, accelerator, **kwargs)+ _matrix._launch(backend, gguf, n_ctx = 0)+ return backend++[email protected]("platform,accelerator", _matrix.MATRIX)+def test_a_model_that_fits_publishes_its_fitted_ceiling(+ tmp_path, monkeypatch, platform, accelerator+):+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator, model_fraction = _matrix.FITS)+ if accelerator.memory or accelerator.apple_budget_bytes:+ assert backend.vram_fit_context_length == backend.max_context_length+ assert backend.vram_fit_context_length > _AUTO_OFFLOAD_CTX+ else:+ # No device and no Metal budget: nothing was fitted, so nothing is claimed.+ assert backend.vram_fit_context_length is None++[email protected]("platform,accelerator", _matrix.MATRIX)+def test_an_overflowing_model_claims_no_fit(tmp_path, monkeypatch, platform, accelerator):+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator)+ assert backend.vram_fit_context_length is None+ if accelerator.memory:+ assert backend.max_context_length == _AUTO_OFFLOAD_CTX+ elif accelerator.apple_budget_bytes:+ assert backend.max_context_length == _FIT_MIN_CTX+++def test_an_unmeasurable_metal_load_claims_no_fit(tmp_path, monkeypatch):+ captured = _metal._launch(+ tmp_path,+ monkeypatch,+ n_ctx = 0,+ metal = True,+ can_estimate_kv = False,+ real_fit = True,+ budget_bytes = 24 * GB,+ )+ backend = captured["backend"]+ assert backend.max_context_length == _FIT_MIN_CTX+ assert backend.vram_fit_context_length is None+++def test_unload_forgets_the_fit(tmp_path, monkeypatch):+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")+ platform = _matrix.PLATFORMS[0]+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator, model_fraction = _matrix.FITS)+ assert backend.vram_fit_context_length is not None+ backend.unload_model()+ assert backend.vram_fit_context_length is None+++ "state",+ [+ {"_gpu_offload_active": False},+ {"_cpu_fallback_reason": "vulkan_startup_crash"},+ {"_arch_gate_forced_cpu": True},+ {"_gpu_memory_mode": "manual", "_gpu_layers": 0},+ ],+ ids = ["landed-on-cpu", "vulkan-cpu-fallback", "arch-gated-cpu", "manual-zero-layers"],+)+def test_a_child_on_cpu_claims_no_fit(state):+ from core.inference.llama_cpp import LlamaCppBackend++ backend = LlamaCppBackend()+ backend._vram_fit_context_length = 65536+ assert backend.vram_fit_context_length == 65536+ for name, value in state.items():+ setattr(backend, name, value)+ assert backend.vram_fit_context_length is None+++def test_a_recovered_crash_drops_the_ceiling_its_plan_priced(tmp_path, monkeypatch):+ import subprocess++ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")+ backend, gguf = _matrix.cell_backend(+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _matrix.FITS+ )+ healthy = iter([False, True])+ backend._wait_for_health = lambda timeout, **_kw: next(healthy)+ launches = []++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return _REAL_POPEN(cmd, **kwargs)+ launches.append(list(cmd))+ code = 1 if len(launches) == 1 else None+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "stdout": (),+ "returncode": code,+ "poll": lambda self: code,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ from unittest.mock import patch++ from core.inference.llama_cpp import GgufLoadIntent++ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ assert backend.load_model(+ GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 0)+ )+ assert len(launches) == 2, launches+ assert backend.max_context_length > _AUTO_OFFLOAD_CTX+ assert backend.vram_fit_context_length is None+++def test_a_zero_layer_metal_load_claims_no_fit(tmp_path, monkeypatch):+ def load(sub, **kwargs):+ (tmp_path / sub).mkdir()+ return _metal._launch(+ tmp_path / sub, monkeypatch, n_ctx = 0, metal = True, real_fit = True, **kwargs+ )["backend"]++ assert load("auto").vram_fit_context_length is not None+ manual = load("manual", gpu_memory_mode = "manual", gpu_layers = 0)+ assert manual.vram_fit_context_length is None+++def test_a_placement_that_raises_claims_no_fit(tmp_path, monkeypatch):+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")+ backend, gguf = _matrix.cell_backend(+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _matrix.FITS+ )++ def boom(*_a, **_kw):+ raise RuntimeError("selection failed")++ # Raises after the subset sweep priced a ceiling, inside the placement try.+ backend._select_gpus_split_aware = boom+ backend._select_gpus = boom+ captured = _matrix._launch(backend, gguf, n_ctx = 4096)+ assert ("--fit", "on") in zip(captured["cmd"], captured["cmd"][1:])+ assert backend.vram_fit_context_length is None+++ "extra_args",+ [+ ["--device", "none"],+ ["-ngl", "8"],+ ["-ot", "exps=CPU"],+ ["--lora", "adapter.gguf"],+ ["--no-kv-offload"],+ ],+ ids = ["cpu-device", "user-layers", "tensors-on-cpu", "lora-adapter", "kv-on-host"],+)+def test_a_user_placement_override_claims_no_fit(tmp_path, monkeypatch, extra_args):+ (tmp_path / "metal").mkdir()+ (tmp_path / "cuda").mkdir()+ metal = _metal._launch(+ tmp_path / "metal",+ monkeypatch,+ n_ctx = 0,+ metal = True,+ real_fit = True,+ extra_args = extra_args,+ )["backend"]+ assert metal.vram_fit_context_length is None++ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")+ backend, gguf = _matrix.cell_backend(+ tmp_path / "cuda",+ monkeypatch,+ _matrix.PLATFORMS[0],+ accelerator,+ model_fraction = _matrix.FITS,+ )+ _matrix._launch(backend, gguf, n_ctx = 0, extra_args = tuple(extra_args))+ assert backend.vram_fit_context_length is None+++# Needs both cards, so a selection or split that leaves one empty strands half the credit.+_TWO_CARD_FRACTION = 0.7+++def _two_card_load(tmp_path, monkeypatch, **load_kwargs):+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-multi")+ backend, gguf = _matrix.cell_backend(+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _TWO_CARD_FRACTION+ )+ load_kwargs.setdefault("n_ctx", 0)+ _matrix._launch(backend, gguf, **load_kwargs)+ return backend+++def test_a_two_card_fit_is_published(tmp_path, monkeypatch):+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is not None+++ "extra_args",+ [+ ["--device", "CUDA0"],+ ["--split-mode", "none"],+ ["--tensor-split", "100,1"],+ ["--fit", "on", "--fit-target", "8192"],+ ["--rpc", "192.168.1.2:50052"],+ ["--spec-draft-device", "CUDA1"],+ ["--mmproj-device", "CUDA1"],+ ["-otd", "blk=CUDA1"],+ ["--model-vocoder", "vocoder.gguf"],+ ],+ ids = [+ "narrower-device",+ "split-mode-none",+ "skewed-split",+ "user-fitter",+ "rpc",+ "draft-device",+ "mmproj-device",+ "draft-tensor-override",+ "vocoder",+ ],+)+def test_a_placement_narrower_than_the_fit_claims_no_fit(tmp_path, monkeypatch, extra_args):+ backend = _two_card_load(tmp_path, monkeypatch, extra_args = tuple(extra_args))+ assert backend.vram_fit_context_length is None+++def test_an_inherited_projector_claims_no_fit(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is None+++def test_a_gpu_ids_pin_ignores_the_device_env_it_clears(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0")+ backend = _two_card_load(tmp_path, monkeypatch, gpu_ids = [0, 1])+ assert backend.vram_fit_context_length is not None+++def test_a_one_card_device_pin_under_a_two_card_cap_claims_no_fit(tmp_path, monkeypatch):+ # The explicit context fits one card, so the selection credits one; the cap pooled two.+ backend = _two_card_load(tmp_path, monkeypatch, n_ctx = 2048, extra_args = ("--device", "CUDA0"))+ assert backend.vram_fit_context_length is None+++def test_an_inherited_rpc_claims_no_fit(tmp_path, monkeypatch):+ monkeypatch.setenv("LLAMA_ARG_RPC", "192.168.1.2:50052")+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is None+++def test_an_auto_discovered_projector_claims_no_fit(tmp_path, monkeypatch):+ backend = _two_card_load(tmp_path, monkeypatch, extra_args = ("--mmproj-auto",))+ assert backend.vram_fit_context_length is None+++def test_a_metal_fit_is_clamped_to_the_wired_limit(tmp_path, monkeypatch):+ # The free budget holds native; 128 MiB of wired room past the ~5 GiB of weights+ # holds about half of it at 1 KiB per token, and a wired limit under the weights none.+ def load(sub, wired_mib):+ (tmp_path / sub).mkdir()+ return _metal._launch(+ tmp_path / sub, monkeypatch, n_ctx = 1024, real_fit = True, wired_bytes = wired_mib * 2**20+ )["backend"]++ unbounded = load("unreadable", 0).vram_fit_context_length+ clamped = load("tight", 5120 + 128).vram_fit_context_length+ assert unbounded == _metal.NATIVE+ assert clamped is not None and clamped < unbounded+ assert load("under-weights", 4096).vram_fit_context_length is None+++def test_a_metal_fit_charges_swa_checkpoints(tmp_path, monkeypatch):+ # The fit floor fits the 20 GiB budget, but not once 15 GiB of checkpoints join it.+ def load(sub, ckpt_bytes):+ (tmp_path / sub).mkdir()+ return _metal._launch(+ tmp_path / sub,+ monkeypatch,+ n_ctx = 1024,+ real_fit = True,+ budget_bytes = 20 * GB,+ weights_bytes = GB,+ kv_per_token = 256 * 1024,+ extra_args = ["--ctx-checkpoints", "1"],+ wired_bytes = 24 * GB,+ ckpt_bytes = ckpt_bytes,+ )["backend"]++ assert load("no-checkpoints", 0).vram_fit_context_length is not None+ assert load("checkpoints", 15 * GB).vram_fit_context_length is None++[email protected]("cuda", [True, False], ids = ["codec-on-gpu", "codec-on-cpu"])+def test_a_tts_codec_drops_the_fit(monkeypatch, cuda):+ import torch++ from core.inference.llama_cpp import LlamaCppBackend++ seen = []++ class _Codecs:+ def load_codec(self, *_a, **_kw):+ # The field is already gone while the codec allocates.+ seen.append(backend.vram_fit_context_length)++ monkeypatch.setattr(LlamaCppBackend, "_codec_mgr", _Codecs())+ monkeypatch.setattr(torch.cuda, "is_available", lambda: cuda)+ backend = LlamaCppBackend()+ backend._vram_fit_context_length = 65536+ backend.init_audio_codec("snac", audio_codec_path = "unused")+ assert seen == [None] and backend.vram_fit_context_length is None