Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: name the /v1/models VRAM-fit context estimate (#12599)

Aziz Muminov1b5e516297fb
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 779463eea31..0b831df1205 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4556,6 +4556,32 @@ _CPU_PLACEMENT_PRESENCE_FLAGS = (_MOE_OFFLOAD_FLAGS - _CPU_MOE_COUNT_FLAGS) | fr
{"-ot", "--override-tensor"}
)
_CPU_PLACEMENT_FLAGS = _MOE_OFFLOAD_FLAGS | frozenset({"-ot", "--override-tensor"})
+# Pass-through that re-places what a fit priced: the fitter, split, remote (RPC) devices,
+# draft tensors moved onto a card of their own, or an unpriced vocoder GGUF.
+_VRAM_FIT_VOIDING_FLAGS = (
+ frozenset(
+ {
+ "-fit",
+ "--fit",
+ "--rpc",
+ "-otd",
+ "--override-tensor-draft",
+ "--spec-draft-override-tensor",
+ "-mv",
+ "--model-vocoder",
+ }
+ )
+ | _FIT_CONTROL_FLAGS
+ | _TENSOR_SPLIT_FLAGS
+ | _SPLIT_MODE_FLAGS
+)
+_VRAM_FIT_VOIDING_ENV_VARS = (
+ "LLAMA_ARG_FIT",
+ "LLAMA_ARG_RPC",
+ *_FIT_CONTROL_ENV_VARS,
+ "LLAMA_ARG_TENSOR_SPLIT",
+ "LLAMA_ARG_SPLIT_MODE",
+)
_THREAD_OVERRIDE_FLAGS = frozenset({"-t", "--threads"})
_GENERATION_CPU_AFFINITY_FLAGS = frozenset({"-C", "--cpu-mask", "-Cr", "--cpu-range"})
_TENSOR_SPLIT_FLAGS = frozenset({"--tensor-split", "-ts"})
@@ -6435,6 +6461,65 @@ def _subsequence_index(tokens: List[str], run: List[str], hint: int) -> int:
_CPU_DEVICE_VALUES = frozenset({"cpu", "none"})
+def _pass_through_overrides_placement(
+ extra_args: Optional[Iterable[str]],
+ source_env: Mapping[str, str],
+ credited: int,
+ *,
+ is_vulkan_backend: bool = False,
+ gpu_indices: Optional[Iterable[int]] = None,
+) -> bool:
+ """Whether pass-through args or env run a placement other than the one a fit priced.
+
+ ``extra_args`` / ``source_env`` are what the child really gets (a ``gpu_ids`` pin
+ strips its device flags and env twins first); ``credited`` is how many devices the
+ fit charged. See ``LlamaCppBackend._fit_derived_load_mode`` for each term.
+ """
+ # A --device naming FEWER devices than this fit charges leaves the credit
+ # paying for cards the child never opens. llama.cpp REPLACES the list on every
+ # occurrence (parse_device_list) and reads LLAMA_ARG_DEVICE before argv, so the
+ # last of the two wins. Counted, not name-matched: CUDA/ROCm ordinals are
+ # assigned after a visibility mask this launch has not written yet.
+ _dev_value = _extra_args_main_device(extra_args)
+ if _dev_value is None:
+ _dev_value = source_env.get("LLAMA_ARG_DEVICE")
+ _dev_names = [d.strip() for d in str(_dev_value or "").split(",") if d.strip()]
+ _credited = credited
+ gpu_indices = list(gpu_indices) if gpu_indices is not None else None
+ # DISTINCT names: llama.cpp appends every entry without deduplicating and
+ # keeps the list verbatim, so "CUDA0,CUDA0" is ONE physical pool listed twice,
+ # and counting it as two leaves the credit paying for a card that is not there.
+ _device_narrows = bool(_dev_names) and len(set(_dev_names)) < _credited
+ # Vulkan is the one backend where names ARE comparable (_vulkan_device_pin ->
+ # "Vulkan<i>"), so hold a surviving --device to an exact match. That catches
+ # what a count cannot: a same-sized list naming DIFFERENT cards, and a longer
+ # one adding devices this footprint never charged for.
+ _vulkan_pin_replaced = bool(
+ is_vulkan_backend
+ and gpu_indices is not None
+ and _dev_names
+ and {name.strip().lower() for name in _dev_names} != {f"vulkan{idx}" for idx in gpu_indices}
+ )
+ return bool(
+ _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
+ # And its env twin, the ONLY layer count the fitting path leaves: it
+ # emits no -ngl, only Manual mode clears LLAMA_ARG_N_GPU_LAYERS, and the
+ # fitter will not lower a count it did not set, so an inherited one stands
+ # and loads a prefix out of host RAM.
+ or _env_fixes_gpu_layers(source_env)
+ # With the env twin: only an explicit gpu_ids pin clears LLAMA_ARG_DEVICE,
+ # so an inherited "none" puts the whole load in host RAM.
+ or _device_selection_is_cpu(extra_args, source_env)
+ or _device_narrows
+ or _vulkan_pin_replaced
+ # Pass-through and last-wins: --split-mode none, or a --tensor-split that
+ # zeroes a device, leaves the pooled credit paying for cards held empty.
+ or split_policy_starves_devices(extra_args, _credited, source_env)
+ or _args_place_tensors_on_cpu(extra_args)
+ or _env_places_tensors_on_cpu(source_env)
+ )
+
+
def _device_selection_is_cpu(
extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None
) -> bool:
@@ -7869,6 +7954,8 @@ class LlamaCppBackend:
self._context_length: Optional[int] = None
self._effective_context_length: Optional[int] = None
self._max_context_length: Optional[int] = None
+ # Ceiling a fit priced inside the GPU / Metal budget; None for an anchor or floor.
+ self._vram_fit_context_length: Optional[int] = None
self._effective_parallel_slots: int = 1
# --parallel the last load asked for, before any fit-time reduction.
self._requested_n_parallel: int = 1
@@ -8429,6 +8516,19 @@ class LlamaCppBackend:
"""
return self._max_context_length or self._context_length
+ @property
+ def vram_fit_context_length(self) -> Optional[int]:
+ """Largest context the load-time fit priced inside GPU / Metal memory, else None."""
+ # A child wholly on CPU (--device none, Vulkan crash fallback, zero layers) holds no VRAM.
+ if (
+ self._gpu_offload_active is False
+ or self._cpu_fallback_reason
+ or self._arch_gate_forced_cpu
+ or (self._gpu_memory_mode == "manual" and self._gpu_layers == 0)
+ ):
+ return None
+ return self._vram_fit_context_length
+
@property
def native_context_length(self) -> Optional[int]:
"""Return the model's native context length from GGUF metadata."""
@@ -15573,48 +15673,13 @@ class LlamaCppBackend:
return None
rows = list(gpus or ())
source_env = os.environ if env is None else env
- # A --device naming FEWER devices than this fit charges leaves the credit
- # paying for cards the child never opens. llama.cpp REPLACES the list on every
- # occurrence (parse_device_list) and reads LLAMA_ARG_DEVICE before argv, so the
- # last of the two wins. Counted, not name-matched: CUDA/ROCm ordinals are
- # assigned after a visibility mask this launch has not written yet.
- _dev_value = _extra_args_main_device(extra_args)
- if _dev_value is None:
- _dev_value = source_env.get("LLAMA_ARG_DEVICE")
- _dev_names = [d.strip() for d in str(_dev_value or "").split(",") if d.strip()]
_credited = len(list(gpu_indices)) if gpu_indices is not None else len(rows)
- # DISTINCT names: llama.cpp appends every entry without deduplicating and
- # keeps the list verbatim, so "CUDA0,CUDA0" is ONE physical pool listed twice,
- # and counting it as two leaves the credit paying for a card that is not there.
- _device_narrows = bool(_dev_names) and len(set(_dev_names)) < _credited
- # Vulkan is the one backend where names ARE comparable (_vulkan_device_pin ->
- # "Vulkan<i>"), so hold a surviving --device to an exact match. That catches
- # what a count cannot: a same-sized list naming DIFFERENT cards, and a longer
- # one adding devices this footprint never charged for.
- _vulkan_pin_replaced = bool(
- is_vulkan_backend
- and gpu_indices is not None
- and _dev_names
- and {name.strip().lower() for name in _dev_names}
- != {f"vulkan{idx}" for idx in gpu_indices}
- )
- if (
- _extra_args_set_any_flag(extra_args, _GPU_LAYER_FLAGS)
- # And its env twin, the ONLY layer count the fitting path leaves: it
- # emits no -ngl, only Manual mode clears LLAMA_ARG_N_GPU_LAYERS, and the
- # fitter will not lower a count it did not set, so an inherited one stands
- # and loads a prefix out of host RAM.
- or _env_fixes_gpu_layers(source_env)
- # With the env twin: only an explicit gpu_ids pin clears LLAMA_ARG_DEVICE,
- # so an inherited "none" puts the whole load in host RAM.
- or _device_selection_is_cpu(extra_args, source_env)
- or _device_narrows
- or _vulkan_pin_replaced
- # Pass-through and last-wins: --split-mode none, or a --tensor-split that
- # zeroes a device, leaves the pooled credit paying for cards held empty.
- or split_policy_starves_devices(extra_args, _credited, source_env)
- or _args_place_tensors_on_cpu(extra_args)
- or _env_places_tensors_on_cpu(source_env)
+ if _pass_through_overrides_placement(
+ extra_args,
+ source_env,
+ _credited,
+ is_vulkan_backend = is_vulkan_backend,
+ gpu_indices = gpu_indices,
):
logger.info(
"Load mode: pass-through arguments override the placement this fit "
@@ -19191,6 +19256,7 @@ class LlamaCppBackend:
# Provisional until the server reports the budget it resolved (auto-size picks it from VRAM).
self._effective_context_length = maxtok or self._context_length
self._max_context_length = self._context_length or maxtok or None
+ self._vram_fit_context_length = None
healthy = self._wait_for_health(timeout = self._HEALTH_STALL_TIMEOUT_S, cancelled = cancelled)
if healthy:
@@ -25008,6 +25074,8 @@ class LlamaCppBackend:
_unmeasured_ctx_notice: Optional[str] = None
_cuda_ctx_notice: Optional[str] = None
_ctx_cap_fits = False
+ _vram_fit_ctx: Optional[int] = None
+ _flat_mtp_engages = False
total_by_idx: dict[int, int] = {}
_gpu_mem: list[tuple[int, int, int]] = []
model_size = None # set in the fit try; used by the APU RAM guard
@@ -26537,6 +26605,8 @@ class LlamaCppBackend:
if best_cap > 0:
max_available_ctx = best_cap
_ctx_cap_fits = True
+ # A forced GPU floor plans subsets this sweep did not hold to it.
+ _vram_fit_ctx = best_cap if _layer_min_gpus <= 1 else None
else:
# Weights exceed 90% of every GPU subset, so no
# context fits. Anchor the UI "safe zone" at the
@@ -27074,6 +27144,44 @@ class LlamaCppBackend:
_metal_ctx_warning = self._metal_context_pressure_message(
effective_ctx, _requested_mib, _apple_fit_budget_mib
)
+ # The paravirtual pin loads on CPU, so no Metal budget held anything.
+ _vram_fit_ctx = None if _paravirtual_cpu_forced else _apple_measured_ceiling
+ # The measured ceiling was priced without SWA checkpoints; re-fit with them.
+ if _vram_fit_ctx and (
+ _apple_footprint_mib(_vram_fit_ctx, _fit_ctx_checkpoints)
+ > _apple_fit_budget_mib
+ ):
+ _ckpt_cap = _apple_ctx_fit(_vram_fit_ctx, _FIT_FLOOR_MIN_CTX)
+ _vram_fit_ctx = (
+ _ckpt_cap
+ if _apple_footprint_mib(_ckpt_cap, _fit_ctx_checkpoints)
+ <= _apple_fit_budget_mib
+ else None
+ )
+ # Past the wired limit Metal refuses, so it is the harder ceiling when readable.
+ _fit_wired_mib = (
+ int(
+ self._apple_metal_wired_ceiling_bytes()
+ // (1024 * 1024)
+ * max(0.0, 1.0 - _flat_mtp_reserve)
+ )
+ if _vram_fit_ctx
+ else 0
+ )
+ if (
+ _fit_wired_mib > 0
+ and _apple_footprint_mib(_vram_fit_ctx, _fit_ctx_checkpoints)
+ > _fit_wired_mib
+ ):
+ _wired_cap = _apple_ctx_fit(
+ _vram_fit_ctx, _FIT_FLOOR_MIN_CTX, _fit_wired_mib
+ )
+ _vram_fit_ctx = (
+ _wired_cap
+ if _apple_footprint_mib(_wired_cap, _fit_ctx_checkpoints)
+ <= _fit_wired_mib
+ else None
+ )
# Unmeasured floors still rely on llama.cpp's fitter.
_metal_full_offload_pinned = bool(
_apple_host_embd and _apple_measured_ceiling is not None
@@ -27250,6 +27358,7 @@ class LlamaCppBackend:
# the launched plan cannot serve, which is the same ceiling /
# selection inversion #9492 removed, pointing the other way.
max_available_ctx = _ceiling[0]
+ _vram_fit_ctx = _ceiling[0]
# MoE experts in host RAM: a larger micro-batch amortises the expert
# copies of prompt processing (see _MOE_SPILL_N_UBATCH). The placement
@@ -27678,6 +27787,8 @@ class LlamaCppBackend:
# land after the dict exists but before the later terms are added
# to it, which would plan against a footprint missing them.
_spill_inputs = None
+ # Nor is a ceiling the fallback's own fitter may spill past.
+ _vram_fit_ctx = None
tp_tensor_split = None
effective_ctx = requested_ctx # fall back to original
@@ -30563,6 +30674,7 @@ class LlamaCppBackend:
run_cmd = _reverted
self._record_memory_state(run_cmd, env)
_did_fit_retry = True
+ self._vram_fit_context_length = None
continue
if (
not _did_fit_retry
@@ -30638,6 +30750,8 @@ class LlamaCppBackend:
run_cmd = _run
self._record_memory_state(_run, env)
_did_fit_retry = True
+ # The fit that vouched for the ceiling is what just failed.
+ self._vram_fit_context_length = None
continue
if (
not _did_fit_retry
@@ -30938,6 +31052,52 @@ class LlamaCppBackend:
self._max_context_length = (
max_available_ctx if max_available_ctx > 0 else self._effective_context_length
)
+ # The fit priced Unsloth's own placement and the base weights alone: a
+ # pass-through placement override, adapter or inherited projector is not.
+ if _vram_fit_ctx is not None:
+ _child_extras = (
+ self._strip_device_extra_args(extra_args)
+ if _gpu_ids_own_device_flags
+ else extra_args
+ )
+ _child_env = dict(os.environ)
+ if gpu_memory_mode == "manual":
+ self._clear_manual_placement_env(_child_env)
+ if _gpu_ids_own_device_flags:
+ self._clear_device_placement_env(_child_env)
+ if (
+ _pass_through_overrides_placement(
+ _child_extras,
+ _child_env,
+ len(gpu_indices) if gpu_indices is not None else len(gpus),
+ is_vulkan_backend = is_vulkan_backend,
+ gpu_indices = gpu_indices,
+ )
+ or _sidecar_adapter_bytes(extra_args) != 0
+ # A drafter the planner could only cover with a flat cushion.
+ or _flat_mtp_engages
+ or _child_env.get("LLAMA_ARG_MMPROJ")
+ or _child_env.get("LLAMA_ARG_MMPROJ_URL")
+ or (
+ not launch_mmproj_path
+ and extra_args_mmproj_auto(_child_extras, _child_env)
+ )
+ # A hand-set fitter or split ratio re-places what the fit priced.
+ or _extra_args_set_any_flag(_child_extras, _VRAM_FIT_VOIDING_FLAGS)
+ or any(_child_env.get(name) for name in _VRAM_FIT_VOIDING_ENV_VARS)
+ # Any surviving --device: the cap sweep may have pooled cards it leaves out.
+ or _extra_args_main_device(_child_extras) is not None
+ # And any companion (projector, drafter) pinned to a card of its own.
+ or any(
+ _extra_args_device(_child_extras, flags) is not None
+ for flags in _COMPANION_DEVICE_FLAG_GROUPS
+ )
+ or _child_env.get("LLAMA_ARG_DEVICE")
+ # --no-kv-offload keeps the whole cache in host RAM at any context.
+ or not _kv_offload_from_args(_child_extras, _child_env)
+ ):
+ _vram_fit_ctx = None
+ self._vram_fit_context_length = _vram_fit_ctx
# Past the host-RAM and offload advisories, so it can say the ceiling is
# the request without costing the user a memory warning it does not
@@ -31042,6 +31202,9 @@ class LlamaCppBackend:
healthy = False
try:
healthy = _spawn_and_wait(cmd)
+ if not healthy:
+ # Every recovery below relaunches a plan the fit never priced.
+ self._vram_fit_context_length = None
finally:
# Only once a child is up, where is_active takes over. An
# unconditional clear undid the re-arm and left the outer rungs (arch
@@ -33233,6 +33396,7 @@ class LlamaCppBackend:
self._context_length = None
self._effective_context_length = None
self._max_context_length = None
+ self._vram_fit_context_length = None
self._reset_effective_parallel_slots()
self._slot_save_dir = None
self._slot_save_binary = None
@@ -41297,6 +41461,10 @@ class LlamaCppBackend:
from utils.utils import hf_env_offline
model_repo_path = resolve_bicodec_repo_path(local_files_only = hf_env_offline())
+ # The codec holds memory the load-time fit never priced: VRAM on CUDA, and on a
+ # unified-memory host the very pool the fit sized even when it lands on the CPU.
+ # Cleared first, as the model is already listed while the codec allocates.
+ self._vram_fit_context_length = None
LlamaCppBackend._codec_mgr.load_codec(audio_type, device, model_repo_path = model_repo_path)
logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 9b59515f6c7..8385e704fd1 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -35692,6 +35692,10 @@ def _slot_model_objects() -> list[dict]:
_max_ctx = _positive_int_or_none(getattr(llama_backend, "max_context_length", None))
if _max_ctx is not None:
entry["max_context_length"] = _max_ctx
+ # Not the running window (context_length): omitted when no fit vouched for one (#12571).
+ _fit_ctx = _positive_int_or_none(getattr(llama_backend, "vram_fit_context_length", None))
+ if _fit_ctx is not None:
+ entry["vram_fit_context_length"] = _fit_ctx
_native_ctx = _positive_int_or_none(getattr(llama_backend, "native_context_length", None))
if _native_ctx is not None:
entry["native_context_length"] = _native_ctx
diff --git a/studio/backend/tests/test_context_overflow_truncation.py b/studio/backend/tests/test_context_overflow_truncation.py
index c7fbad6cda3..a6209583b92 100644
--- a/studio/backend/tests/test_context_overflow_truncation.py
+++ b/studio/backend/tests/test_context_overflow_truncation.py
@@ -1051,6 +1051,39 @@ def test_v1_models_exposes_real_context_window(monkeypatch):
assert entry["max_context_length"] == 262144
+class _FakeSpillingLlamaBackend(_FakeLlamaBackend):
+ # #12571: a 100k load on a Mac whose Metal budget only fits 8k without spilling.
+ context_length = 100000
+ max_context_length = 8192
+ vram_fit_context_length = 8192
+ native_context_length = 262144
+
+
+class _FakeNothingFitsLlamaBackend(_FakeSpillingLlamaBackend):
+ # Weights over every budget: max_context_length is the Auto anchor, not a fit.
+ vram_fit_context_length = None
+
+
+def test_v1_models_names_the_vram_fit_estimate(monkeypatch):
+ monkeypatch.setattr(routes_mod, "get_llama_cpp_backend", lambda: _FakeSpillingLlamaBackend())
+ monkeypatch.setattr(routes_mod, "get_inference_backend", lambda: _FakeEmptyBackend())
+ entry = _openai_model_objects()[0]
+ # The window the server runs at stays the one to size requests by.
+ assert entry["context_length"] == 100000
+ assert entry["vram_fit_context_length"] == 8192
+ # The old name still answers, with the same estimate.
+ assert entry["max_context_length"] == 8192
+ assert entry["native_context_length"] == 262144
+
+
+def test_v1_models_claims_no_vram_fit_when_nothing_fits(monkeypatch):
+ monkeypatch.setattr(routes_mod, "get_llama_cpp_backend", lambda: _FakeNothingFitsLlamaBackend())
+ monkeypatch.setattr(routes_mod, "get_inference_backend", lambda: _FakeEmptyBackend())
+ entry = _openai_model_objects()[0]
+ assert "vram_fit_context_length" not in entry
+ assert entry["max_context_length"] == 8192
+
+
def _conversation_with_big_reasoning(trace_chars: int = 40000) -> list[dict]:
messages = [{"role": "system", "content": "sys"}]
for index in range(8):
diff --git a/studio/backend/tests/test_vram_fit_context_length.py b/studio/backend/tests/test_vram_fit_context_length.py
new file mode 100644
index 00000000000..1d73defe34b
--- /dev/null
+++ b/studio/backend/tests/test_vram_fit_context_length.py
@@ -0,0 +1,357 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""``vram_fit_context_length`` names only a ceiling a fit priced inside the budget (#12571).
+
+``max_context_length`` also carries anchors and floors nobody measured (the Auto offload
+context, the Metal fit floor, native when no fit ran), so mirroring it would advertise a
+no-spill capacity the machine does not have.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIR = Path(__file__).resolve().parent
+_BACKEND_DIR = str(_TESTS_DIR.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+if str(_TESTS_DIR) not in sys.path:
+ sys.path.insert(0, str(_TESTS_DIR))
+
+
+def _load(module_name: str, file_name: str):
+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
+ module = importlib.util.module_from_spec(spec)
+ # The matrix module declares dataclasses, which look their module up by name.
+ sys.modules[module_name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+_matrix = _load("_matrix_for_vram_fit_ctx", "test_auto_offload_ctx_platform_matrix.py")
+_metal = _load("_metal_for_vram_fit_ctx", "test_metal_explicit_context_guard.py")
+
+from core.inference.llama_cpp import _AUTO_OFFLOAD_CTX, _FIT_MIN_CTX # noqa: E402
+
+GB = 1024**3
+_REAL_POPEN = __import__("subprocess").Popen
+
+
+def _loaded(tmp_path, monkeypatch, platform, accelerator, **kwargs):
+ backend, gguf = _matrix.cell_backend(tmp_path, monkeypatch, platform, accelerator, **kwargs)
+ _matrix._launch(backend, gguf, n_ctx = 0)
+ return backend
+
+
[email protected]("platform,accelerator", _matrix.MATRIX)
+def test_a_model_that_fits_publishes_its_fitted_ceiling(
+ tmp_path, monkeypatch, platform, accelerator
+):
+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator, model_fraction = _matrix.FITS)
+ if accelerator.memory or accelerator.apple_budget_bytes:
+ assert backend.vram_fit_context_length == backend.max_context_length
+ assert backend.vram_fit_context_length > _AUTO_OFFLOAD_CTX
+ else:
+ # No device and no Metal budget: nothing was fitted, so nothing is claimed.
+ assert backend.vram_fit_context_length is None
+
+
[email protected]("platform,accelerator", _matrix.MATRIX)
+def test_an_overflowing_model_claims_no_fit(tmp_path, monkeypatch, platform, accelerator):
+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator)
+ assert backend.vram_fit_context_length is None
+ if accelerator.memory:
+ assert backend.max_context_length == _AUTO_OFFLOAD_CTX
+ elif accelerator.apple_budget_bytes:
+ assert backend.max_context_length == _FIT_MIN_CTX
+
+
+def test_an_unmeasurable_metal_load_claims_no_fit(tmp_path, monkeypatch):
+ captured = _metal._launch(
+ tmp_path,
+ monkeypatch,
+ n_ctx = 0,
+ metal = True,
+ can_estimate_kv = False,
+ real_fit = True,
+ budget_bytes = 24 * GB,
+ )
+ backend = captured["backend"]
+ assert backend.max_context_length == _FIT_MIN_CTX
+ assert backend.vram_fit_context_length is None
+
+
+def test_unload_forgets_the_fit(tmp_path, monkeypatch):
+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")
+ platform = _matrix.PLATFORMS[0]
+ backend = _loaded(tmp_path, monkeypatch, platform, accelerator, model_fraction = _matrix.FITS)
+ assert backend.vram_fit_context_length is not None
+ backend.unload_model()
+ assert backend.vram_fit_context_length is None
+
+
+ "state",
+ [
+ {"_gpu_offload_active": False},
+ {"_cpu_fallback_reason": "vulkan_startup_crash"},
+ {"_arch_gate_forced_cpu": True},
+ {"_gpu_memory_mode": "manual", "_gpu_layers": 0},
+ ],
+ ids = ["landed-on-cpu", "vulkan-cpu-fallback", "arch-gated-cpu", "manual-zero-layers"],
+)
+def test_a_child_on_cpu_claims_no_fit(state):
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ backend = LlamaCppBackend()
+ backend._vram_fit_context_length = 65536
+ assert backend.vram_fit_context_length == 65536
+ for name, value in state.items():
+ setattr(backend, name, value)
+ assert backend.vram_fit_context_length is None
+
+
+def test_a_recovered_crash_drops_the_ceiling_its_plan_priced(tmp_path, monkeypatch):
+ import subprocess
+
+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")
+ backend, gguf = _matrix.cell_backend(
+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _matrix.FITS
+ )
+ healthy = iter([False, True])
+ backend._wait_for_health = lambda timeout, **_kw: next(healthy)
+ launches = []
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return _REAL_POPEN(cmd, **kwargs)
+ launches.append(list(cmd))
+ code = 1 if len(launches) == 1 else None
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "stdout": (),
+ "returncode": code,
+ "poll": lambda self: code,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: 0,
+ "kill": lambda self: None,
+ },
+ )()
+
+ from unittest.mock import patch
+
+ from core.inference.llama_cpp import GgufLoadIntent
+
+ with patch.object(subprocess, "Popen", side_effect = fake_popen):
+ assert backend.load_model(
+ GgufLoadIntent(gguf_path = str(gguf), model_identifier = "test", n_ctx = 0)
+ )
+ assert len(launches) == 2, launches
+ assert backend.max_context_length > _AUTO_OFFLOAD_CTX
+ assert backend.vram_fit_context_length is None
+
+
+def test_a_zero_layer_metal_load_claims_no_fit(tmp_path, monkeypatch):
+ def load(sub, **kwargs):
+ (tmp_path / sub).mkdir()
+ return _metal._launch(
+ tmp_path / sub, monkeypatch, n_ctx = 0, metal = True, real_fit = True, **kwargs
+ )["backend"]
+
+ assert load("auto").vram_fit_context_length is not None
+ manual = load("manual", gpu_memory_mode = "manual", gpu_layers = 0)
+ assert manual.vram_fit_context_length is None
+
+
+def test_a_placement_that_raises_claims_no_fit(tmp_path, monkeypatch):
+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")
+ backend, gguf = _matrix.cell_backend(
+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _matrix.FITS
+ )
+
+ def boom(*_a, **_kw):
+ raise RuntimeError("selection failed")
+
+ # Raises after the subset sweep priced a ceiling, inside the placement try.
+ backend._select_gpus_split_aware = boom
+ backend._select_gpus = boom
+ captured = _matrix._launch(backend, gguf, n_ctx = 4096)
+ assert ("--fit", "on") in zip(captured["cmd"], captured["cmd"][1:])
+ assert backend.vram_fit_context_length is None
+
+
+ "extra_args",
+ [
+ ["--device", "none"],
+ ["-ngl", "8"],
+ ["-ot", "exps=CPU"],
+ ["--lora", "adapter.gguf"],
+ ["--no-kv-offload"],
+ ],
+ ids = ["cpu-device", "user-layers", "tensors-on-cpu", "lora-adapter", "kv-on-host"],
+)
+def test_a_user_placement_override_claims_no_fit(tmp_path, monkeypatch, extra_args):
+ (tmp_path / "metal").mkdir()
+ (tmp_path / "cuda").mkdir()
+ metal = _metal._launch(
+ tmp_path / "metal",
+ monkeypatch,
+ n_ctx = 0,
+ metal = True,
+ real_fit = True,
+ extra_args = extra_args,
+ )["backend"]
+ assert metal.vram_fit_context_length is None
+
+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-single")
+ backend, gguf = _matrix.cell_backend(
+ tmp_path / "cuda",
+ monkeypatch,
+ _matrix.PLATFORMS[0],
+ accelerator,
+ model_fraction = _matrix.FITS,
+ )
+ _matrix._launch(backend, gguf, n_ctx = 0, extra_args = tuple(extra_args))
+ assert backend.vram_fit_context_length is None
+
+
+# Needs both cards, so a selection or split that leaves one empty strands half the credit.
+_TWO_CARD_FRACTION = 0.7
+
+
+def _two_card_load(tmp_path, monkeypatch, **load_kwargs):
+ accelerator = next(a for a in _matrix.ACCELERATORS if a.label == "nvidia-multi")
+ backend, gguf = _matrix.cell_backend(
+ tmp_path, monkeypatch, _matrix.PLATFORMS[0], accelerator, model_fraction = _TWO_CARD_FRACTION
+ )
+ load_kwargs.setdefault("n_ctx", 0)
+ _matrix._launch(backend, gguf, **load_kwargs)
+ return backend
+
+
+def test_a_two_card_fit_is_published(tmp_path, monkeypatch):
+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is not None
+
+
+ "extra_args",
+ [
+ ["--device", "CUDA0"],
+ ["--split-mode", "none"],
+ ["--tensor-split", "100,1"],
+ ["--fit", "on", "--fit-target", "8192"],
+ ["--rpc", "192.168.1.2:50052"],
+ ["--spec-draft-device", "CUDA1"],
+ ["--mmproj-device", "CUDA1"],
+ ["-otd", "blk=CUDA1"],
+ ["--model-vocoder", "vocoder.gguf"],
+ ],
+ ids = [
+ "narrower-device",
+ "split-mode-none",
+ "skewed-split",
+ "user-fitter",
+ "rpc",
+ "draft-device",
+ "mmproj-device",
+ "draft-tensor-override",
+ "vocoder",
+ ],
+)
+def test_a_placement_narrower_than_the_fit_claims_no_fit(tmp_path, monkeypatch, extra_args):
+ backend = _two_card_load(tmp_path, monkeypatch, extra_args = tuple(extra_args))
+ assert backend.vram_fit_context_length is None
+
+
+def test_an_inherited_projector_claims_no_fit(tmp_path, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")
+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is None
+
+
+def test_a_gpu_ids_pin_ignores_the_device_env_it_clears(tmp_path, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_DEVICE", "CUDA0")
+ backend = _two_card_load(tmp_path, monkeypatch, gpu_ids = [0, 1])
+ assert backend.vram_fit_context_length is not None
+
+
+def test_a_one_card_device_pin_under_a_two_card_cap_claims_no_fit(tmp_path, monkeypatch):
+ # The explicit context fits one card, so the selection credits one; the cap pooled two.
+ backend = _two_card_load(tmp_path, monkeypatch, n_ctx = 2048, extra_args = ("--device", "CUDA0"))
+ assert backend.vram_fit_context_length is None
+
+
+def test_an_inherited_rpc_claims_no_fit(tmp_path, monkeypatch):
+ monkeypatch.setenv("LLAMA_ARG_RPC", "192.168.1.2:50052")
+ assert _two_card_load(tmp_path, monkeypatch).vram_fit_context_length is None
+
+
+def test_an_auto_discovered_projector_claims_no_fit(tmp_path, monkeypatch):
+ backend = _two_card_load(tmp_path, monkeypatch, extra_args = ("--mmproj-auto",))
+ assert backend.vram_fit_context_length is None
+
+
+def test_a_metal_fit_is_clamped_to_the_wired_limit(tmp_path, monkeypatch):
+ # The free budget holds native; 128 MiB of wired room past the ~5 GiB of weights
+ # holds about half of it at 1 KiB per token, and a wired limit under the weights none.
+ def load(sub, wired_mib):
+ (tmp_path / sub).mkdir()
+ return _metal._launch(
+ tmp_path / sub, monkeypatch, n_ctx = 1024, real_fit = True, wired_bytes = wired_mib * 2**20
+ )["backend"]
+
+ unbounded = load("unreadable", 0).vram_fit_context_length
+ clamped = load("tight", 5120 + 128).vram_fit_context_length
+ assert unbounded == _metal.NATIVE
+ assert clamped is not None and clamped < unbounded
+ assert load("under-weights", 4096).vram_fit_context_length is None
+
+
+def test_a_metal_fit_charges_swa_checkpoints(tmp_path, monkeypatch):
+ # The fit floor fits the 20 GiB budget, but not once 15 GiB of checkpoints join it.
+ def load(sub, ckpt_bytes):
+ (tmp_path / sub).mkdir()
+ return _metal._launch(
+ tmp_path / sub,
+ monkeypatch,
+ n_ctx = 1024,
+ real_fit = True,
+ budget_bytes = 20 * GB,
+ weights_bytes = GB,
+ kv_per_token = 256 * 1024,
+ extra_args = ["--ctx-checkpoints", "1"],
+ wired_bytes = 24 * GB,
+ ckpt_bytes = ckpt_bytes,
+ )["backend"]
+
+ assert load("no-checkpoints", 0).vram_fit_context_length is not None
+ assert load("checkpoints", 15 * GB).vram_fit_context_length is None
+
+
[email protected]("cuda", [True, False], ids = ["codec-on-gpu", "codec-on-cpu"])
+def test_a_tts_codec_drops_the_fit(monkeypatch, cuda):
+ import torch
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ seen = []
+
+ class _Codecs:
+ def load_codec(self, *_a, **_kw):
+ # The field is already gone while the codec allocates.
+ seen.append(backend.vram_fit_context_length)
+
+ monkeypatch.setattr(LlamaCppBackend, "_codec_mgr", _Codecs())
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: cuda)
+ backend = LlamaCppBackend()
+ backend._vram_fit_context_length = 65536
+ backend.init_audio_codec("snac", audio_codec_path = "unused")
+ assert seen == [None] and backend.vram_fit_context_length is None