Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""A hand-set context a discrete GPU cannot hold must say so (issues #11349, #11336)."""
import logging
import sys
import types as _types
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
sys.path.insert(0, _TESTS_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: logging.getLogger(name)
_loggers_stub.__path__ = [str(Path(_BACKEND_DIR) / "loggers")]
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: logging.getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
from test_kv_cache_estimation import _backend_from_gguf # noqa: E402
GIB = 1024**3
MIB = 1024**2
QWEN3VL_8B = {
"block_count": 36,
"attention.head_count": 32,
"attention.head_count_kv": 8,
"attention.key_length": 128,
"attention.value_length": 128,
"embedding_length": 4096,
"context_length": 262144,
}
REPORTED_CTX = 65536
MEASURED_KV_MIB = 9216.0
WEIGHTS_BYTES = int(4.80 * GIB)
MMPROJ_BYTES = int(1.40 * GIB) # mmproj-F16, on GPU because -ngl 99
CARD_16GIB_MIB = 16.0 * 1024
@pytest.fixture(scope = "module")
def backend():
return _backend_from_gguf("qwen3vl", QWEN3VL_8B)
class TestKVEstimateAgainstHardware:
"""The estimator must reproduce what llama-server actually allocated."""
def test_kv_cache_estimate_matches_measured_allocation(self, backend):
kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
assert kv / MIB == pytest.approx(
MEASURED_KV_MIB, abs = 0.5
), f"expected the measured {MEASURED_KV_MIB} MiB allocation, got {kv / MIB:.1f} MiB"
assert kv / GIB == pytest.approx(9.0, abs = 0.01)
def test_kv_scales_linearly_with_context(self, backend):
"""Halving the context halves the cache: the term that dominates the budget."""
full = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
half = backend._estimate_kv_cache_bytes(REPORTED_CTX // 2, None)
assert half / GIB == pytest.approx(full / GIB / 2, rel = 0.01)
def test_q8_0_cache_is_substantially_smaller(self, backend):
"""q8_0 is the lever the planner never pulls; it must be worth suggesting."""
f16 = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
q8 = backend._estimate_kv_cache_bytes(REPORTED_CTX, "q8_0")
assert q8 < f16
assert q8 / f16 == pytest.approx(0.53, abs = 0.03)
assert (f16 - q8) / GIB > 4.0, "q8_0 must free more than 4 GiB to be worth offering"
class TestSixteenGibBudget:
"""The reported configuration must be recognised as not fitting a 16 GiB card."""
def test_reported_config_exceeds_a_16gib_card(self, backend):
kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
footprint = (
WEIGHTS_BYTES
+ MMPROJ_BYTES
+ int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))
+ backend._CUDA_CONTEXT_RESERVE_BYTES
+ kv
)
footprint_mib = footprint / MIB
assert (
footprint_mib > CARD_16GIB_MIB
), f"footprint {footprint_mib:.0f} MiB should exceed a {CARD_16GIB_MIB:.0f} MiB card"
assert 0 < footprint_mib - CARD_16GIB_MIB < 200
def test_the_budget_rejects_it_by_a_clear_margin(self, backend):
"""Against the 0.97 budget rather than the raw card, the verdict is unambiguous."""
kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
footprint_mib = (
WEIGHTS_BYTES
+ MMPROJ_BYTES
+ int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))
+ backend._CUDA_CONTEXT_RESERVE_BYTES
+ kv
) / MIB
budget_mib = CARD_16GIB_MIB * 0.97
assert footprint_mib > budget_mib
assert footprint_mib - budget_mib > 500
def test_q8_0_brings_the_same_config_under_the_card(self, backend):
"""The suggestion the notice makes has to be true."""
kv_q8 = backend._estimate_kv_cache_bytes(REPORTED_CTX, "q8_0")
footprint = (
WEIGHTS_BYTES
+ MMPROJ_BYTES
+ int(MMPROJ_BYTES * (backend._MMPROJ_VRAM_SAFETY - 1.0))
+ backend._CUDA_CONTEXT_RESERVE_BYTES
+ kv_q8
)
assert footprint / MIB < CARD_16GIB_MIB
def test_text_only_at_16gib_fits_which_is_why_it_was_hard_to_see(self, backend):
"""Without the mmproj the same context fits, matching the Linux run that passed."""
kv = backend._estimate_kv_cache_bytes(REPORTED_CTX, None)
footprint = WEIGHTS_BYTES + backend._CUDA_CONTEXT_RESERVE_BYTES + kv
assert footprint / MIB < CARD_16GIB_MIB
class TestOvercommitNotice:
"""The advisory itself: when it speaks, when it stays quiet, and what it says."""
def test_silent_when_the_context_fits(self):
assert LlamaCppBackend._cuda_context_overcommit_notice(8192, 65536, None) is None
def test_silent_without_a_measured_ceiling(self):
"""No ceiling means no fit maths ran; _unmeasured_context_notice owns that case."""
assert LlamaCppBackend._cuda_context_overcommit_notice(65536, 0, None) is None
assert LlamaCppBackend._cuda_context_overcommit_notice(0, 65536, None) is None
def test_silent_at_exactly_the_ceiling(self):
assert LlamaCppBackend._cuda_context_overcommit_notice(32768, 32768, None) is None
def test_reports_both_numbers(self):
msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)
assert msg is not None
assert "65,536" in msg and "32,768" in msg
def test_describes_the_planned_cpu_offload(self):
msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)
assert "moved to the CPU" in msg
assert "Sysmem" not in msg
def test_q8_hint_only_when_it_actually_fixes_it(self):
"""A suggestion that would not help is worse than no suggestion."""
helped = LlamaCppBackend._cuda_context_overcommit_notice(
65536, 32768, None, quantised_ctx_fits = True
)
assert "q8_0" in helped
not_helped = LlamaCppBackend._cuda_context_overcommit_notice(
65536, 32768, None, quantised_ctx_fits = False
)
assert "q8_0" not in not_helped
def test_no_q8_hint_when_the_cache_is_already_quantised(self):
msg = LlamaCppBackend._cuda_context_overcommit_notice(
65536, 32768, "q8_0", quantised_ctx_fits = True
)
assert "q8_0" not in msg
def test_never_raises_and_always_returns_text(self):
msg = LlamaCppBackend._cuda_context_overcommit_notice(
65536, 4096, None, quantised_ctx_fits = True
)
assert isinstance(msg, str) and msg.strip()
def test_is_an_advisory_not_a_refusal(self):
"""Unlike Metal, a discrete GPU spills, so this warns rather than refuses."""
msg = LlamaCppBackend._cuda_context_overcommit_notice(65536, 32768, None)
for word in ("cannot load", "refused", "aborted", "will not start"):
assert word not in msg.lower()
def _launch_explicit_ctx(tmp_path, monkeypatch, model_gb, n_ctx, **load_kwargs):
from test_llama_cpp_placement import _backend as _placement_backend, _launch
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
backend._get_gguf_size_bytes = lambda _path: int(model_gb * GIB)
backend._can_estimate_kv = lambda: True
backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024
backend._estimate_compute_buffer_bytes = lambda **k: 1
cmd = _launch(backend, gguf, n_ctx = n_ctx, **load_kwargs)["cmd"]
assert cmd[cmd.index("--fit") + 1] == "on"
return backend.last_load_warning or ""
def test_the_notice_names_a_context_that_fits(tmp_path, monkeypatch):
warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)
assert "does not fit in this GPU's memory" in warning
def test_no_context_is_offered_when_the_weights_alone_overflow(tmp_path, monkeypatch):
"""40 GB of weights on a 24 GB card: no context fits, so none may be offered."""
warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 40, n_ctx = 32768)
assert "The largest that fits is" not in warning
def test_a_windows_cuda_build_is_not_told_the_driver_spills(tmp_path, monkeypatch):
"""--fit on and spill plans place host tensors on purpose; WDDM advice cannot undo that."""
monkeypatch.setattr(LlamaCppBackend, "_sysmem_fallback_risk", staticmethod(lambda b = None: True))
warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)
assert "moved to the CPU" in warning
assert "NVIDIA Control Panel" not in warning
@pytest.mark.parametrize(
"q8_scratch_gib, extra_args, offered",
[(0, [], True), (8, [], False), (0, ["--flash-attn", "off"], False)],
)
def test_the_q8_hint_prices_q8_compute_scratch(
tmp_path, monkeypatch, q8_scratch_gib, extra_args, offered
):
"""q8_0 adds dequant scratch; the hint must hold with it, not with the f16 figure."""
from test_llama_cpp_placement import _backend as _placement_backend, _launch
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
backend._get_gguf_size_bytes = lambda _path: 14 * GIB
backend._can_estimate_kv = lambda: True
backend._estimate_kv_cache_bytes = (
lambda ctx, cache = None, *a, **k: int(ctx)
* (32 if str(cache).startswith("q8") else 64)
* 1024
)
backend._estimate_compute_buffer_bytes = lambda **k: 1
backend._compute_buffer_ctx_bytes = lambda ctx, ub, cache_type = None, **k: (
q8_scratch_gib * GIB if str(cache_type).startswith("q8") else 0
)
_launch(backend, gguf, n_ctx = 196608, extra_args = extra_args)
warning = backend.last_load_warning or ""
assert "does not fit in this GPU's memory" in warning
assert ("q8_0" in warning) is offered
def test_no_notice_when_the_kv_cache_stays_on_the_host(tmp_path, monkeypatch):
"""-nkvo keeps the cache off the GPU, so the priced KV overflow does not happen."""
warning = _launch_explicit_ctx(
tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072, extra_args = ["-nkvo"]
)
assert "does not fit in this GPU's memory" not in warning
@pytest.mark.parametrize(
"extra_args", [["--device", "none"], ["-ot", "exps=CPU"], ["--gpu-layers", "10"]]
)
def test_no_notice_for_a_deliberate_cpu_placement(tmp_path, monkeypatch, extra_args):
warning = _launch_explicit_ctx(
tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072, extra_args = extra_args
)
assert "does not fit in this GPU's memory" not in warning
def test_no_notice_when_an_inherited_env_fixes_the_layer_count(tmp_path, monkeypatch):
monkeypatch.setenv("LLAMA_ARG_N_GPU_LAYERS", "20")
warning = _launch_explicit_ctx(tmp_path, monkeypatch, model_gb = 18, n_ctx = 131072)
assert "does not fit in this GPU's memory" not in warning
def test_no_largest_context_claimed_at_the_native_ceiling(tmp_path, monkeypatch):
"""Past the native window the sweep stops at native, so the true maximum is unknown."""
monkeypatch.setattr(LlamaCppBackend, "_context_length", 32768, raising = False)
from test_llama_cpp_placement import _backend as _placement_backend, _launch
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
backend._read_gguf_metadata = lambda _path: setattr(backend, "_context_length", 32768)
backend._get_gguf_size_bytes = lambda _path: 14 * GIB
backend._can_estimate_kv = lambda: True
backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 96 * 1024
backend._estimate_compute_buffer_bytes = lambda **k: 1
_launch(backend, gguf, n_ctx = 131072)
assert "The largest that fits is 32,768" not in (backend.last_load_warning or "")
@pytest.mark.parametrize("extra_args, warned", [(["--fit", "off"], False), (["--fit=on"], True)])
def test_a_user_fit_override_decides_the_notice(tmp_path, monkeypatch, extra_args, warned):
from test_llama_cpp_placement import _backend as _placement_backend, _launch
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
backend, gguf = _placement_backend(tmp_path, vulkan = False, memory = [(0, 24_576, 24_576)])
backend._get_gguf_size_bytes = lambda _path: 18 * GIB
backend._can_estimate_kv = lambda: True
backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024
backend._estimate_compute_buffer_bytes = lambda **k: 1
_launch(backend, gguf, n_ctx = 131072, extra_args = extra_args)
assert ("does not fit in this GPU's memory" in (backend.last_load_warning or "")) is warned