Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""An unmeasured context fallback is a guess, so it must not overrule an explicit request."""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
_TESTS_DIR = Path(__file__).resolve().parent
_BACKEND_DIR = str(_TESTS_DIR.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
if str(_TESTS_DIR) not in sys.path:
sys.path.insert(0, str(_TESTS_DIR))
def _load(module_name: str, file_name: str):
spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
# The no-GPU probe in this harness IS the Metal condition.
_metal = _load("_metal_guard_for_unmeasured_ctx", "test_metal_explicit_context_guard.py")
_metal_launch = _metal._launch
_ctx_values = _metal._ctx_values
from core.inference.llama_cpp import ( # noqa: E402
_FIT_MIN_CTX,
_TP_UNMEASURED_CTX,
LlamaCppBackend,
)
GB = 1024**3
NATIVE = 262144
TWO_80 = [(0, 80 * 1024), (1, 80 * 1024)]
def _measurable() -> LlamaCppBackend:
backend = LlamaCppBackend()
backend._n_layers = 28
backend._embedding_length = 1024
backend._n_heads = 16
backend._n_kv_heads = 8
backend._kv_key_length = 128
backend._kv_value_length = 128
backend._context_length = NATIVE
return backend
def _unmeasurable() -> LlamaCppBackend:
backend = LlamaCppBackend()
backend._context_length = NATIVE
assert not backend._can_estimate_kv()
return backend
class TestTheTensorParallelFallback:
def test_an_explicit_request_is_honoured(self):
effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
)
assert effective == NATIVE
assert ceiling >= effective
def test_auto_still_takes_the_conservative_fallback(self):
effective, ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE
)
assert effective == _TP_UNMEASURED_CTX
assert ceiling == _TP_UNMEASURED_CTX
def test_a_measured_cap_still_binds_an_explicit_request(self):
"""--fit is a no-op in tensor mode, so only a measured cap prevents startup OOM."""
tight = [(0, 6 * 1024), (1, 6 * 1024)]
capped, _ceiling, *_ = _measurable()._plan_tensor_parallel(
tight, 8 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
)
assert 0 < capped < NATIVE
def test_a_measurable_plan_that_fits_is_unchanged_by_the_flag(self):
with_flag = _measurable()._plan_tensor_parallel(
TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE, explicit_ctx = True
)
without = _measurable()._plan_tensor_parallel(
TWO_80, 20 * GB, NATIVE, max_target_ctx = NATIVE
)
assert with_flag == without
def test_the_flag_defaults_off_for_every_existing_caller(self):
assert _unmeasurable()._plan_tensor_parallel(TWO_80, 20 * GB, NATIVE)[0] == (
_TP_UNMEASURED_CTX
)
@pytest.mark.parametrize("target", [0, -1])
def test_no_request_is_not_a_request(self, target):
effective, _ceiling, *_ = _unmeasurable()._plan_tensor_parallel(
TWO_80, 20 * GB, target, explicit_ctx = True
)
assert effective == _TP_UNMEASURED_CTX
class TestTheNotice:
def test_it_names_the_context_that_was_launched(self):
message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
assert message is not None
assert "262,144" in message
def test_it_says_the_maximum_is_not_a_measurement(self):
message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
assert "not a measured ceiling" in message
def test_it_does_not_read_as_a_refusal(self):
message = LlamaCppBackend._unmeasured_context_notice(NATIVE)
assert "launched as requested" in message
for blame in ("cannot", "refus", "not supported"):
assert blame not in message.lower()
@pytest.mark.parametrize("cache_type", [None, "", "f16", "fp16"])
def test_the_kv_hint_is_offered_on_an_unquantized_cache(self, cache_type):
assert "q8_0" in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)
@pytest.mark.parametrize("cache_type", ["q8_0", "q4_0"])
def test_it_is_not_offered_once_the_cache_is_quantized(self, cache_type):
assert "q8_0" not in LlamaCppBackend._unmeasured_context_notice(NATIVE, cache_type)
@pytest.mark.parametrize("requested", [0, -1])
def test_nothing_to_report_without_a_request(self, requested):
assert LlamaCppBackend._unmeasured_context_notice(requested) is None
def test_it_does_not_cost_the_load_a_memory_warning(self):
"""_record_load_warning keeps one slot, so the ceiling notice is held apart from memory warnings."""
import ast
import inspect
import textwrap
source = inspect.getsource(LlamaCppBackend.load_model)
func = ast.parse(textwrap.dedent(source)).body[0]
for node in ast.walk(func):
if not (
isinstance(node, ast.Call)
and getattr(node.func, "attr", None) == "_record_load_warning"
):
continue
for arg in node.args:
assert not (
isinstance(arg, ast.Call)
and getattr(arg.func, "attr", None) == "_unmeasured_context_notice"
), f"the notice claims the warning slot at line {node.lineno}"
held = [
node.lineno
for node in ast.walk(func)
if isinstance(node, ast.Assign)
and getattr(node.targets[0], "id", None) == "_unmeasured_ctx_notice"
and isinstance(node.value, ast.Call)
]
flushed = [
node.lineno
for node in ast.walk(func)
if isinstance(node, ast.If)
and isinstance(node.test, ast.Name)
and node.test.id == "_unmeasured_ctx_notice"
]
# Both arms hold it, and there is one flush.
assert len(held) == 2, held
assert len(flushed) == 1, flushed
# The flush must come after this attempt's memory advisories; the arch-crash retry's
# pair belongs to a later attempt.
advisories = {
arg.id: node.lineno
for node in ast.walk(func)
if isinstance(node, ast.Call)
and getattr(node.func, "attr", None) == "_record_load_warning"
for arg in node.args
if isinstance(arg, ast.Name)
}
for name in ("_ram_msg", "_offload_msg"):
assert name in advisories, (name, sorted(advisories))
assert flushed[0] > advisories[name], (name, flushed, advisories)
class TestTheMetalArm:
"""Simulated with an empty GPU probe; no Metal device is exercised."""
def _launch_unmeasurable(self, tmp_path, monkeypatch, **kwargs):
return _metal_launch(
tmp_path,
monkeypatch,
metal = True,
can_estimate_kv = False,
real_fit = True,
budget_bytes = 24 * GB,
native = NATIVE,
**kwargs,
)
def test_an_explicit_request_reaches_llama_server(self, tmp_path, monkeypatch):
captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
assert _ctx_values(captured["cmd"])[-1] == str(NATIVE)
def test_the_published_ceiling_is_not_below_what_launched(self, tmp_path, monkeypatch):
"""The floor used to be published while the child ran at the request."""
captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
backend = captured["backend"]
assert backend.max_context_length >= NATIVE
assert backend.native_context_length == NATIVE
def test_the_load_says_the_ceiling_was_never_measured(self, tmp_path, monkeypatch):
captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = NATIVE)
assert "not a measured ceiling" in (captured["backend"].last_load_warning or "")
def test_auto_keeps_the_floor(self, tmp_path, monkeypatch):
captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)
assert _ctx_values(captured["cmd"])[-1] == str(_FIT_MIN_CTX)
assert captured["backend"].max_context_length == _FIT_MIN_CTX
def test_auto_is_not_told_anything(self, tmp_path, monkeypatch):
"""Auto made no request; an advisory on every unreadable header is noise."""
captured = self._launch_unmeasurable(tmp_path, monkeypatch, n_ctx = 0)
assert captured["backend"].last_load_warning is None
def test_a_measured_ceiling_still_refuses(self, tmp_path, monkeypatch):
with pytest.raises(RuntimeError, match = "unified"):
_metal_launch(
tmp_path,
monkeypatch,
n_ctx = 32768,
metal = True,
can_estimate_kv = True,
)