Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The raw GGUF _context_length must never be overwritten by VRAM-capping logic."""
import io
import json
import re
import struct
import sys
import types as _types
from pathlib import Path
from unittest.mock import patch
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **kw: __import__("logging").getLogger(
a[0] if a else __name__
)
sys.modules.setdefault("structlog", _structlog_stub)
_httpx_stub = _types.ModuleType("httpx")
for _exc_name in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
):
setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {}))
class _FakeTimeout:
def __init__(self, *a, **kw):
pass
_httpx_stub.Timeout = _FakeTimeout
_httpx_stub.Client = type(
"Client",
(),
{
"__init__": lambda self, **kw: None,
"__enter__": lambda self: self,
"__exit__": lambda self, *a: None,
},
)
# Only when httpx is absent: this stub lacks Response and would break starlette.testclient.
try:
import httpx # noqa: F401
except ImportError:
sys.modules.setdefault("httpx", _httpx_stub)
from core.inference.llama_cpp import LlamaCppBackend
from models.inference import LoadResponse, InferenceStatusResponse
def _write_kv(buf: io.BytesIO, key: str, value, vtype: int) -> None:
"""Append a single GGUF KV pair to *buf*."""
key_bytes = key.encode("utf-8")
buf.write(struct.pack("<Q", len(key_bytes)))
buf.write(key_bytes)
buf.write(struct.pack("<I", vtype))
# GGUF value types: 4 = UINT32, 10 = UINT64, 8 = STRING.
if vtype == 4:
buf.write(struct.pack("<I", value))
elif vtype == 10:
buf.write(struct.pack("<Q", value))
elif vtype == 8:
val_bytes = value.encode("utf-8")
buf.write(struct.pack("<Q", len(val_bytes)))
buf.write(val_bytes)
else:
raise ValueError(f"Unsupported vtype in test helper: {vtype}")
def make_gguf(
tmp_path: Path,
arch: str,
kvs: list,
*,
arch_first: bool = True,
filename: str = "test.gguf",
) -> str:
"""Create a minimal valid GGUF v3 binary in *tmp_path*."""
buf = io.BytesIO()
buf.write(struct.pack("<I", 0x46554747))
buf.write(struct.pack("<I", 3))
buf.write(struct.pack("<Q", 0))
ordered = []
arch_entry = ("general.architecture", arch, 8)
if arch_first:
ordered.append(arch_entry)
for suffix, val, vt in kvs:
ordered.append((f"{arch}.{suffix}", val, vt))
if not arch_first:
ordered.append(arch_entry)
buf.write(struct.pack("<Q", len(ordered)))
for key, val, vt in ordered:
_write_kv(buf, key, val, vt)
path = tmp_path / filename
path.write_bytes(buf.getvalue())
return str(path)
@pytest.fixture
def backend():
"""Create a fresh LlamaCppBackend with side effects disabled."""
with patch.object(LlamaCppBackend, "_kill_orphaned_servers"):
with patch("atexit.register"):
return LlamaCppBackend()
class TestNativeContextLengthProperty:
"""Tests the new `native_context_length` property on LlamaCppBackend."""
def test_none_on_fresh_backend(self, backend):
"""Returns None when no model loaded."""
assert backend.native_context_length is None
def test_returns_raw_gguf_value(self, backend):
"""Directly returns _context_length when set."""
backend._context_length = 131072
assert backend.native_context_length == 131072
def test_not_capped_by_effective(self, backend):
"""native_context_length ignores _effective_context_length."""
backend._context_length = 131072
backend._effective_context_length = 32768
assert backend.native_context_length == 131072
def test_not_capped_by_max(self, backend):
"""native_context_length ignores _max_context_length."""
backend._context_length = 131072
backend._max_context_length = 65536
assert backend.native_context_length == 131072
def test_none_after_unload(self, backend):
"""After unload_model(), returns None."""
backend._context_length = 131072
assert backend.native_context_length == 131072
backend.unload_model()
assert backend.native_context_length is None
def test_after_gguf_parse(self, tmp_path, backend):
"""Synthetic GGUF with context_length=16384 populates the property."""
path = make_gguf(
tmp_path,
"llama",
[("context_length", 16384, 4)],
)
backend._read_gguf_metadata(path)
assert backend.native_context_length == 16384
def test_resets_between_parses(self, tmp_path, backend):
"""Second GGUF without context_length resets native to None."""
path_a = make_gguf(
tmp_path,
"llama",
[("context_length", 16384, 4)],
filename = "a.gguf",
)
backend._read_gguf_metadata(path_a)
assert backend.native_context_length == 16384
path_b = make_gguf(
tmp_path,
"gpt2",
[("block_count", 12, 4)],
filename = "b.gguf",
)
backend._read_gguf_metadata(path_b)
assert backend.native_context_length is None
class TestContextValueSeparation:
"""_context_length is never overwritten by VRAM logic."""
def test_preserved_after_effective_set(self, backend):
"""Setting _effective_context_length does not change _context_length."""
backend._context_length = 131072
backend._effective_context_length = 32768
assert backend._context_length == 131072
assert backend.native_context_length == 131072
def test_ordering_when_capped(self, backend):
"""native >= max >= effective holds when VRAM-capped."""
backend._context_length = 131072
backend._max_context_length = 65536
backend._effective_context_length = 32768
assert backend.native_context_length >= backend.max_context_length
assert backend.max_context_length >= backend.context_length
def test_all_equal_when_uncapped(self, backend):
"""All three equal when no VRAM constraint."""
backend._context_length = 8192
assert backend.native_context_length == 8192
assert backend.max_context_length == 8192
assert backend.context_length == 8192
def test_fit_context_does_not_modify(self, backend):
"""_fit_context_to_vram() does not touch _context_length."""
backend._context_length = 131072
backend._n_layers = 32
backend._n_kv_heads = 8
backend._n_heads = 32
backend._embedding_length = 4096
original = backend._context_length
result = backend._fit_context_to_vram(
requested_ctx = 131072,
available_mib = 512,
model_size_bytes = 0,
)
assert backend._context_length == original
assert backend.native_context_length == original
assert result <= 131072
def test_native_gt_context_when_capped(self, backend):
"""native_context_length > context_length after VRAM capping."""
backend._context_length = 131072
backend._effective_context_length = 16384
assert backend.native_context_length > backend.context_length
class TestPydanticModels:
"""Tests native_context_length field on Pydantic models."""
def test_load_response_has_field(self):
"""Field exists in LoadResponse.model_fields."""
assert "native_context_length" in LoadResponse.model_fields
assert "context_length" in LoadResponse.model_fields
def test_load_response_defaults_none(self):
"""Omitting native_context_length defaults to None."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
)
assert resp.native_context_length is None
def test_load_response_accepts_int(self):
"""native_context_length=131072 stores correctly."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
native_context_length = 131072,
)
assert resp.native_context_length == 131072
def test_load_response_json_null(self):
"""None serializes to JSON null."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
)
data = json.loads(resp.model_dump_json())
assert data["native_context_length"] is None
def test_load_response_json_int(self):
"""131072 serializes to JSON number."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
native_context_length = 131072,
)
data = json.loads(resp.model_dump_json())
assert data["native_context_length"] == 131072
def test_status_response_has_field(self):
"""Field exists in InferenceStatusResponse.model_fields."""
assert "native_context_length" in InferenceStatusResponse.model_fields
assert "context_length" in InferenceStatusResponse.model_fields
def test_status_response_has_chat_template_field(self):
"""Status includes chat_template so the UI can rehydrate after refresh."""
assert "chat_template" in InferenceStatusResponse.model_fields
def test_status_response_defaults_none(self):
"""Omitting native_context_length defaults to None."""
resp = InferenceStatusResponse()
assert resp.native_context_length is None
def test_status_response_chat_template_roundtrip(self):
"""chat_template serializes and validates as part of status."""
resp = InferenceStatusResponse(chat_template = "{{ messages }}")
roundtripped = InferenceStatusResponse.model_validate_json(resp.model_dump_json())
assert roundtripped.chat_template == "{{ messages }}"
def test_roundtrip_preserves_value(self):
"""model_validate_json(model_dump_json()) round-trips."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
native_context_length = 131072,
)
roundtripped = LoadResponse.model_validate_json(resp.model_dump_json())
assert roundtripped.native_context_length == 131072
def test_context_length_roundtrip(self):
"""Runtime context_length serializes for non-GGUF/hub models."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
context_length = 8192,
)
roundtripped = LoadResponse.model_validate_json(resp.model_dump_json())
assert roundtripped.context_length == 8192
class TestRouteCompleteness:
"""All response construction sites in routes/inference.py include native_context_length."""
@pytest.fixture(autouse = True)
def _load_source(self):
"""Read routes/inference.py source once."""
routes_path = Path(__file__).resolve().parent.parent / "routes" / "inference.py"
self._source = routes_path.read_text(encoding = "utf-8")
def _find_construction_blocks(self, class_name: str) -> list[str]:
"""Extract all code blocks that construct a given response class."""
blocks = []
idx = 0
while True:
start = self._source.find(f"{class_name}(", idx)
if start == -1:
break
depth = 0
end = start
for i, ch in enumerate(self._source[start:], start):
if ch == "(":
depth += 1
elif ch == ")":
depth -= 1
if depth == 0:
end = i + 1
break
blocks.append(self._source[start:end])
idx = end
return blocks
def test_gguf_load_responses_have_field(self):
"""Every GGUF LoadResponse (is_gguf = True) includes native_context_length."""
blocks = self._find_construction_blocks("LoadResponse")
gguf_blocks = [b for b in blocks if "is_gguf = True" in b or "is_gguf=True" in b]
assert (
len(gguf_blocks) == 1
), f"Expected one shared GGUF LoadResponse block, found {len(gguf_blocks)}"
for i, block in enumerate(gguf_blocks):
assert (
"_llama_runtime_fields(llama_backend)" in block
), f"GGUF LoadResponse block #{i} missing runtime fields:\n{block[:200]}"
assert "for name in _InferenceRuntimeFields.model_fields" in self._source
def test_non_gguf_load_responses_report_the_native_window(self):
"""Non-GGUF LoadResponse must set native_context_length, or the control blanks on a None default."""
blocks = self._find_construction_blocks("LoadResponse")
npu = [b for b in blocks if "is_npu = True" in b]
non_gguf = [
b
for b in blocks
if "is_gguf = True" not in b and "is_gguf=True" not in b and b not in npu
]
assert non_gguf, "Expected at least one non-GGUF LoadResponse block"
for block in non_gguf:
for field in ("native_context_length", "max_context_length"):
assert re.search(
rf"{field} = _positive_int_or_none\(\s*_model_info\.get\(", block
), f"Non-GGUF LoadResponse should read {field} from _model_info:\n{block[:200]}"
assert npu, "Expected the NPU LoadResponse block"
for block in npu:
for field in ("native_context_length", "max_context_length"):
assert f"{field} = model.max_context_length" in block, block[:200]
def test_non_gguf_load_responses_set_runtime_context_length(self):
"""Non-GGUF LoadResponse blocks report runtime context_length."""
blocks = self._find_construction_blocks("LoadResponse")
non_gguf = [b for b in blocks if "is_gguf = True" not in b and "is_gguf=True" not in b]
assert non_gguf, "Expected at least one non-GGUF LoadResponse block"
for block in non_gguf:
assert (
"context_length" in block
), f"Non-GGUF LoadResponse should set context_length:\n{block[:200]}"
def test_status_path(self):
"""InferenceStatusResponse built from llama_backend must carry the runtime fields, however bound."""
blocks = self._find_construction_blocks("InferenceStatusResponse")
found = False
for block in blocks:
if "llama_backend" not in block:
continue
if "_llama_runtime_fields(llama_backend)" in block:
found = True
break
if "**_runtime_fields" in block:
# Only counts if that dict is the helper's, not any local name.
assert (
"_runtime_fields = _llama_runtime_fields(llama_backend)" in self._source
), "**_runtime_fields is not built from _llama_runtime_fields(llama_backend)"
found = True
break
assert found, "No InferenceStatusResponse block with llama_backend has runtime fields"
assert "for name in _InferenceRuntimeFields.model_fields" in self._source
def test_non_gguf_status_path_reports_runtime_context_length(self):
"""Non-GGUF InferenceStatusResponse reports context_length from model_info."""
blocks = self._find_construction_blocks("InferenceStatusResponse")
found = False
for block in blocks:
if "is_gguf = False" in block and "context_length" in block:
found = True
break
assert found, "No non-GGUF InferenceStatusResponse block with context_length"
def test_openai_models_listing_reports_context_length(self):
"""/v1/models includes context_length when the backend knows it."""
assert 'entry["context_length"]' in self._source
assert 'model_info.get("context_length")' in self._source
class TestNativeContextEdgeCases:
"""Edge cases for native_context_length."""
def test_context_length_zero(self, tmp_path, backend):
"""GGUF context_length=0 returns 0, not None."""
path = make_gguf(tmp_path, "llama", [("context_length", 0, 4)])
backend._read_gguf_metadata(path)
assert backend.native_context_length == 0
def test_context_length_uint32_max(self, tmp_path, backend):
"""2^32 - 1 survives without truncation."""
val = 2**32 - 1
path = make_gguf(tmp_path, "llama", [("context_length", val, 4)])
backend._read_gguf_metadata(path)
assert backend.native_context_length == val
def test_context_length_uint64(self, tmp_path, backend):
"""UINT64 type context_length parsed correctly."""
val = 2**33
path = make_gguf(tmp_path, "llama", [("context_length", val, 10)])
backend._read_gguf_metadata(path)
assert backend.native_context_length == val
def test_no_context_length_in_gguf(self, tmp_path, backend):
"""GGUF without context_length key yields None."""
path = make_gguf(tmp_path, "llama", [("block_count", 32, 4)])
backend._read_gguf_metadata(path)
assert backend.native_context_length is None
def test_native_equals_context_when_uncapped(self, backend):
"""Both equal when no VRAM cap applied."""
backend._context_length = 8192
assert backend.native_context_length == backend.context_length
def test_native_survives_parse_then_cap(self, tmp_path, backend):
"""Parse then set effective cap: native unchanged."""
path = make_gguf(
tmp_path,
"llama",
[
("context_length", 131072, 4),
("block_count", 32, 4),
("attention.head_count", 32, 4),
("attention.head_count_kv", 8, 4),
("embedding_length", 4096, 4),
],
)
backend._read_gguf_metadata(path)
assert backend.native_context_length == 131072
backend._effective_context_length = 16384
backend._max_context_length = 32768
assert backend.native_context_length == 131072
class TestCrossPlatform:
"""Binary I/O and serialization correctness across platforms."""
def test_le_uint64_context_length(self, tmp_path, backend):
"""Little-endian UINT64 parsed correctly."""
path = make_gguf(tmp_path, "llama", [("context_length", 16384, 10)])
backend._read_gguf_metadata(path)
assert backend.native_context_length == 16384
def test_gguf_magic_le_byte_order(self, tmp_path):
"""Magic 0x46554747 matches GGUF spec (little-endian 'GGUF')."""
path = tmp_path / "magic_check.gguf"
buf = io.BytesIO()
buf.write(struct.pack("<I", 0x46554747))
raw = buf.getvalue()
assert raw == b"GGUF"
def test_json_serialization_deterministic(self):
"""model_dump_json() is consistent across calls."""
resp = LoadResponse(
status = "loaded",
model = "test",
display_name = "Test",
inference = {},
native_context_length = 131072,
)
json1 = resp.model_dump_json()
json2 = resp.model_dump_json()
assert json1 == json2
assert '"native_context_length":131072' in json1
def test_the_status_route_reports_what_a_self_sizing_load_asked_for():
"""The UI re-seeds its context pin from it, because the resolved window cannot say
whether anyone chose that length; without it a pin is invisible after a refresh."""
route_src = (Path(__file__).resolve().parents[1] / "routes" / "inference.py").read_text(
encoding = "utf-8"
)
assert "requested_context_length = llama_backend.requested_n_ctx" in route_src
# The non-GGUF branch is covered by behaviour in test_non_gguf_reload_settings.py.
# The parent cannot recompute the group once the worker holds the model: it mirrors all.
src = (Path(__file__).resolve().parents[1] / "core/inference/worker.py").read_text("utf-8")
assert '"native_context_length",\n "max_context_length",' in src