Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

10.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Adopt the server's /props n_ctx when smaller than the requested -c, or clients get 400s."""

from __future__ import annotations

import json
import sys
import types as _types
from pathlib import Path

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

# Prefer real modules so this file cannot poison later tests with stubs.
try:
    import loggers  # noqa: F401
except ImportError:
    _loggers_stub = _types.ModuleType("loggers")
    _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
    sys.modules.setdefault("loggers", _loggers_stub)

try:
    import structlog  # noqa: F401
except ImportError:
    sys.modules.setdefault("structlog", _types.ModuleType("structlog"))

try:
    import httpx  # noqa: F401
except ImportError:
    _httpx_stub = _types.ModuleType("httpx")
    for _exc_name in (
        "ConnectError",
        "TimeoutException",
        "ReadTimeout",
        "ReadError",
        "RemoteProtocolError",
        "CloseError",
        "WriteError",
        "HTTPError",
    ):
        setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {}))

    class _FakeTimeout:
        def __init__(self, *a, **kw):
            pass

    _httpx_stub.Timeout = _FakeTimeout
    _httpx_stub.Client = type(
        "Client",
        (),
        {
            "__init__": lambda self, **kw: None,
            "__enter__": lambda self: self,
            "__exit__": lambda self, *a: None,
        },
    )
    _httpx_stub.get = lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("unstubbed httpx.get"))
    sys.modules.setdefault("httpx", _httpx_stub)

from core.inference.llama_cpp import LlamaCppBackend
import core.inference.llama_cpp as llama_cpp_mod


class _FakeResponse:
    def __init__(
        self,
        status_code = 200,
        body = None,
    ):
        self.status_code = status_code
        self._body = body or {}

    def json(self):
        return self._body


def _make_backend(
    effective_ctx = 98304,
    port = 51234,
    api_key = None,
):
    inst = LlamaCppBackend.__new__(LlamaCppBackend)
    inst._port = port
    # __new__ skips __init__; _auth_headers reads this for --api-key servers.
    inst._api_key = api_key
    inst._effective_context_length = effective_ctx
    inst._context_length = 262144
    inst._effective_parallel_slots = 1
    inst._kv_cache_unified = False
    inst._kv_cache_context_total = None
    return inst


def _stub_props(
    monkeypatch,
    status_code = 200,
    body = None,
    exc = None,
):
    def fake_get(
        url,
        headers = None,
        timeout = None,
        trust_env = None,
        verify = None,
    ):
        assert url.endswith("/props")

        assert trust_env is False
        # Shared context: httpx otherwise parses the CA bundle on every /props read.
        assert verify is llama_cpp_mod._local_ssl_context()
        # /props is behind the api-key middleware; no key means no header, not 'Bearer None'.
        assert headers is None or headers == {"Authorization": "Bearer test-key"}
        if exc is not None:
            raise exc
        return _FakeResponse(status_code, body)

    monkeypatch.setattr(llama_cpp_mod.httpx, "get", fake_get, raising = False)


def test_query_n_ctx_reads_default_generation_settings(monkeypatch):
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 67584}},
    )
    assert _make_backend()._query_server_n_ctx() == 67584


def test_query_n_ctx_non_200_returns_none(monkeypatch):
    _stub_props(monkeypatch, status_code = 503)
    assert _make_backend()._query_server_n_ctx() is None


def test_query_n_ctx_missing_key_returns_none(monkeypatch):
    _stub_props(monkeypatch, body = {"default_generation_settings": {}})
    assert _make_backend()._query_server_n_ctx() is None


def test_query_n_ctx_swallows_transport_errors(monkeypatch):
    _stub_props(monkeypatch, exc = RuntimeError("connection refused"))
    assert _make_backend()._query_server_n_ctx() is None


def test_fit_shrunk_ctx_overwrites_advertised_value(monkeypatch):
    """The Nick repro: requested/advertised 98304, server really at 67584."""
    inst = _make_backend(effective_ctx = 98304)
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 67584}},
    )
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 67584
    assert inst.context_length == 67584


def test_props_keeps_total_cache_context_for_slot_preflight(monkeypatch):
    inst = _make_backend(effective_ctx = 32768)
    inst._effective_parallel_slots = 4
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 8192}},
    )
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 8192
    assert inst._kv_cache_context_total == 32768


def test_props_does_not_multiply_unified_cache_context(monkeypatch):
    inst = _make_backend(effective_ctx = 32768)
    inst._effective_parallel_slots = 4
    inst._kv_cache_unified = True
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 32768}},
    )
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 32768
    assert inst._kv_cache_context_total == 32768


def test_matching_ctx_is_left_alone(monkeypatch):
    inst = _make_backend(effective_ctx = 98304)
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 98304}},
    )
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 98304


def test_larger_server_ctx_does_not_inflate_advertised_value(monkeypatch):
    """Never advertise more than the user asked for, even if the server could."""
    inst = _make_backend(effective_ctx = 32768)
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 65536}},
    )
    inst._reconcile_effective_ctx_with_server(requested_n_ctx = 32768)
    assert inst._effective_context_length == 32768


def test_explicit_extra_arg_ctx_adopts_larger_confirmed_server_value(monkeypatch):
    """A trailing --ctx-size that /props confirms is adopted, while the VRAM warning threshold is kept."""
    inst = _make_backend(effective_ctx = 65983)
    inst._max_context_length = 65983
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 100352}},
    )

    inst._reconcile_effective_ctx_with_server(requested_n_ctx = 100352)

    assert inst._effective_context_length == 100352
    assert inst.context_length == 100352
    assert inst.max_context_length == 65983


def test_no_explicit_flag_never_adopts_a_larger_server_value(monkeypatch):
    """Only a pass-through --ctx-size can exceed the fit; an unflagged load never adopts a larger value."""
    inst = _make_backend(effective_ctx = 65983)
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 66048}},
    )

    inst._reconcile_effective_ctx_with_server(requested_n_ctx = 0)

    assert inst._effective_context_length == 65983


def test_unset_effective_ctx_adopts_server_value(monkeypatch):
    inst = _make_backend(effective_ctx = None)
    inst._context_length = None
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 40960}},
    )
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 40960


def test_unset_effective_ctx_still_honours_the_explicit_ceiling(monkeypatch):
    """The unset arm publishes too, so the same ceiling has to bind there."""
    inst = _make_backend(effective_ctx = None)
    inst._context_length = None
    _stub_props(
        monkeypatch,
        body = {"default_generation_settings": {"n_ctx": 8192}},
    )

    inst._reconcile_effective_ctx_with_server(requested_n_ctx = 4096)

    assert inst._effective_context_length == 4096


def test_props_failure_keeps_studio_value(monkeypatch):
    """A flaky /props must never wipe the computed context."""
    inst = _make_backend(effective_ctx = 98304)
    _stub_props(monkeypatch, exc = RuntimeError("boom"))
    inst._reconcile_effective_ctx_with_server()
    assert inst._effective_context_length == 98304


_CAPS_ALL = {"supports_kv_unified": True, "supports_fit_ctx": True}
_CAPS_NONE = {"supports_kv_unified": False, "supports_fit_ctx": False}


def test_kv_unified_added_for_multi_slot():
    """Explicit --parallel N disables the auto-slots kv-unified default, splitting -c into -c/N
    windows; Unsloth must restore the shared pool so one request gets the full context."""
    flags = LlamaCppBackend._ctx_integrity_flags(4, False, False, 98304, 98304, _CAPS_ALL)
    assert "--kv-unified" in flags


def test_kv_unified_skipped_for_single_slot_or_old_build():
    assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags(
        1, False, False, 98304, 98304, _CAPS_ALL
    )
    assert "--kv-unified" not in LlamaCppBackend._ctx_integrity_flags(
        4, False, False, 98304, 98304, _CAPS_NONE
    )


def test_fit_ctx_floors_explicit_request_under_fit():
    flags = LlamaCppBackend._ctx_integrity_flags(1, True, False, 98304, 98304, _CAPS_ALL)
    assert flags[flags.index("--fit-ctx") + 1] == "98304"


def test_fit_ctx_skipped_without_fit_or_support():
    assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags(
        1, False, False, 98304, 98304, _CAPS_ALL
    )
    assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags(
        1, True, True, 98304, 98304, _CAPS_NONE
    )


def test_fit_ctx_floors_auto_request_at_8192_only_under_auto_fit():
    flags = LlamaCppBackend._ctx_integrity_flags(1, True, True, 0, 262144, _CAPS_ALL)
    assert flags[flags.index("--fit-ctx") + 1] == "8192"
    # Legacy auto pins native with -c 0, so the 8192 floor must not override it.
    assert "--fit-ctx" not in LlamaCppBackend._ctx_integrity_flags(
        1, True, False, 0, 262144, _CAPS_ALL
    )


def test_probe_missing_binary_reports_new_capabilities_false():
    info = LlamaCppBackend.probe_server_capabilities(binary = "/nonexistent/llama-server")
    assert info["found"] is False
    assert info["supports_kv_unified"] is False
    assert info["supports_fit_ctx"] is False