Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

4.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Old callers send n_ctx and read existing fields; omitting n_ctx is the one newly accepted shape."""

from __future__ import annotations

import asyncio
import inspect
import sys
from pathlib import Path

_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
    sys.path.insert(0, _TESTS_DIR)

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

from test_kv_cache_estimation import _make_gguf_bytes  # noqa: E402

import routes.models as models_routes  # noqa: E402

_FIELDS = {
    "context_length": 8192,
    "block_count": 32,
    "attention.head_count": 32,
    "attention.head_count_kv": 8,
    "embedding_length": 4096,
    "attention.key_length": 128,
    "attention.value_length": 128,
}

_LEGACY_KEYS = {"kv_bytes", "weights_bytes", "native_context"}


def _gguf(tmp_path: Path) -> Path:
    kv = {"general.architecture": "llama"}
    for k, v in _FIELDS.items():
        kv[f"llama.{k}"] = v
    p = tmp_path / "model-Q4_K_M.gguf"
    p.write_bytes(_make_gguf_bytes("llama", kv))
    return p


def _call(monkeypatch, path: Path | None, **overrides):
    if path is not None:
        monkeypatch.setattr(
            models_routes,
            "_resolve_quant_gguf",
            lambda *_a, **_k: (str(path), 4_000_000_000),
        )
    kwargs = dict(
        repo_id = "org/repo",
        quant = "Q4_K_M",
        n_ctx = 4096,
        cache_type_kv = None,
        n_parallel = None,
        speculative_type = None,
        spec_draft_n_max = None,
        spec_draft_cache_type = None,
        ctx_checkpoints = None,
        disable_vision = False,
        n_batch = None,
        n_ubatch = None,
        tensor_parallel = False,
        request = None,
        current_subject = "test",
    )
    kwargs.update(overrides)
    return asyncio.run(models_routes.get_kv_cache_estimate(**kwargs))


def test_every_parameter_an_old_caller_sent_is_still_accepted(tmp_path):
    """Signature check: nothing an existing client passes may have been removed
    or made required in a way it does not satisfy."""
    sig = inspect.signature(models_routes.get_kv_cache_estimate)
    for name in ("repo_id", "quant", "n_ctx", "cache_type_kv"):
        assert name in sig.parameters, f"{name} was removed from the route"
    # The new ones must be optional, or an old caller's request 422s.
    for name in ("n_parallel", "speculative_type"):
        assert sig.parameters[name].default is not inspect.Parameter.empty


def test_an_old_callers_request_still_answers_the_old_keys(monkeypatch, tmp_path):
    out = _call(monkeypatch, _gguf(tmp_path), n_ctx = 4096)
    assert _LEGACY_KEYS <= set(out), f"missing legacy keys: {_LEGACY_KEYS - set(out)}"
    assert out["kv_bytes"] and out["kv_bytes"] > 0
    assert out["weights_bytes"] == 4_000_000_000
    assert out["native_context"] == 8192


def test_the_answer_for_a_pinned_context_did_not_move(monkeypatch, tmp_path):
    """New parameters default to the old behaviour, so an unchanged request returns an unchanged number."""
    gguf = _gguf(tmp_path)
    before = _call(monkeypatch, gguf, n_ctx = 4096)
    after = _call(monkeypatch, gguf, n_ctx = 4096, n_parallel = None, speculative_type = None)
    assert before["kv_bytes"] == after["kv_bytes"]


def test_omitting_the_context_sizes_at_the_models_native_length(monkeypatch, tmp_path):
    """The new shape: no n_ctx. The response says which length it used."""
    gguf = _gguf(tmp_path)
    native = _call(monkeypatch, gguf, n_ctx = None)
    assert native["n_ctx"] == 8192
    assert native["native_context"] == 8192
    explicit = _call(monkeypatch, gguf, n_ctx = 8192)
    assert native["kv_bytes"] == explicit["kv_bytes"]


def test_the_failure_answer_carries_every_key(monkeypatch):
    """An unsizable row must still carry every key, or a short dict would KeyError in the caller."""
    monkeypatch.setattr(models_routes, "_resolve_quant_gguf", lambda *_a, **_k: (None, 0))
    out = _call(monkeypatch, None)
    assert _LEGACY_KEYS <= set(out)
    assert {"spec_bytes", "n_ctx", "projector_bytes", "spec_unpriced"} <= set(out)
    assert all(v is None for k, v in out.items() if k != "spec_unpriced")
    assert out["spec_unpriced"] is False


def test_speculative_modes_that_cost_nothing_report_none(monkeypatch, tmp_path):
    gguf = _gguf(tmp_path)
    for mode in (None, "", "off", "ngram"):
        out = _call(monkeypatch, gguf, speculative_type = mode)
        assert out["spec_bytes"] is None, f"{mode!r} reserved memory"