Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

4.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""The /v1/models servability cache is keyed on the catalog stamp; residency is never cached."""

from types import SimpleNamespace

import pytest

import routes.inference as inf


@pytest.fixture(autouse = True)
def _clean_cache():
    inf._SERVABLE_SCAN_CACHE["entry"] = None
    yield
    inf._SERVABLE_SCAN_CACHE["entry"] = None


def _catalog(n = 3):
    return [SimpleNamespace(id = f"repo/m{i}", path = f"/models/m{i}", task = None) for i in range(n)]


@pytest.fixture
def _stub(monkeypatch):
    """Count servability resolutions and make residency observable."""
    counts = {"servable": 0, "resident": 0}

    def _servable(info):
        counts["servable"] += 1
        return (True, ("Q4_K_M",))

    monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
    monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)

    def _resident(key, **kw):
        counts["resident"] += 1
        return False

    monkeypatch.setattr(inf, "_resolves_to_resident", _resident)
    return counts


def test_same_catalog_stamp_scans_once(_stub):
    catalog = _catalog(3)
    for _ in range(10):
        inf._servable_catalog_rows(catalog, 111.0)
    assert _stub["servable"] == 3, "one scan for three entries, not one per call"


def test_residency_is_recomputed_every_call(_stub):
    catalog = _catalog(3)
    for _ in range(10):
        inf._servable_catalog_rows(catalog, 111.0)
    assert _stub["resident"] == 30


def test_new_catalog_stamp_rescans(_stub):
    catalog = _catalog(2)
    inf._servable_catalog_rows(catalog, 111.0)
    assert _stub["servable"] == 2
    inf._servable_catalog_rows(catalog, 222.0)
    assert _stub["servable"] == 4, "a replaced catalog must be rescanned"


def test_no_stamp_disables_caching(_stub):
    catalog = _catalog(2)
    inf._servable_catalog_rows(catalog)
    inf._servable_catalog_rows(catalog)
    assert _stub["servable"] == 4


def test_rows_shape_is_unchanged(_stub):
    rows = inf._servable_catalog_rows(_catalog(1), 111.0)
    assert len(rows) == 1
    info, is_gguf, quants, resident = rows[0]
    assert info.id == "repo/m0"
    assert is_gguf is True
    assert quants == ("Q4_K_M",)
    assert resident is False


def test_unservable_entries_are_dropped(monkeypatch):
    monkeypatch.setattr(
        "core.inference.local_model_resolver.local_servable_model", lambda info: None
    )
    monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
    assert inf._servable_catalog_rows(_catalog(3), 111.0) == []


def test_a_managed_account_scans_the_shared_catalog_object(monkeypatch):
    """Filtering first handed a fresh list to the scan, so every managed poll rescanned."""
    import asyncio

    from hub.services.models import account_access

    catalog = _catalog(3)
    seen = []

    async def cached():
        return catalog

    monkeypatch.setattr(inf, "_cached_local_catalog", cached)
    monkeypatch.setattr(inf, "_account_catalog_cache", lambda: {"at": 1.0})
    monkeypatch.setattr(inf, "_openai_model_objects", lambda: [])
    monkeypatch.setattr(inf, "_peek_inference_backend", lambda: None)
    monkeypatch.setattr(account_access, "managed_account", lambda: True)
    monkeypatch.setattr(account_access, "filter_model_rows", lambda rows: list(rows[:1]))
    monkeypatch.setattr(account_access, "resident_hidden", lambda *a, **k: False)
    monkeypatch.setattr(inf, "_servable_catalog_rows", lambda rows, at: seen.append(rows) or [])
    asyncio.run(inf._openai_catalog_objects())
    assert seen[0] is catalog


def test_scan_detects_hardware_before_judging_servability(_stub, monkeypatch):
    from utils.hardware import hardware as hw

    order = []
    monkeypatch.setattr(hw, "ensure_hardware_detected", lambda: order.append("detect"))
    monkeypatch.setattr(
        "core.inference.local_model_resolver.local_servable_model",
        lambda info: order.append("scan") or (True, ()),
    )
    inf._servable_catalog_rows(_catalog(1), 111.0)
    assert order == ["detect", "scan"]