Skip to content

Self-improving AI Employees for everyone

7.0 KB
Raw Blame
"""RFC-0003 D15 / D16 / AG18: the model registry's three sources.

Models registered from the user's own servers live apart from the OpenRouter
snapshot (so a refresh cannot wipe them, and they never dirty the tracked
``config/model_registry.json``), and the LiteLLM table is trimmed, cached
under DATA_DIR, and matched conservatively.
"""

from __future__ import annotations

import asyncio
import json

import httpx
import pytest
import respx

import services.model_registry as mr

OPENROUTER = {
    "data": [
        {
            "id": "openai/gpt-test",
            "name": "OpenAI: GPT Test",
            "context_length": 400000,
            "top_provider": {"max_completion_tokens": 128000},
            "pricing": {"prompt": "0.000001", "completion": "0.000002"},
        }
    ]
}
LITELLM = {
    "sample_spec": {"mode": "chat", "max_input_tokens": "set to max input tokens"},
    "gpt-4o": {
        "mode": "chat",
        "max_input_tokens": 128000,
        "max_output_tokens": 16384,
        "input_cost_per_token": 2.5e-06,
        "output_cost_per_token": 1e-05,
        "litellm_provider": "openai",
        "supports_vision": True,
    },
    "together_ai/meta-llama/Llama-3.3-70B": {"mode": "chat", "max_input_tokens": 131072},
    "deepinfra/Qwen/Qwen3-8B": {"mode": "chat", "max_input_tokens": 32768},
    "fireworks_ai/Qwen/Qwen3-8B": {"mode": "chat", "max_input_tokens": 40960},
    "text-embedding-3-small": {"mode": "embedding", "max_input_tokens": 8191},
}


@pytest.fixture
def paths(tmp_path, monkeypatch):
    snapshot = tmp_path / "config" / "model_registry.json"
    monkeypatch.setattr(mr, "CACHE_FILE", snapshot)
    monkeypatch.setattr(mr, "_local_models_path", lambda: tmp_path / "data" / "local_models.json")
    monkeypatch.setattr(mr, "_litellm_cache_path", lambda: tmp_path / "data" / "litellm_models.json")
    return tmp_path


@pytest.fixture
def registry(paths):
    svc = mr.ModelRegistryService()
    svc._load_llm_defaults()
    return svc


def _mock_feeds(litellm=LITELLM):
    respx.get(mr.OPENROUTER_MODELS_URL).mock(return_value=httpx.Response(200, json=OPENROUTER))
    return respx.get(mr.LITELLM_MODELS_URL).mock(return_value=httpx.Response(200, json=litellm))


class TestLocalModels:
    @respx.mock
    async def test_a_refresh_does_not_wipe_models_registered_from_the_users_servers(self, registry):
        registry.register_local_model("ollama", "llama3:latest", {"context_length": 4096})
        _mock_feeds()

        await registry.refresh()

        assert registry.get_context_length("llama3:latest", "ollama") == 4096
        assert registry.get_context_length("gpt-test", "openai") == 400000

    @respx.mock
    async def test_they_persist_under_data_dir_never_in_the_tracked_snapshot(self, registry, paths):
        registry.register_local_model("openai_compatible:home", "qwen3", {"context_length": 32768})
        _mock_feeds()
        await registry.refresh()

        local = json.loads((paths / "data" / "local_models.json").read_text(encoding="utf-8"))
        snapshot = json.loads((paths / "config" / "model_registry.json").read_text(encoding="utf-8"))
        assert "openai_compatible:home/qwen3" in local["models"]
        assert not any(key.startswith(("openai_compatible:", "ollama/")) for key in snapshot["models"])

        reloaded = mr.ModelRegistryService()
        reloaded.startup()
        assert reloaded.get_context_length("qwen3", "openai_compatible:home") == 32768

    def test_forgetting_a_provider_drops_only_its_models(self, registry):
        registry.register_local_model("openai_compatible:a", "m", {"context_length": 1000})
        registry.register_local_model("openai_compatible:b", "m", {"context_length": 2000})

        registry.forget_local_models("openai_compatible:a")

        assert registry.get_model_info("m", "openai_compatible:a") is None
        assert registry.get_context_length("m", "openai_compatible:b") == 2000

    def test_a_registered_price_is_kept_for_pricing(self, registry):
        registry.register_local_model(
            "openai_compatible:gw", "gpt-4o", {"context_length": 128000, "input_price_per_mtok": 2.5, "output_price_per_mtok": 10.0}
        )

        info = registry.get_model_info("gpt-4o", "openai_compatible:gw")
        assert (info.input_price_per_mtok, info.output_price_per_mtok) == (2.5, 10.0)


class TestLiteLLMTable:
    @respx.mock
    async def test_refresh_keeps_chat_models_and_the_four_fields_read(self, registry, paths):
        _mock_feeds()

        await registry.refresh()

        assert registry.lookup_litellm("gpt-4o") == {
            "max_input_tokens": 128000,
            "max_output_tokens": 16384,
            "input_cost_per_token": 2.5e-06,
            "output_cost_per_token": 1e-05,
        }
        assert registry.lookup_litellm("text-embedding-3-small") is None
        assert registry.lookup_litellm("sample_spec") is None
        assert (paths / "data" / "litellm_models.json").exists()

    @respx.mock
    async def test_a_host_prefixed_key_matches_only_when_unambiguous(self, registry):
        _mock_feeds()
        await registry.refresh()

        assert registry.lookup_litellm("meta-llama/Llama-3.3-70B") == {"max_input_tokens": 131072}
        # Two hosts serve it with different windows: no guess.
        assert registry.lookup_litellm("Qwen/Qwen3-8B") is None
        assert registry.lookup_litellm("never-heard-of-it") is None

    @respx.mock
    async def test_a_failed_fetch_keeps_the_cached_copy(self, registry):
        _mock_feeds()
        await registry.refresh()
        respx.get(mr.LITELLM_MODELS_URL).mock(side_effect=httpx.ConnectError("offline"))

        await registry._refresh_litellm()

        assert registry.lookup_litellm("gpt-4o") is not None

    @respx.mock
    async def test_offline_with_no_cache_simply_matches_nothing(self, registry):
        respx.get(mr.LITELLM_MODELS_URL).mock(side_effect=httpx.ConnectError("offline"))

        await registry.ensure_litellm_table()

        assert registry.lookup_litellm("gpt-4o") is None

    @respx.mock
    async def test_an_on_demand_fetch_that_failed_is_not_repeated_on_every_save(self, registry):
        route = respx.get(mr.LITELLM_MODELS_URL).mock(side_effect=httpx.ConnectError("offline"))

        await registry.ensure_litellm_table()
        await registry.ensure_litellm_table()

        assert route.call_count == 1

    async def test_an_on_demand_fetch_is_bounded_in_total(self, registry, monkeypatch):
        calls = 0

        async def slow_refresh():
            nonlocal calls
            calls += 1
            await asyncio.sleep(5)

        monkeypatch.setattr(registry, "_refresh_litellm", slow_refresh)

        await asyncio.wait_for(registry.ensure_litellm_table(timeout=0.05), 1)
        await registry.ensure_litellm_table(timeout=0.05)

        assert calls == 1
        assert registry.lookup_litellm("gpt-4o") is None

    @respx.mock
    async def test_the_cache_is_reloaded_on_startup(self, registry):
        _mock_feeds()
        await registry.refresh()

        reloaded = mr.ModelRegistryService()
        reloaded.startup()

        assert reloaded.lookup_litellm("gpt-4o")["max_input_tokens"] == 128000