Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

18.4 KB
Raw Blame
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import threading
from typing import Any

import pytest

from deeptutor.services.config.model_catalog import SERVICE_NAMES, ModelCatalogService


def test_load_creates_empty_catalog_without_dotenv_hydration(tmp_path: Path):
    env_path = tmp_path / ".env"
    env_path.write_text(
        "LLM_MODEL=legacy-model\nLLM_API_KEY=legacy-key\nEMBEDDING_MODEL=legacy-embedding\n",
        encoding="utf-8",
    )
    catalog_path = tmp_path / "model_catalog.json"

    catalog = ModelCatalogService(path=catalog_path).load()

    assert catalog["services"]["llm"]["profiles"] == []
    assert catalog["services"]["embedding"]["profiles"] == []
    assert catalog["services"]["search"]["profiles"] == []


def test_load_does_not_sync_existing_active_profiles_from_dotenv(tmp_path: Path):
    (tmp_path / ".env").write_text(
        "LLM_MODEL=qwen3.5-plus\nEMBEDDING_MODEL=text-embedding-v4\n",
        encoding="utf-8",
    )
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text(
        """{
  "version": 1,
  "services": {
    "llm": {
      "active_profile_id": "llm-profile-default",
      "active_model_id": "llm-model-default",
      "profiles": [
        {
          "id": "llm-profile-default",
          "name": "Default LLM Endpoint",
          "binding": "openai",
          "base_url": "https://old-llm.example/v1",
          "api_key": "old-llm-key",
          "api_version": "",
          "extra_headers": {},
          "models": [
            {"id": "llm-model-default", "name": "old-model", "model": "old-model"}
          ]
        }
      ]
    },
    "embedding": {
      "active_profile_id": "embedding-profile-default",
      "active_model_id": "embedding-model-default",
      "profiles": [
        {
          "id": "embedding-profile-default",
          "name": "Default Embedding Endpoint",
          "binding": "openai",
          "base_url": "https://old-emb.example/v1",
          "api_key": "old-emb-key",
          "api_version": "",
          "extra_headers": {},
          "models": [
            {
              "id": "embedding-model-default",
              "name": "old-embedding",
              "model": "old-embedding",
              "dimension": "3072"
            }
          ]
        }
      ]
    },
    "search": {"active_profile_id": null, "profiles": []}
  }
}
""",
        encoding="utf-8",
    )

    service = ModelCatalogService(path=catalog_path)
    catalog = service.load()

    llm_profile = catalog["services"]["llm"]["profiles"][0]
    llm_model = llm_profile["models"][0]
    emb_profile = catalog["services"]["embedding"]["profiles"][0]
    emb_model = emb_profile["models"][0]

    assert llm_profile["binding"] == "openai"
    assert llm_profile["base_url"] == "https://old-llm.example/v1"
    assert llm_profile["api_key"] == "old-llm-key"
    assert llm_model["model"] == "old-model"
    assert llm_model["name"] == "old-model"
    assert emb_profile["binding"] == "openai"
    assert emb_profile["base_url"] == "https://old-emb.example/v1/embeddings"
    assert emb_profile["api_key"] == "old-emb-key"
    assert emb_model["model"] == "old-embedding"
    assert emb_model["name"] == "old-embedding"
    assert emb_model["dimension"] == "3072"


def test_load_recovers_invalid_catalog_with_defaults(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text("{not-json", encoding="utf-8")

    catalog = ModelCatalogService(path=catalog_path).load()

    # Derived rather than re-listed: a new service should not need this test
    # edited to keep passing, only the one that describes it.
    expected_services = set(SERVICE_NAMES)
    assert set(catalog["services"]) == expected_services
    saved = json.loads(catalog_path.read_text(encoding="utf-8"))
    assert set(saved["services"]) == expected_services


def test_catalog_migration_read_cannot_overwrite_concurrent_settings_update(tmp_path, monkeypatch):
    """A runtime read may migrate defaults while the settings endpoint saves."""
    path = tmp_path / "model_catalog.json"
    path.write_text(json.dumps({"version": 1, "services": {}}), encoding="utf-8")
    service = ModelCatalogService(path)
    read = service._read_existing_catalog
    captured = threading.Event()
    release = threading.Event()
    updating = threading.Event()
    reader_id = None

    def paused_read():
        value = read()
        if threading.get_ident() == reader_id:
            captured.set()
            assert release.wait(5)
        return value

    def reader():
        nonlocal reader_id
        reader_id = threading.get_ident()
        return service.load()

    def writer():
        updating.set()
        return service.update(lambda catalog: catalog.update({"saved_selection": "new-model"}))

    monkeypatch.setattr(service, "_read_existing_catalog", paused_read)
    with ThreadPoolExecutor(max_workers=2) as pool:
        loading = pool.submit(reader)
        try:
            assert captured.wait(5)
            saving = pool.submit(writer)
            assert updating.wait(5)
            try:
                saving.result(timeout=1)
            except TimeoutError:
                pass  # Correct serialization waits until the migration read completes.
        finally:
            release.set()
        loading.result(timeout=5)
        saving.result(timeout=5)
    assert json.loads(path.read_text())["saved_selection"] == "new-model"


@pytest.mark.parametrize("operation", ["load", "update", "apply"])
@pytest.mark.parametrize("error_type", [OSError, PermissionError])
def test_catalog_read_error_preserves_existing_file(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    operation: str,
    error_type: type[OSError],
) -> None:
    """An unreadable catalog must not be replaced by an empty catalog."""
    catalog_path = tmp_path / "model_catalog.json"
    original = b'{"version": 1, "services": {}, "saved_selection": "retain-me"}'
    catalog_path.write_bytes(original)
    read_text = Path.read_text

    def fail_catalog_read(path: Path, *args: Any, **kwargs: Any) -> str:
        """Fail only the source read while leaving the actual writer usable."""
        if path == catalog_path:
            raise error_type("catalog temporarily unreadable")
        return read_text(path, *args, **kwargs)

    monkeypatch.setattr(Path, "read_text", fail_catalog_read)
    service = ModelCatalogService(path=catalog_path)
    with pytest.raises(error_type, match="catalog temporarily unreadable"):
        if operation == "update":
            service.update(lambda catalog: catalog.update({"new_setting": True}))
        else:
            getattr(service, operation)()

    assert catalog_path.read_bytes() == original


def test_load_migrates_only_legacy_dashscope_stt_model(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text(
        json.dumps(
            {
                "services": {
                    "stt": {
                        "active_profile_id": "dashscope-profile",
                        "active_model_id": "dashscope-model",
                        "profiles": [
                            {
                                "id": "dashscope-profile",
                                "binding": "bailian",
                                "models": [
                                    {
                                        "id": "dashscope-model",
                                        "name": "paraformer-v2",
                                        "model": "paraformer-v2",
                                    },
                                    {
                                        "id": "custom-name",
                                        "name": "My speech model",
                                        "model": "paraformer-v2",
                                    },
                                ],
                            },
                            {
                                "id": "other-profile",
                                "binding": "openai",
                                "models": [
                                    {
                                        "id": "other-model",
                                        "name": "paraformer-v2",
                                        "model": "paraformer-v2",
                                    }
                                ],
                            },
                        ],
                    }
                }
            }
        ),
        encoding="utf-8",
    )

    loaded = ModelCatalogService(path=catalog_path).load()
    models = loaded["services"]["stt"]["profiles"]
    assert models[0]["models"][0]["model"] == "paraformer-realtime-v2"
    assert models[0]["models"][0]["name"] == "paraformer-realtime-v2"
    assert models[0]["models"][1]["model"] == "paraformer-realtime-v2"
    assert models[0]["models"][1]["name"] == "My speech model"
    assert models[1]["models"][0]["model"] == "paraformer-v2"
    assert json.loads(catalog_path.read_text(encoding="utf-8")) == loaded


def test_load_migrates_legacy_stt_for_effective_provider_refs(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text(
        json.dumps(
            {
                "connections": [
                    {"id": "dashscope", "provider": "dashscope"},
                    {"id": "openai", "provider": "openai"},
                ],
                "services": {
                    "stt": {
                        "profiles": [
                            {
                                "id": "source",
                                "binding": "dashscope",
                                "models": [
                                    {"id": "source-model", "model": "paraformer-realtime-v2"}
                                ],
                            },
                            {
                                "id": "profile-ref",
                                "binding": "custom",
                                "provider_ref": {"connection_id": "dashscope"},
                                "models": [{"id": "profile-model", "model": "paraformer-v2"}],
                            },
                            {
                                "id": "model-ref",
                                "binding": "openai",
                                "models": [
                                    {
                                        "id": "model-ref-model",
                                        "model": "paraformer-v2",
                                        "provider_ref": {"service": "stt", "profile_id": "source"},
                                    }
                                ],
                            },
                            {
                                "id": "model-override",
                                "binding": "dashscope",
                                "provider_ref": {"connection_id": "dashscope"},
                                "models": [
                                    {
                                        "id": "other-model",
                                        "model": "paraformer-v2",
                                        "provider_ref": {"connection_id": "openai"},
                                    }
                                ],
                            },
                            {
                                "id": "profile-override",
                                "binding": "dashscope",
                                "provider_ref": {"connection_id": "openai"},
                                "models": [{"id": "other-profile-model", "model": "paraformer-v2"}],
                            },
                        ]
                    }
                },
            }
        ),
        encoding="utf-8",
    )

    loaded = ModelCatalogService(path=catalog_path).load()
    models = {
        profile["id"]: profile["models"][0]["model"]
        for profile in loaded["services"]["stt"]["profiles"]
    }
    assert models == {
        "source": "paraformer-realtime-v2",
        "profile-ref": "paraformer-realtime-v2",
        "model-ref": "paraformer-realtime-v2",
        "model-override": "paraformer-v2",
        "profile-override": "paraformer-v2",
    }
    assert json.loads(catalog_path.read_text(encoding="utf-8")) == loaded


def test_load_normalizes_wire_api_to_supported_profile_backends(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text(
        json.dumps(
            {
                "services": {
                    "llm": {
                        "active_profile_id": "azure-profile",
                        "active_model_id": "azure-model",
                        "profiles": [
                            {
                                "id": "azure-profile",
                                "name": "Azure",
                                "binding": "azure_openai",
                                "wire_api": "responses",
                                "models": [
                                    {
                                        "id": "azure-model",
                                        "name": "Deployment",
                                        "model": "deployment-name",
                                    }
                                ],
                            },
                            {
                                "id": "custom-profile",
                                "name": "Custom",
                                "binding": "custom",
                                "wire_api": "unknown-protocol",
                                "models": [],
                            },
                        ],
                    }
                }
            }
        ),
        encoding="utf-8",
    )

    loaded = ModelCatalogService(path=catalog_path).load()
    profiles = loaded["services"]["llm"]["profiles"]

    assert profiles[0]["wire_api"] == "auto"
    assert profiles[1]["wire_api"] == "auto"


def _gemini_embedding_catalog(path: Path, model: str) -> Path:
    path.write_text(
        json.dumps(
            {
                "services": {
                    "embedding": {
                        "active_profile_id": "gemini-profile",
                        "active_model_id": "gemini-model",
                        "profiles": [
                            {
                                "id": "gemini-profile",
                                "name": "Gemini",
                                "binding": "gemini",
                                "base_url": "",
                                "api_key": "test-key",
                                "models": [
                                    {
                                        "id": "gemini-model",
                                        "name": model,
                                        "model": model,
                                    }
                                ],
                            }
                        ],
                    }
                }
            }
        ),
        encoding="utf-8",
    )
    return path


def test_load_sets_gemini_native_endpoint_from_active_embedding_model(tmp_path: Path):
    catalog_path = _gemini_embedding_catalog(tmp_path / "model_catalog.json", "gemini-embedding-2")

    catalog = ModelCatalogService(path=catalog_path).load()

    profile = catalog["services"]["embedding"]["profiles"][0]
    assert profile["base_url"].endswith("/models/gemini-embedding-2:batchEmbedContents")


def test_load_keeps_older_gemini_embedding_models_on_the_openai_path(tmp_path: Path):
    """The native route sends a taskType and L2-normalizes, so moving an
    existing gemini-embedding-001 profile there would change its document
    vectors and invalidate the index built from them."""
    catalog_path = _gemini_embedding_catalog(
        tmp_path / "model_catalog.json", "gemini-embedding-001"
    )

    catalog = ModelCatalogService(path=catalog_path).load()

    profile = catalog["services"]["embedding"]["profiles"][0]
    assert profile["base_url"] == (
        "https://generativelanguage.googleapis.com/v1beta/openai/embeddings"
    )


def test_load_persists_normalized_active_ids(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    catalog_path.write_text(
        json.dumps(
            {
                "services": {
                    "llm": {
                        "active_profile_id": "missing-profile",
                        "active_model_id": "missing-model",
                        "profiles": [
                            {
                                "id": "llm-profile-a",
                                "name": "A",
                                "binding": "openai",
                                "base_url": "https://example.test/v1",
                                "api_key": "sk",
                                "models": [
                                    {
                                        "id": "llm-model-a",
                                        "name": "gpt",
                                        "model": "gpt-test",
                                    }
                                ],
                            }
                        ],
                    }
                }
            }
        ),
        encoding="utf-8",
    )

    ModelCatalogService(path=catalog_path).load()

    saved = json.loads(catalog_path.read_text(encoding="utf-8"))
    llm = saved["services"]["llm"]
    assert llm["active_profile_id"] == "llm-profile-a"
    assert llm["active_model_id"] == "llm-model-a"
    assert saved["services"]["embedding"]["profiles"] == []
    assert saved["services"]["search"]["profiles"] == []


def test_update_serializes_concurrent_catalog_mutations(tmp_path: Path):
    service = ModelCatalogService(path=tmp_path / "model_catalog.json")
    initial = service.load()
    initial["mutation_count"] = 0
    service.save(initial)

    def increment(_index: int) -> None:
        def mutate(catalog: dict) -> None:
            catalog["mutation_count"] = int(catalog.get("mutation_count", 0)) + 1

        service.update(mutate)

    with ThreadPoolExecutor(max_workers=4) as executor:
        list(executor.map(increment, range(40)))

    assert service.load()["mutation_count"] == 40


def test_atomic_save_leaves_no_temporary_file(tmp_path: Path):
    catalog_path = tmp_path / "model_catalog.json"
    service = ModelCatalogService(path=catalog_path)

    service.save({"version": 1, "services": {}})

    assert catalog_path.exists()
    assert not list(tmp_path.glob(".model_catalog.json.*"))