DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import threading
from typing import Any
import pytest
from deeptutor.services.config.model_catalog import SERVICE_NAMES, ModelCatalogService
def test_load_creates_empty_catalog_without_dotenv_hydration(tmp_path: Path):
env_path = tmp_path / ".env"
env_path.write_text(
"LLM_MODEL=legacy-model\nLLM_API_KEY=legacy-key\nEMBEDDING_MODEL=legacy-embedding\n",
encoding="utf-8",
)
catalog_path = tmp_path / "model_catalog.json"
catalog = ModelCatalogService(path=catalog_path).load()
assert catalog["services"]["llm"]["profiles"] == []
assert catalog["services"]["embedding"]["profiles"] == []
assert catalog["services"]["search"]["profiles"] == []
def test_load_does_not_sync_existing_active_profiles_from_dotenv(tmp_path: Path):
(tmp_path / ".env").write_text(
"LLM_MODEL=qwen3.5-plus\nEMBEDDING_MODEL=text-embedding-v4\n",
encoding="utf-8",
)
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text(
"""{
"version": 1,
"services": {
"llm": {
"active_profile_id": "llm-profile-default",
"active_model_id": "llm-model-default",
"profiles": [
{
"id": "llm-profile-default",
"name": "Default LLM Endpoint",
"binding": "openai",
"base_url": "https://old-llm.example/v1",
"api_key": "old-llm-key",
"api_version": "",
"extra_headers": {},
"models": [
{"id": "llm-model-default", "name": "old-model", "model": "old-model"}
]
}
]
},
"embedding": {
"active_profile_id": "embedding-profile-default",
"active_model_id": "embedding-model-default",
"profiles": [
{
"id": "embedding-profile-default",
"name": "Default Embedding Endpoint",
"binding": "openai",
"base_url": "https://old-emb.example/v1",
"api_key": "old-emb-key",
"api_version": "",
"extra_headers": {},
"models": [
{
"id": "embedding-model-default",
"name": "old-embedding",
"model": "old-embedding",
"dimension": "3072"
}
]
}
]
},
"search": {"active_profile_id": null, "profiles": []}
}
}
""",
encoding="utf-8",
)
service = ModelCatalogService(path=catalog_path)
catalog = service.load()
llm_profile = catalog["services"]["llm"]["profiles"][0]
llm_model = llm_profile["models"][0]
emb_profile = catalog["services"]["embedding"]["profiles"][0]
emb_model = emb_profile["models"][0]
assert llm_profile["binding"] == "openai"
assert llm_profile["base_url"] == "https://old-llm.example/v1"
assert llm_profile["api_key"] == "old-llm-key"
assert llm_model["model"] == "old-model"
assert llm_model["name"] == "old-model"
assert emb_profile["binding"] == "openai"
assert emb_profile["base_url"] == "https://old-emb.example/v1/embeddings"
assert emb_profile["api_key"] == "old-emb-key"
assert emb_model["model"] == "old-embedding"
assert emb_model["name"] == "old-embedding"
assert emb_model["dimension"] == "3072"
def test_load_recovers_invalid_catalog_with_defaults(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text("{not-json", encoding="utf-8")
catalog = ModelCatalogService(path=catalog_path).load()
# Derived rather than re-listed: a new service should not need this test
# edited to keep passing, only the one that describes it.
expected_services = set(SERVICE_NAMES)
assert set(catalog["services"]) == expected_services
saved = json.loads(catalog_path.read_text(encoding="utf-8"))
assert set(saved["services"]) == expected_services
def test_catalog_migration_read_cannot_overwrite_concurrent_settings_update(tmp_path, monkeypatch):
"""A runtime read may migrate defaults while the settings endpoint saves."""
path = tmp_path / "model_catalog.json"
path.write_text(json.dumps({"version": 1, "services": {}}), encoding="utf-8")
service = ModelCatalogService(path)
read = service._read_existing_catalog
captured = threading.Event()
release = threading.Event()
updating = threading.Event()
reader_id = None
def paused_read():
value = read()
if threading.get_ident() == reader_id:
captured.set()
assert release.wait(5)
return value
def reader():
nonlocal reader_id
reader_id = threading.get_ident()
return service.load()
def writer():
updating.set()
return service.update(lambda catalog: catalog.update({"saved_selection": "new-model"}))
monkeypatch.setattr(service, "_read_existing_catalog", paused_read)
with ThreadPoolExecutor(max_workers=2) as pool:
loading = pool.submit(reader)
try:
assert captured.wait(5)
saving = pool.submit(writer)
assert updating.wait(5)
try:
saving.result(timeout=1)
except TimeoutError:
pass # Correct serialization waits until the migration read completes.
finally:
release.set()
loading.result(timeout=5)
saving.result(timeout=5)
assert json.loads(path.read_text())["saved_selection"] == "new-model"
@pytest.mark.parametrize("operation", ["load", "update", "apply"])
@pytest.mark.parametrize("error_type", [OSError, PermissionError])
def test_catalog_read_error_preserves_existing_file(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
operation: str,
error_type: type[OSError],
) -> None:
"""An unreadable catalog must not be replaced by an empty catalog."""
catalog_path = tmp_path / "model_catalog.json"
original = b'{"version": 1, "services": {}, "saved_selection": "retain-me"}'
catalog_path.write_bytes(original)
read_text = Path.read_text
def fail_catalog_read(path: Path, *args: Any, **kwargs: Any) -> str:
"""Fail only the source read while leaving the actual writer usable."""
if path == catalog_path:
raise error_type("catalog temporarily unreadable")
return read_text(path, *args, **kwargs)
monkeypatch.setattr(Path, "read_text", fail_catalog_read)
service = ModelCatalogService(path=catalog_path)
with pytest.raises(error_type, match="catalog temporarily unreadable"):
if operation == "update":
service.update(lambda catalog: catalog.update({"new_setting": True}))
else:
getattr(service, operation)()
assert catalog_path.read_bytes() == original
def test_load_migrates_only_legacy_dashscope_stt_model(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text(
json.dumps(
{
"services": {
"stt": {
"active_profile_id": "dashscope-profile",
"active_model_id": "dashscope-model",
"profiles": [
{
"id": "dashscope-profile",
"binding": "bailian",
"models": [
{
"id": "dashscope-model",
"name": "paraformer-v2",
"model": "paraformer-v2",
},
{
"id": "custom-name",
"name": "My speech model",
"model": "paraformer-v2",
},
],
},
{
"id": "other-profile",
"binding": "openai",
"models": [
{
"id": "other-model",
"name": "paraformer-v2",
"model": "paraformer-v2",
}
],
},
],
}
}
}
),
encoding="utf-8",
)
loaded = ModelCatalogService(path=catalog_path).load()
models = loaded["services"]["stt"]["profiles"]
assert models[0]["models"][0]["model"] == "paraformer-realtime-v2"
assert models[0]["models"][0]["name"] == "paraformer-realtime-v2"
assert models[0]["models"][1]["model"] == "paraformer-realtime-v2"
assert models[0]["models"][1]["name"] == "My speech model"
assert models[1]["models"][0]["model"] == "paraformer-v2"
assert json.loads(catalog_path.read_text(encoding="utf-8")) == loaded
def test_load_migrates_legacy_stt_for_effective_provider_refs(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text(
json.dumps(
{
"connections": [
{"id": "dashscope", "provider": "dashscope"},
{"id": "openai", "provider": "openai"},
],
"services": {
"stt": {
"profiles": [
{
"id": "source",
"binding": "dashscope",
"models": [
{"id": "source-model", "model": "paraformer-realtime-v2"}
],
},
{
"id": "profile-ref",
"binding": "custom",
"provider_ref": {"connection_id": "dashscope"},
"models": [{"id": "profile-model", "model": "paraformer-v2"}],
},
{
"id": "model-ref",
"binding": "openai",
"models": [
{
"id": "model-ref-model",
"model": "paraformer-v2",
"provider_ref": {"service": "stt", "profile_id": "source"},
}
],
},
{
"id": "model-override",
"binding": "dashscope",
"provider_ref": {"connection_id": "dashscope"},
"models": [
{
"id": "other-model",
"model": "paraformer-v2",
"provider_ref": {"connection_id": "openai"},
}
],
},
{
"id": "profile-override",
"binding": "dashscope",
"provider_ref": {"connection_id": "openai"},
"models": [{"id": "other-profile-model", "model": "paraformer-v2"}],
},
]
}
},
}
),
encoding="utf-8",
)
loaded = ModelCatalogService(path=catalog_path).load()
models = {
profile["id"]: profile["models"][0]["model"]
for profile in loaded["services"]["stt"]["profiles"]
}
assert models == {
"source": "paraformer-realtime-v2",
"profile-ref": "paraformer-realtime-v2",
"model-ref": "paraformer-realtime-v2",
"model-override": "paraformer-v2",
"profile-override": "paraformer-v2",
}
assert json.loads(catalog_path.read_text(encoding="utf-8")) == loaded
def test_load_normalizes_wire_api_to_supported_profile_backends(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text(
json.dumps(
{
"services": {
"llm": {
"active_profile_id": "azure-profile",
"active_model_id": "azure-model",
"profiles": [
{
"id": "azure-profile",
"name": "Azure",
"binding": "azure_openai",
"wire_api": "responses",
"models": [
{
"id": "azure-model",
"name": "Deployment",
"model": "deployment-name",
}
],
},
{
"id": "custom-profile",
"name": "Custom",
"binding": "custom",
"wire_api": "unknown-protocol",
"models": [],
},
],
}
}
}
),
encoding="utf-8",
)
loaded = ModelCatalogService(path=catalog_path).load()
profiles = loaded["services"]["llm"]["profiles"]
assert profiles[0]["wire_api"] == "auto"
assert profiles[1]["wire_api"] == "auto"
def _gemini_embedding_catalog(path: Path, model: str) -> Path:
path.write_text(
json.dumps(
{
"services": {
"embedding": {
"active_profile_id": "gemini-profile",
"active_model_id": "gemini-model",
"profiles": [
{
"id": "gemini-profile",
"name": "Gemini",
"binding": "gemini",
"base_url": "",
"api_key": "test-key",
"models": [
{
"id": "gemini-model",
"name": model,
"model": model,
}
],
}
],
}
}
}
),
encoding="utf-8",
)
return path
def test_load_sets_gemini_native_endpoint_from_active_embedding_model(tmp_path: Path):
catalog_path = _gemini_embedding_catalog(tmp_path / "model_catalog.json", "gemini-embedding-2")
catalog = ModelCatalogService(path=catalog_path).load()
profile = catalog["services"]["embedding"]["profiles"][0]
assert profile["base_url"].endswith("/models/gemini-embedding-2:batchEmbedContents")
def test_load_keeps_older_gemini_embedding_models_on_the_openai_path(tmp_path: Path):
"""The native route sends a taskType and L2-normalizes, so moving an
existing gemini-embedding-001 profile there would change its document
vectors and invalidate the index built from them."""
catalog_path = _gemini_embedding_catalog(
tmp_path / "model_catalog.json", "gemini-embedding-001"
)
catalog = ModelCatalogService(path=catalog_path).load()
profile = catalog["services"]["embedding"]["profiles"][0]
assert profile["base_url"] == (
"https://generativelanguage.googleapis.com/v1beta/openai/embeddings"
)
def test_load_persists_normalized_active_ids(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
catalog_path.write_text(
json.dumps(
{
"services": {
"llm": {
"active_profile_id": "missing-profile",
"active_model_id": "missing-model",
"profiles": [
{
"id": "llm-profile-a",
"name": "A",
"binding": "openai",
"base_url": "https://example.test/v1",
"api_key": "sk",
"models": [
{
"id": "llm-model-a",
"name": "gpt",
"model": "gpt-test",
}
],
}
],
}
}
}
),
encoding="utf-8",
)
ModelCatalogService(path=catalog_path).load()
saved = json.loads(catalog_path.read_text(encoding="utf-8"))
llm = saved["services"]["llm"]
assert llm["active_profile_id"] == "llm-profile-a"
assert llm["active_model_id"] == "llm-model-a"
assert saved["services"]["embedding"]["profiles"] == []
assert saved["services"]["search"]["profiles"] == []
def test_update_serializes_concurrent_catalog_mutations(tmp_path: Path):
service = ModelCatalogService(path=tmp_path / "model_catalog.json")
initial = service.load()
initial["mutation_count"] = 0
service.save(initial)
def increment(_index: int) -> None:
def mutate(catalog: dict) -> None:
catalog["mutation_count"] = int(catalog.get("mutation_count", 0)) + 1
service.update(mutate)
with ThreadPoolExecutor(max_workers=4) as executor:
list(executor.map(increment, range(40)))
assert service.load()["mutation_count"] == 40
def test_atomic_save_leaves_no_temporary_file(tmp_path: Path):
catalog_path = tmp_path / "model_catalog.json"
service = ModelCatalogService(path=catalog_path)
service.save({"version": 1, "services": {}})
assert catalog_path.exists()
assert not list(tmp_path.glob(".model_catalog.json.*"))