DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Provider/model separation keeps existing catalog identities and runtime behavior."""
from contextlib import nullcontext
from copy import deepcopy
import pytest
from deeptutor.services.config.model_catalog import ModelCatalogService, redact_catalog_secrets
from deeptutor.services.config.provider_links import resolve_profile_provider
from deeptutor.services.config.provider_runtime import (
resolve_embedding_runtime_config,
resolve_llm_runtime_config,
resolve_search_runtime_config,
)
from deeptutor.services.model_selection.llm import list_llm_options
from deeptutor.services.settings.registry_edit import merge_registry_edit
def fixture(tmp_path):
store = ModelCatalogService(tmp_path / "catalog.json")
catalog = store.load()
for service in ("llm", "task", "embedding", "search", "tts", "stt", "imagegen", "videogen"):
catalog["services"][service] = {
"active_profile_id": service,
"active_model_id": service + "-one",
"profiles": [
{
"id": service,
"name": "Old account",
"binding": "custom",
"provider": "tavily",
"base_url": "https://legacy.test/v1/embeddings"
if service == "embedding"
else "https://legacy.test/v1",
"api_key": "old-secret",
"extra_headers": {"X-Saved": "header-secret"},
"unknown_profile_setting": {"stay": True},
"models": []
if service == "search"
else [
{
"id": service + "-" + suffix,
"name": suffix,
"model": "existing-" + suffix,
"context_window": "64000",
"dimension": "768",
"voice": "saved-voice",
"duration": "7",
"unknown_model_setting": [1, 2],
}
for suffix in ("one", "two")
],
}
],
}
catalog["unknown_catalog_setting"] = {"keep": True}
return store, store.save(catalog)
def provider_edit(**fields):
return {
"kind": "provider",
"ref": {"connection_id": "account"},
"fields": {
"provider": "custom",
"name": "New account",
"base_url": "https://new.test/v1",
"api_key": "new-secret",
**fields,
},
}
def add_model(catalog, service="llm", ref=None):
model = {
"id": "new-model",
"model": "new-model-id",
"name": "My model",
"provider_ref": ref or {"connection_id": "account", "binding": "custom"},
}
return merge_registry_edit(
catalog, {"kind": "model", "service": service, "profile_id": "new-profile", "model": model}
)
def test_legacy_roundtrip_and_provider_only_edit_keeps_all_models_defaults_and_secrets(tmp_path):
store, catalog = fixture(tmp_path)
before = deepcopy(catalog)
patch = {
"kind": "provider",
"ref": {"service": "llm", "profile_id": "llm"},
"fields": {"name": "Renamed account", "api_key": "***", "models": []},
}
after = store.save(merge_registry_edit(catalog, patch))
before["services"]["llm"]["profiles"][0]["name"] = "Renamed account"
assert after == before
assert store.load() == before
def test_new_model_reuses_connection_and_later_key_changes_without_copying_secrets(tmp_path):
store, current = fixture(tmp_path)
catalog = add_model(merge_registry_edit(current, provider_edit()))
catalog = merge_registry_edit(
catalog,
{"kind": "default", "service": "llm", "profile_id": "new-profile", "model_id": "new-model"},
)
catalog = store.save(catalog)
assert catalog["services"]["llm"]["profiles"][-1]["api_key"] == ""
assert resolve_llm_runtime_config(catalog, service=store).api_key == "new-secret"
catalog = store.save(
merge_registry_edit(catalog, provider_edit(api_key="rotated-key", api_format="anthropic"))
)
resolved = resolve_llm_runtime_config(catalog, service=store)
assert resolved.api_key == "rotated-key"
assert resolved.api_format == "anthropic"
assert catalog["services"]["llm"]["profiles"][0] == current["services"]["llm"]["profiles"][0]
options = list_llm_options(catalog)["options"]
assert [(o["profile_id"], o["model_id"]) for o in options[:2]] == [
("llm", "llm-one"),
("llm", "llm-two"),
]
assert options[-1]["model_name"] == "My model"
assert options[-1]["profile_name"] == "New account"
@pytest.mark.parametrize("service", ["llm", "task"])
def test_minimax_model_references_follow_protocol_changes_without_rewriting_models(
tmp_path, service
):
store, current = fixture(tmp_path)
catalog = merge_registry_edit(
current, provider_edit(provider="minimax", base_url="", api_format="auto")
)
catalog = add_model(
catalog,
service=service,
ref={
"connection_id": "account",
"binding": "minimax",
"default_base_url": "https://api.minimax.io/v1",
},
)
catalog = merge_registry_edit(
catalog,
{
"kind": "default",
"service": service,
"profile_id": "new-profile",
"model_id": "new-model",
},
)
catalog = store.save(catalog)
original_models = deepcopy(catalog["services"])
assert (
resolve_llm_runtime_config(catalog, service=store, service_name=service).effective_url
== "https://api.minimax.io/v1"
)
catalog = store.save(
merge_registry_edit(
catalog,
provider_edit(
provider="minimax",
base_url="",
api_format="anthropic",
),
)
)
resolved = resolve_llm_runtime_config(catalog, service=store, service_name=service)
assert resolved.api_format == "anthropic"
assert resolved.effective_url == "https://api.minimax.io/anthropic"
assert catalog["services"] == original_models
catalog = store.save(
merge_registry_edit(
catalog,
provider_edit(
provider="minimax",
base_url="https://gateway.test/minimax",
api_format="anthropic",
),
)
)
assert (
resolve_llm_runtime_config(catalog, service=store, service_name=service).effective_url
== "https://gateway.test/minimax"
)
catalog = store.save(
merge_registry_edit(
catalog,
provider_edit(
provider="minimax",
base_url="",
api_format="openai_chat",
),
)
)
resolved = resolve_llm_runtime_config(catalog, service=store, service_name=service)
assert resolved.api_format == "openai_chat"
assert resolved.effective_url == "https://api.minimax.io/v1"
assert catalog["services"] == original_models
def test_switch_one_legacy_model_provider_without_touching_sibling_or_unknown_fields(tmp_path):
store, current = fixture(tmp_path)
catalog = merge_registry_edit(current, provider_edit())
original_profile = deepcopy(catalog["services"]["llm"]["profiles"][0])
model = deepcopy(original_profile["models"][0])
model["provider_ref"] = {"connection_id": "account", "binding": "custom"}
catalog = store.save(
merge_registry_edit(
catalog, {"kind": "model", "service": "llm", "profile_id": "llm", "model": model}
)
)
assert resolve_llm_runtime_config(catalog, service=store).api_key == "new-secret"
assert (
resolve_llm_runtime_config(
catalog, service=store, llm_selection={"profile_id": "llm", "model_id": "llm-two"}
).api_key
== "old-secret"
)
assert catalog["services"]["llm"]["profiles"][0]["models"][1] == original_profile["models"][1]
assert (
catalog["services"]["llm"]["profiles"][0]["extra_headers"]
== original_profile["extra_headers"]
)
assert (
store.get_active_model(catalog, "llm")
is catalog["services"]["llm"]["profiles"][0]["models"][0]
)
def test_missing_source_rejected_and_referenced_source_cannot_be_removed(tmp_path):
_, catalog = fixture(tmp_path)
with pytest.raises(ValueError, match="no longer configured"):
add_model(catalog)
catalog = add_model(merge_registry_edit(catalog, provider_edit()))
with pytest.raises(ValueError, match="used by models"):
merge_registry_edit(
catalog, {"kind": "provider", "ref": {"connection_id": "account"}, "delete": True}
)
with pytest.raises(ValueError, match="used by models"):
merge_registry_edit(
catalog,
{"kind": "provider", "ref": {"service": "llm", "profile_id": "llm"}, "delete": True},
)
def test_model_save_does_not_promote_parked_credentials_or_sibling_drafts(tmp_path):
_, current = fixture(tmp_path)
parked = deepcopy(current)
parked["services"]["llm"]["profiles"][0]["api_key"] = "unsaved-provider-key"
parked["services"]["llm"]["profiles"][0]["models"][1]["model"] = "unsaved-sibling"
model = deepcopy(current["services"]["llm"]["profiles"][0]["models"][0])
model["name"] = "Renamed model"
live = merge_registry_edit(
current,
{"kind": "model", "service": "llm", "profile_id": "llm", "model": model},
stored_draft=parked,
)
assert live["services"]["llm"]["profiles"][0]["api_key"] == "old-secret"
assert (
live["services"]["llm"]["profiles"][0]["models"][1]
== current["services"]["llm"]["profiles"][0]["models"][1]
)
@pytest.mark.asyncio
async def test_registry_api_preserves_sibling_drafts_extensions_and_secret_masks(
tmp_path, monkeypatch
):
from deeptutor.api.routers import settings as router
from deeptutor.services.config.settings_draft import SettingsDraftService
store, current = fixture(tmp_path)
drafts = SettingsDraftService(tmp_path / "draft.json")
parked = deepcopy(current)
parked["services"]["llm"]["profiles"][0]["models"][1]["model"] = "unsaved-sibling"
parked["services"]["llm"]["profiles"][0]["api_key"] = "unsaved-key"
drafts.save({"catalog": parked, "extensions": {"appearance": {"keep": True}}})
monkeypatch.setattr(router, "_require_settings_admin", lambda: None)
monkeypatch.setattr(router, "get_model_catalog_service", lambda: store)
monkeypatch.setattr(router, "get_settings_draft_service", lambda: drafts)
monkeypatch.setattr(router, "_runtime_catalog_write", nullcontext)
model = deepcopy(current["services"]["llm"]["profiles"][0]["models"][0])
model["name"] = "Saved name"
response = await router.apply_registry_edit(
router.RegistryEditPayload(kind="model", service="llm", profile_id="llm", model=model)
)
assert response["catalog"]["services"]["llm"]["profiles"][0]["api_key"] == "***"
assert store.load()["services"]["llm"]["profiles"][0]["api_key"] == "old-secret"
assert (
drafts.load()["catalog"]["services"]["llm"]["profiles"][0]["models"][1]["model"]
== "unsaved-sibling"
)
assert drafts.load()["catalog"]["services"]["llm"]["profiles"][0]["api_key"] == "unsaved-key"
assert drafts.load()["extensions"] == {"appearance": {"keep": True}}
def test_search_reference_preserves_source_and_uses_saved_credentials(tmp_path):
store, catalog = fixture(tmp_path)
result = merge_registry_edit(
catalog,
{
"kind": "model",
"service": "search",
"profile_id": "new-search",
"config": {
"provider_ref": {"service": "search", "profile_id": "search", "binding": "tavily"},
"display_name": "My search",
"max_results": 3,
},
},
)
result = merge_registry_edit(
result, {"kind": "default", "service": "search", "profile_id": "new-search"}
)
result = store.save(result)
config = resolve_search_runtime_config(result, service=store)
assert config.provider == "tavily"
assert config.api_key == "old-secret"
assert config.max_results == 3
result = merge_registry_edit(
result, {"kind": "model", "service": "search", "profile_id": "search", "delete": True}
)
assert result["services"]["search"]["profiles"][0]["provider_only"] is True
assert resolve_search_runtime_config(result, service=store).api_key == "old-secret"
@pytest.mark.parametrize(
"binding,base,expected",
[
("custom", "https://example.test/v1", "https://example.test/v1/embeddings"),
("ollama", "http://localhost:11434/v1", "http://localhost:11434/api/embed"),
("cohere", "https://api.cohere.com/v2", "https://api.cohere.com/v2/embed"),
],
)
def test_new_embedding_reference_resolves_service_endpoint(tmp_path, binding, base, expected):
store, catalog = fixture(tmp_path)
catalog = merge_registry_edit(catalog, provider_edit(provider=binding, base_url=base))
catalog = add_model(catalog, "embedding", {"connection_id": "account", "binding": binding})
catalog = merge_registry_edit(
catalog,
{
"kind": "default",
"service": "embedding",
"profile_id": "new-profile",
"model_id": "new-model",
},
)
assert resolve_embedding_runtime_config(catalog, service=store).base_url == expected
def test_empty_selected_key_never_borrows_another_account_key(tmp_path):
store, catalog = fixture(tmp_path)
catalog = add_model(merge_registry_edit(catalog, provider_edit(api_key="")))
catalog = merge_registry_edit(
catalog,
{"kind": "default", "service": "llm", "profile_id": "new-profile", "model_id": "new-model"},
)
assert resolve_llm_runtime_config(catalog, service=store).api_key == ""
def test_owner_bound_accounts_cannot_be_copied_into_unmanaged_models(tmp_path):
_, catalog = fixture(tmp_path)
catalog["services"]["llm"]["profiles"][0]["owner_bound"] = True
with pytest.raises(ValueError, match="sign-in"):
add_model(catalog, ref={"service": "llm", "profile_id": "llm", "binding": "openai_codex"})
def test_task_choice_saves_only_assignment_and_preserves_models(tmp_path):
_, catalog = fixture(tmp_path)
before = deepcopy(catalog["services"]["task"]["profiles"])
result = merge_registry_edit(
catalog,
{
"kind": "task_choice",
"task": {
"mode": "reference",
"selection": {"profile_id": "llm", "model_id": "llm-two"},
},
},
)
assert result["services"]["task"]["profiles"] == before
assert result["services"]["task"]["selection"] == {"profile_id": "llm", "model_id": "llm-two"}
with pytest.raises(ValueError):
merge_registry_edit(
catalog,
{
"kind": "task_choice",
"task": {
"mode": "reference",
"selection": {"profile_id": "missing", "model_id": "missing"},
},
},
)
def test_one_task_can_be_pinned_without_moving_the_global_choice(tmp_path):
_, catalog = fixture(tmp_path)
result = merge_registry_edit(
catalog,
{
"kind": "task_choice",
"task": {
"task_kind": "session_title",
"mode": "profiles",
"active_profile_id": "task",
"active_model_id": "task-two",
},
},
)
bucket = result["services"]["task"]
assert bucket["overrides"] == {
"session_title": {
"mode": "profiles",
"active_profile_id": "task",
"active_model_id": "task-two",
}
}
# The global choice is untouched — pinning one task is not choosing for all.
assert bucket["active_model_id"] == "task-one"
# Back to following the global model: the row disappears rather than becoming
# a fourth stored mode.
cleared = merge_registry_edit(
result, {"kind": "task_choice", "task": {"task_kind": "session_title", "mode": "global"}}
)
assert "overrides" not in cleared["services"]["task"]
with pytest.raises(ValueError, match="Unknown background task"):
merge_registry_edit(
catalog,
{"kind": "task_choice", "task": {"task_kind": "not_a_task", "mode": "inherit"}},
)
with pytest.raises(ValueError):
merge_registry_edit(
catalog,
{
"kind": "task_choice",
"task": {
"task_kind": "session_title",
"mode": "profiles",
"active_profile_id": "task",
"active_model_id": "missing",
},
},
)
def test_capability_detection_uses_metadata_not_names():
from deeptutor.services.settings.provider_probe import detect_capabilities
assert detect_capabilities([{"id": "gpt-embedding-search-video"}]) == []
assert detect_capabilities(
[
{
"id": "image",
"architecture": {"input_modalities": ["text"], "output_modalities": ["image"]},
}
]
) == [
{"category": "generation", "evidence": "metadata"},
{"category": "imagegen", "evidence": "metadata"},
]
assert detect_capabilities(
[
{"supportedGenerationMethods": ["embedContent"]},
{"architecture": {"input_modalities": ["image"], "output_modalities": ["text"]}},
]
) == [
{"category": "embedding", "evidence": "metadata"},
{"category": "llm", "evidence": "metadata"},
]
assert detect_capabilities([{"architecture": "invalid", "output_modalities": None}]) == []
def test_search_probe_uses_draft_credentials_and_cannot_fall_back(tmp_path, monkeypatch):
from types import SimpleNamespace
from deeptutor.services.config.test_runner import ConfigTestRunner, TestRun
from deeptutor.services.search import providers
from deeptutor.services.search.types import WebSearchResponse
_, catalog = fixture(tmp_path)
catalog["services"]["search"]["profiles"][0]["api_key"] = "draft-key"
received = []
def get(name, **kwargs):
received.append((name, kwargs))
return SimpleNamespace(
search=lambda *a, **k: WebSearchResponse(query="test", answer="found", provider=name)
)
monkeypatch.setattr(providers, "get_provider", get)
ConfigTestRunner()._test_search(TestRun(id="search-test", service="search"), catalog)
assert received[0][0] == "tavily"
assert received[0][1]["api_key"] == "draft-key"
catalog["services"]["search"]["profiles"][0]["api_key"] = ""
with pytest.raises(ValueError, match="api_key"):
ConfigTestRunner()._test_search(TestRun(id="search-test", service="search"), catalog)
assert len(received) == 1
def test_clear_legacy_context_alias_and_preserve_key_pool_on_rename(tmp_path):
store, catalog = fixture(tmp_path)
profile = catalog["services"]["llm"]["profiles"][0]
profile["api_key"] = ["key-one", "key-two"]
profile["models"][0].pop("context_window")
profile["models"][0]["context_window_tokens"] = 32000
edit = {
"kind": "provider",
"ref": {"service": "llm", "profile_id": "llm"},
"fields": {"name": "Account with key pool", "api_key": ["***", "***"]},
}
renamed = merge_registry_edit(catalog, edit)
assert renamed["services"]["llm"]["profiles"][0]["api_key"] == ["key-one", "key-two"]
model = deepcopy(profile["models"][0])
model.pop("context_window_tokens")
model.update(context_window="", context_window_source="default")
result = merge_registry_edit(
catalog, {"kind": "model", "service": "llm", "profile_id": "llm", "model": model}
)
assert resolve_llm_runtime_config(result, service=store).context_window is None
def test_managed_account_and_model_names_survive_catalog_refresh(tmp_path):
from deeptutor.services.codex_auth.contracts import CatalogSnapshot, CodexModel
from deeptutor.services.codex_auth.service import (
CODEX_PROFILE_ID,
reconcile_codex_catalog_update,
sync_codex_catalog,
)
store, catalog = fixture(tmp_path)
model = CodexModel(
slug="test-model",
display_name="Provider name",
priority=1,
visibility="list",
default_reasoning_level="medium",
supported_reasoning_levels=("medium", "high"),
supports_reasoning_summary=True,
supports_parallel_tool_calls=True,
use_responses_lite=False,
context_window=128000,
)
snapshot = CatalogSnapshot(
models=(model,),
source="live",
fetched_at=1000,
etag=None,
generation=1,
account_hash="account",
)
sync_codex_catalog(store, snapshot, account_id="account")
current = store.load()
proposed = deepcopy(current)
profile = next(
p for p in proposed["services"]["llm"]["profiles"] if p["id"] == CODEX_PROFILE_ID
)
profile["user_name"] = "My signed-in account"
profile["models"][0]["user_name"] = "My model label"
profile["models"][0]["context_window"] = "999999"
saved = store.save(reconcile_codex_catalog_update(current, proposed))
managed = next(p for p in saved["services"]["llm"]["profiles"] if p["id"] == CODEX_PROFILE_ID)
assert managed["name"] == "My signed-in account"
assert managed["models"][0]["name"] == "My model label"
assert managed["models"][0]["context_window"] == "128000"
sync_codex_catalog(store, snapshot, account_id="account")
managed = next(
p for p in store.load()["services"]["llm"]["profiles"] if p["id"] == CODEX_PROFILE_ID
)
assert managed["name"] == "My signed-in account"
assert managed["models"][0]["name"] == "My model label"
@pytest.mark.parametrize("link", ["reference", "legacy_connection"])
def test_manual_service_adapter_and_url_survive_save_and_reach_runtime(tmp_path, link):
store, catalog = fixture(tmp_path)
overrides = {
"tts": {"enabled": True, "binding": "minimax", "base_url": "https://speech.test/v1"}
}
catalog = merge_registry_edit(catalog, provider_edit(service_overrides=overrides))
profile = catalog["services"]["tts"]["profiles"][0]
if link == "reference":
profile["provider_ref"] = {"connection_id": "account", "binding": "custom"}
else:
profile["connection_id"] = "account"
saved = store.save(catalog)
source = next(c for c in saved["connections"] if c["id"] == "account")
assert source["service_overrides"] == overrides
resolved = resolve_profile_provider(saved, "tts", saved["services"]["tts"]["profiles"][0])
assert resolved["binding"] == "minimax"
assert resolved["base_url"] == "https://speech.test/v1"
assert resolved["api_key"] == "new-secret"
def test_manual_service_changes_do_not_rekey_or_remove_existing_models(tmp_path):
store, catalog = fixture(tmp_path)
before = deepcopy(catalog["services"])
catalog = merge_registry_edit(
catalog,
{
"kind": "provider",
"ref": {"service": "llm", "profile_id": "llm"},
"fields": {
"service_overrides": {
"llm": {"enabled": False},
"tts": {"enabled": True, "binding": "custom"},
}
},
},
)
saved = store.save(catalog)
saved["services"]["llm"]["profiles"][0].pop("service_overrides")
assert saved["services"] == before