DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Task call sites actually run inside the scope they claim to.
Configuring a task model is only worth anything if the LLM call at the other
end resolves it. Every test here asserts the model *observed from inside the
call*: a scope wrapped around the wrong statement, or a call site that was
never wired at all, still looks correct from the outside. That goes double for
a per-task pin, whose whole point is that one call site resolves something the
others do not.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
import pytest
from deeptutor.services.config import model_catalog as model_catalog_module
from deeptutor.services.config.model_catalog import ModelCatalogService
from deeptutor.services.llm.config import clear_llm_config_cache, get_llm_config
pytestmark = pytest.mark.real_llm_resolver
class _FakePathService:
def __init__(self, root: Path) -> None:
self._root = root
def get_settings_file(self, name: str) -> Path:
return self._root / "settings" / f"{name}.json"
def _pin(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
task_model: str | None,
*,
overrides: dict[str, Any] | None = None,
) -> None:
"""Point every catalog reader at a tmp catalog, optionally with a task model.
Patching the path service rather than ``get_model_catalog_service`` is
deliberate: several modules import that function by value, so patching it
in one place would leave the others resolving the real user's catalog.
"""
fake = _FakePathService(tmp_path)
monkeypatch.setattr(model_catalog_module, "get_path_service", lambda: fake)
service = ModelCatalogService(path=fake.get_settings_file("model_catalog"))
catalog = service.load()
catalog["services"]["llm"]["profiles"] = [
{
"id": "llm-1",
"name": "OpenAI",
"binding": "openai",
"base_url": "https://api.openai.com/v1",
"api_key": "sk-test",
"models": [
{"id": "model-default", "model": "gpt-5"},
{"id": "model-task", "model": "gpt-5-mini"},
],
}
]
catalog["services"]["llm"]["active_profile_id"] = "llm-1"
catalog["services"]["llm"]["active_model_id"] = "model-default"
if task_model:
catalog["services"]["task"]["profiles"] = [
{
"id": "task-1",
"name": "OpenAI",
"binding": "openai",
"base_url": "https://api.openai.com/v1",
"api_key": "sk-test",
"models": [
{"id": "task-model", "model": task_model},
{"id": "task-second", "model": "gpt-5-nano"},
],
}
]
catalog["services"]["task"]["active_profile_id"] = "task-1"
catalog["services"]["task"]["active_model_id"] = "task-model"
if overrides is not None:
catalog["services"]["task"]["overrides"] = overrides
service.save(catalog)
# get_instance memoizes per resolved path; every reader must land on the
# instance holding this tmp file rather than a cached admin-scope one.
monkeypatch.setattr(
ModelCatalogService, "get_instance", classmethod(lambda cls, path=None: service)
)
clear_llm_config_cache()
@pytest.fixture(autouse=True)
def _clear_llm_cache() -> Any:
clear_llm_config_cache()
yield
clear_llm_config_cache()
def _run_starters(monkeypatch: pytest.MonkeyPatch, observed: list[str]) -> None:
from deeptutor.services import suggestions
import deeptutor.services.llm as llm
async def _complete(prompt: str, **kwargs: Any) -> str:
observed.append(get_llm_config().model)
return "[]"
monkeypatch.setattr(llm, "complete", _complete)
material = suggestions._Material(
profile="A learner.",
topics=[suggestions._Topic(surface="chat", label="Agentic RAG", days_ago=1)],
)
asyncio.run(suggestions._generate("en", material))
def test_starter_generation_calls_the_task_model(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(tmp_path, monkeypatch, "gpt-5-mini")
observed: list[str] = []
_run_starters(monkeypatch, observed)
assert observed == ["gpt-5-mini"]
def test_starter_generation_inherits_when_unconfigured(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(tmp_path, monkeypatch, None)
observed: list[str] = []
_run_starters(monkeypatch, observed)
assert observed == ["gpt-5"]
def test_a_pinned_task_runs_on_its_own_model(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(
tmp_path,
monkeypatch,
"gpt-5-mini",
overrides={
"chat_starters": {
"mode": "profiles",
"active_profile_id": "task-1",
"active_model_id": "task-second",
}
},
)
observed: list[str] = []
_run_starters(monkeypatch, observed)
assert observed == ["gpt-5-nano"]
def test_a_pin_on_another_task_leaves_this_one_on_the_global_model(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(
tmp_path,
monkeypatch,
"gpt-5-mini",
overrides={
"session_title": {
"mode": "profiles",
"active_profile_id": "task-1",
"active_model_id": "task-second",
}
},
)
observed: list[str] = []
_run_starters(monkeypatch, observed)
assert observed == ["gpt-5-mini"]
def test_a_task_pinned_to_the_chat_model_ignores_the_task_model(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Opting one call *out* of the task model is a real choice, not a no-op."""
_pin(tmp_path, monkeypatch, "gpt-5-mini", overrides={"chat_starters": {"mode": "inherit"}})
observed: list[str] = []
_run_starters(monkeypatch, observed)
assert observed == ["gpt-5"]
class _Store:
def __init__(self) -> None:
self.title = ""
async def get_session(self, session_id: str) -> dict[str, Any]:
return {"id": session_id, "title": "New conversation"}
async def get_messages(self, session_id: str) -> list[dict[str, Any]]:
return [
{"role": "user", "content": "Explain the chain rule."},
{"role": "assistant", "content": "It composes derivatives."},
]
async def update_session_title(self, session_id: str, title: str) -> None:
self.title = title
class _Runtime:
def __init__(self) -> None:
self.store = _Store()
async def _publish_live_event(self, execution: Any, event: Any) -> Any:
return event
def _run_title(monkeypatch: pytest.MonkeyPatch, observed: list[str]) -> _Runtime:
import deeptutor.services.llm as llm
from deeptutor.services.session import turn_runtime as turn_runtime_module
async def _stream(**kwargs: Any):
observed.append(get_llm_config().model)
yield "Chain rule basics"
monkeypatch.setattr(llm, "stream", _stream)
runtime = _Runtime()
asyncio.run(
turn_runtime_module.TurnRuntimeManager._maybe_generate_session_title(
runtime,
execution=object(),
session_id="session-1",
ui_language="en",
)
)
return runtime
def test_title_generation_calls_the_task_model(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(tmp_path, monkeypatch, "gpt-5-mini")
observed: list[str] = []
runtime = _run_title(monkeypatch, observed)
assert observed == ["gpt-5-mini"]
assert runtime.store.title == "Chain rule basics"
def test_title_generation_inherits_the_turn_model_when_unconfigured(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_pin(tmp_path, monkeypatch, None)
from deeptutor.services.llm.config import (
LLMConfig,
reset_scoped_llm_config,
set_scoped_llm_config,
)
# Stand in for the turn's own scope: unpinned must keep resolving whatever
# the conversation is already running on, not the global default.
token = set_scoped_llm_config(LLMConfig(model="turn-model", api_key="sk-test"))
try:
observed: list[str] = []
_run_title(monkeypatch, observed)
finally:
reset_scoped_llm_config(token)
assert observed == ["turn-model"]