Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(config): probe the selected task model without losing provider settings

Bingxi Zhao (Frank)2918f3cf4959
diff --git a/deeptutor/services/config/test_runner.py b/deeptutor/services/config/test_runner.py
index c38f8a2af..b61fcfac3 100644
--- a/deeptutor/services/config/test_runner.py
+++ b/deeptutor/services/config/test_runner.py
@@ -122,7 +122,7 @@ class ConfigTestRunner:
model=model,
)
- if service in {"llm", "task"}:
+ if service == "llm":
asyncio.run(self._test_llm(run, catalog))
elif service == "task":
asyncio.run(self._test_task(run, catalog))
@@ -296,20 +296,20 @@ class ConfigTestRunner:
)
async def _test_task(self, run: TestRun, catalog: dict[str, Any]) -> None:
- from deeptutor.services.llm import complete as llm_complete
- from deeptutor.services.llm.config import get_token_limit_kwargs
- from deeptutor.services.model_selection.runtime import llm_config_from_resolved
from deeptutor.services.model_selection.tasks import task_service_configured
if not task_service_configured(catalog):
run.status = "completed"
run.emit(
"completed",
- "Task model skipped because no active task profile is configured. "
- "Conversation titles and composer starting points inherit the main LLM.",
+ "No global task model is configured; background tasks inherit the main LLM.",
)
return
+ from deeptutor.services.llm.config import get_token_limit_kwargs
+ from deeptutor.services.llm.factory import complete_with_config
+ from deeptutor.services.model_selection.runtime import llm_config_from_resolved
+
run.emit("info", "Loading task model config from the active catalog selection.")
resolved = resolve_llm_runtime_config(catalog=catalog, service_name="task")
llm_config = llm_config_from_resolved(resolved)
@@ -321,20 +321,14 @@ class ConfigTestRunner:
# Mirror the production calls (conversation titles, composer starting
# points): short prompt, tiny token budget, no context-window probe.
run.emit("info", "Generating a title-style probe response (short, bounded).")
- token_kwargs = get_token_limit_kwargs(llm_config.model, max_tokens=64)
- response = await llm_complete(
- model=llm_config.model,
+ token_kwargs = get_token_limit_kwargs(llm_config.model, max_tokens=80)
+ response = await complete_with_config(
+ llm_config,
prompt=(
- 'Write a title of at most four words for a conversation about: '
+ "Write a title of at most four words for a conversation about: "
'"DeepTutor configuration health check".'
),
system_prompt="You write very short conversation titles.",
- binding=llm_config.binding,
- api_key=llm_config.api_key or "sk-no-key-required",
- base_url=llm_config.effective_url or llm_config.base_url or "",
- api_version=llm_config.api_version,
- extra_headers=llm_config.extra_headers,
- reasoning_effort=llm_config.reasoning_effort,
**token_kwargs,
)
snippet = (response or "").strip()
diff --git a/tests/services/config/test_task_probe.py b/tests/services/config/test_task_probe.py
new file mode 100644
index 000000000..ab3f64449
--- /dev/null
+++ b/tests/services/config/test_task_probe.py
@@ -0,0 +1,76 @@
+"""The Settings task-model probe follows the background-task selection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from deeptutor.services.config import test_runner as test_runner_module
+from deeptutor.services.config.model_catalog import ModelCatalogService
+from deeptutor.services.config.test_runner import ConfigTestRunner, TestRun
+from deeptutor.services.llm import factory as llm_factory
+
+
+def test_unconfigured_task_probe_skips_the_llm_request(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ service = ModelCatalogService(path=tmp_path / "model_catalog.json")
+ monkeypatch.setattr(test_runner_module, "get_model_catalog_service", lambda: service)
+
+ async def unexpected_llm_probe(_run: TestRun, _catalog: dict) -> None:
+ raise AssertionError("an inherited task model should not start an LLM probe")
+
+ runner = ConfigTestRunner()
+ monkeypatch.setattr(runner, "_test_llm", unexpected_llm_probe)
+ run = TestRun(id="task-inherits", service="task")
+
+ runner._run_sync(run, service.load())
+
+ assert run.status == "completed"
+ assert any(
+ event["type"] == "completed" and "inherit the main LLM" in event["message"]
+ for event in run.events
+ )
+
+
+async def test_configured_task_probe_preserves_the_selected_wire_format(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ service = ModelCatalogService(path=tmp_path / "model_catalog.json")
+ catalog = service.load()
+ catalog["services"]["task"] = {
+ "active_profile_id": "task-profile",
+ "active_model_id": "task-model",
+ "profiles": [
+ {
+ "id": "task-profile",
+ "name": "Responses relay",
+ "binding": "custom",
+ "base_url": "https://relay.example/v1",
+ "api_key": "test-key",
+ "api_format": "openai_responses",
+ "models": [{"id": "task-model", "model": "gpt-4o-mini"}],
+ }
+ ],
+ }
+ captured: dict = {}
+
+ async def fake_complete(config, **kwargs):
+ captured["config"] = config
+ captured["kwargs"] = kwargs
+ return "Configuration check"
+
+ monkeypatch.setattr(llm_factory, "complete_with_config", fake_complete)
+ run = TestRun(id="task-responses", service="task")
+
+ await ConfigTestRunner()._test_task(run, catalog)
+
+ assert captured["config"].model == "gpt-4o-mini"
+ assert captured["config"].wire_api == "responses"
+ assert captured["config"].api_format == "openai_responses"
+ assert captured["kwargs"]["prompt"].startswith("Write a title")
+ assert any(key in captured["kwargs"] for key in ("max_tokens", "max_completion_tokens"))
+ assert not any(event["type"] == "context_window" for event in run.events)
+ assert any(event["type"] == "response" for event in run.events)