DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
import json
import threading
from threading import Lock
import time
from typing import Any
from uuid import uuid4
from .context_window_detection import detect_context_window
from .embedding_endpoint import redact_embedding_endpoint_for_display
from .model_catalog import get_model_catalog_service, redact_catalog_secrets
from .provider_runtime import (
resolve_embedding_runtime_config,
resolve_llm_runtime_config,
resolve_search_runtime_config,
supported_search_providers_hint,
)
def _redact(value: str) -> str:
if not value:
return "(empty)"
if len(value) <= 8:
return "****"
return f"{value[:4]}...{value[-4:]}"
def _coerce_int(value: Any, default: int, *, minimum: int = 1) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
return default
return max(minimum, parsed)
def _coerce_float(value: Any, default: float) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
@dataclass
class TestRun:
id: str
service: str
status: str = "running"
events: list[dict[str, Any]] = field(default_factory=list)
lock: Lock = field(default_factory=Lock)
cancelled: bool = False
# A caller-supplied catalog is a draft, not authorization to apply it.
persist_results: bool = True
def emit(self, kind: str, message: str, **extra: Any) -> None:
payload = {
"type": kind,
"message": message,
"timestamp": time.time(),
**extra,
}
with self.lock:
self.events.append(payload)
def snapshot(self, start: int) -> list[dict[str, Any]]:
with self.lock:
return self.events[start:]
class ConfigTestRunner:
_instance: "ConfigTestRunner | None" = None
def __init__(self) -> None:
self._runs: dict[str, TestRun] = {}
self._lock = Lock()
@classmethod
def get_instance(cls) -> "ConfigTestRunner":
if cls._instance is None:
cls._instance = cls()
return cls._instance
def start(self, service: str, catalog: dict[str, Any] | None = None) -> TestRun:
run = TestRun(
id=f"{service}-{uuid4().hex[:10]}",
service=service,
persist_results=catalog is None,
)
with self._lock:
self._runs[run.id] = run
resolved = catalog or get_model_catalog_service().load()
thread = threading.Thread(target=self._run_sync, args=(run, resolved), daemon=True)
thread.start()
return run
def get(self, run_id: str) -> TestRun:
return self._runs[run_id]
def cancel(self, run_id: str) -> None:
self.get(run_id).cancelled = True
def _run_sync(self, run: TestRun, catalog: dict[str, Any]) -> None:
try:
service = run.service
profile = get_model_catalog_service().get_active_profile(catalog, service)
model = get_model_catalog_service().get_active_model(catalog, service)
run.emit("info", "Preparing configuration snapshot.")
if profile:
run.emit(
"config",
"Using active profile.",
profile={
"name": profile.get("name", ""),
"base_url": profile.get("base_url", ""),
"binding": profile.get("binding") or profile.get("provider"),
"api_key": _redact(str(profile.get("api_key", ""))),
"api_version": profile.get("api_version", ""),
},
model=model,
)
if service == "llm":
asyncio.run(self._test_llm(run, catalog))
elif service == "task":
asyncio.run(self._test_task(run, catalog))
elif service == "embedding":
asyncio.run(self._test_embedding(run, model or {}, catalog))
elif service == "search":
self._test_search(run, catalog)
elif service == "tts":
asyncio.run(self._test_tts(run, catalog))
elif service == "stt":
asyncio.run(self._test_stt(run, catalog))
elif service == "imagegen":
asyncio.run(self._test_imagegen(run, catalog))
elif service == "videogen":
asyncio.run(self._test_videogen(run, catalog))
else:
raise ValueError(f"Unsupported service: {service}")
if not run.cancelled and run.status == "running":
run.status = "completed"
message = {
"stt": "Speech endpoint responded to a silent connection probe. Recognition quality was not tested.",
"videogen": "Video task submitted. Generation and playback were not verified.",
}.get(service, f"{service.upper()} test completed successfully.")
run.emit("completed", message)
except Exception as exc:
run.status = "failed"
run.emit("failed", str(exc))
def _persist_embedding_dimension(
self,
catalog: dict[str, Any],
model: dict[str, Any],
actual_dimension: int,
) -> dict[str, Any]:
"""Write the probe-detected dim onto the active embedding model entry.
Called after every successful "Test connection" — the probe is the
single source of truth, so any prior catalog dim is overwritten.
Refreshes the embedding client singleton so subsequent embed calls
use the new dim.
"""
from deeptutor.services.embedding.client import reset_embedding_client
service = get_model_catalog_service()
if model is None:
return catalog
model["dimension"] = str(actual_dimension)
saved = service.save(catalog)
reset_embedding_client()
return redact_catalog_secrets(saved)
@staticmethod
def _capabilities_from_adapter(adapter: Any, model_name: str) -> dict[str, Any]:
"""Normalize an adapter's static-model knowledge into a uniform shape.
Adapters disagree on which keys they expose from ``get_model_info()``
(Cohere/Ollama omit ``supported_dimensions`` even though the data is
in their ``MODELS_INFO``). This helper folds both sources together
so the SSE event payload is always the same shape.
"""
info: dict[str, Any] = {}
try:
info = adapter.get_model_info() or {}
except Exception:
info = {}
models_info = getattr(adapter, "MODELS_INFO", {}) or {}
model_known = bool(model_name and model_name in models_info)
raw_supported = info.get("supported_dimensions")
if not isinstance(raw_supported, list):
entry = models_info.get(model_name) if model_known else None
if isinstance(entry, dict):
raw_supported = entry.get("dimensions")
else:
raw_supported = None
supported: list[int] = []
if isinstance(raw_supported, list):
for value in raw_supported:
try:
supported.append(int(value))
except (TypeError, ValueError):
continue
default_raw = info.get("dimensions")
try:
default_dim = int(default_raw) if default_raw is not None else 0
except (TypeError, ValueError):
default_dim = 0
return {
"default_dim": default_dim,
"supported_dimensions": supported,
"supports_variable_dimensions": bool(info.get("supports_variable_dimensions")),
"model_known": model_known,
}
async def _test_llm(self, run: TestRun, catalog: dict[str, Any]) -> None:
from deeptutor.services.llm import get_token_limit_kwargs
from deeptutor.services.llm.config import LLMConfig
from deeptutor.services.llm.factory import complete_with_config
run.emit("info", "Loading LLM config from the active catalog selection.")
resolved = resolve_llm_runtime_config(catalog=catalog, service_name=run.service)
llm_config = LLMConfig(
model=resolved.model,
api_key=resolved.api_key,
base_url=resolved.base_url,
effective_url=resolved.effective_url,
binding=resolved.binding,
provider_name=resolved.provider_name,
provider_mode=resolved.provider_mode,
api_version=resolved.api_version,
extra_headers=resolved.extra_headers,
wire_api=resolved.wire_api,
api_format=resolved.api_format,
reasoning_effort=resolved.reasoning_effort,
)
run.emit(
"info", f"Resolved model `{llm_config.model}` with binding `{llm_config.binding}`."
)
run.emit("info", f"Request target: {llm_config.base_url}")
# Reasoning models spend part of the budget on internal thinking;
# too tight a cap makes them return empty content. Configurable
# via diagnostics.llm_probe.max_tokens in agents.yaml.
from .loader import get_agent_params
probe_params = get_agent_params("llm_probe")
max_tokens = _coerce_int(probe_params.get("max_tokens"), 4096)
temperature = _coerce_float(probe_params.get("temperature"), 0.1)
token_kwargs: dict[str, Any] = get_token_limit_kwargs(
llm_config.model, max_tokens=max_tokens
)
run.emit("info", f"Token options: {json.dumps(token_kwargs)}")
if llm_config.reasoning_effort:
run.emit("info", f"Reasoning effort: {llm_config.reasoning_effort}")
response = await complete_with_config(
llm_config,
prompt="Reply with OK.",
system_prompt="This is a connection test. Respond briefly.",
temperature=temperature,
**token_kwargs,
)
snippet = (response or "").strip()
run.emit("response", "Received LLM response.", snippet=snippet[:400])
if not snippet:
raise ValueError("LLM returned an empty response.")
run.emit(
"info",
(
"Basic LLM completion succeeded. Chat additionally validates "
"streaming and provider tool compatibility at runtime."
),
)
run.emit("info", "Detecting model context window.")
detection = await detect_context_window(
llm_config,
on_log=lambda message: run.emit("info", message),
)
run.emit(
"context_window",
(
f"Context capacity unknown; conservative budget {detection.context_window} tokens."
if detection.source == "default"
else f"Detected context window {detection.context_window} tokens ({detection.source})."
),
context_window=detection.context_window,
source=detection.source,
detail=detection.detail,
detected_at=detection.detected_at,
)
run.emit(
"info",
"Context window detection is available in Settings and was not written automatically.",
)
async def _test_task(self, run: TestRun, catalog: dict[str, Any]) -> None:
from deeptutor.services.model_selection.tasks import task_service_configured
if not task_service_configured(catalog):
run.status = "completed"
run.emit(
"completed",
"No global task model is configured; background tasks inherit the main LLM.",
)
return
from deeptutor.services.llm.config import get_token_limit_kwargs
from deeptutor.services.llm.factory import complete_with_config
from deeptutor.services.model_selection.runtime import llm_config_from_resolved
run.emit("info", "Loading task model config from the active catalog selection.")
resolved = resolve_llm_runtime_config(catalog=catalog, service_name="task")
llm_config = llm_config_from_resolved(resolved)
run.emit(
"info",
f"Resolved task model `{llm_config.model}` with binding `{llm_config.binding}`.",
)
run.emit("info", f"Request target: {llm_config.base_url}")
# Mirror the production calls (conversation titles, composer starting
# points): short prompt, tiny token budget, no context-window probe.
run.emit("info", "Generating a title-style probe response (short, bounded).")
token_kwargs: dict[str, Any] = get_token_limit_kwargs(llm_config.model, max_tokens=80)
response = await complete_with_config(
llm_config,
prompt=(
"Write a title of at most four words for a conversation about: "
'"DeepTutor configuration health check".'
),
system_prompt="You write very short conversation titles.",
**token_kwargs,
)
snippet = (response or "").strip()
run.emit("response", "Received task model response.", snippet=snippet[:400])
if not snippet:
raise ValueError("Task model returned an empty response.")
run.emit(
"info",
"Task model completion succeeded. Conversation titles and composer "
"starting points use this same path at runtime.",
)
async def _test_embedding(
self, run: TestRun, model: dict[str, Any], catalog: dict[str, Any]
) -> None:
from deeptutor.services.embedding.client import EmbeddingClient
from deeptutor.services.embedding.config import EmbeddingConfig
run.emit("info", "Loading embedding config from the active catalog selection.")
resolved = resolve_embedding_runtime_config(catalog=catalog)
catalog_dim = _coerce_int(model.get("dimension"), 0, minimum=0)
# Force the smoke probe to send NO `dimensions=` parameter so we get
# the model's native max dim back. If we used the configured dim,
# Matryoshka models (OpenAI text-embedding-3-*, Cohere embed-v4,
# Jina v3/v4, DashScope qwen3-vl-embedding) would just truncate and
# return whatever we asked for — making "detected_dim" meaningless.
config = EmbeddingConfig(
model=resolved.model,
api_key=resolved.api_key,
base_url=resolved.base_url,
effective_url=resolved.effective_url,
binding=resolved.binding,
provider_name=resolved.provider_name,
provider_mode=resolved.provider_mode,
api_version=resolved.api_version,
extra_headers=resolved.extra_headers,
dim=0,
send_dimensions=False,
request_timeout=max(1, resolved.request_timeout),
batch_size=max(1, resolved.batch_size),
batch_delay=max(0.0, resolved.batch_delay),
)
run.emit(
"info", f"Resolved embedding model `{config.model}` with binding `{config.binding}`."
)
run.emit(
"info",
"Request target (POSTed exactly as shown in Settings): "
f"{redact_embedding_endpoint_for_display(config.base_url)}",
)
run.emit(
"info",
"Probing native max dimension with a small batch (sending no `dimensions=` param).",
)
client = EmbeddingClient(config)
probe_texts = [
"DeepTutor embedding smoke test",
"DeepTutor retrieval batch probe",
]
vectors = await client.embed(probe_texts)
if len(vectors) != len(probe_texts):
raise ValueError(
"Embedding service returned an unexpected number of vectors "
f"(expected {len(probe_texts)}, got {len(vectors)})."
)
if any(not vector for vector in vectors):
raise ValueError("Embedding service returned an empty vector.")
detected_dim = len(vectors[0])
if any(len(vector) != detected_dim for vector in vectors):
raise ValueError("Embedding service returned inconsistent vector dimensions.")
capabilities = self._capabilities_from_adapter(client.adapter, config.model)
supported = capabilities["supported_dimensions"]
default_dim = capabilities["default_dim"]
model_known = capabilities["model_known"]
# Probe is the source of truth: always overwrite the catalog dim with
# the detected value. Matryoshka users who want a truncated variant
# can edit the field manually after the test. Source code stays
# ``"detected"`` so the UI shows "Source: detected from API probe".
active_dim = detected_dim
active_source = "detected"
if catalog_dim and catalog_dim != detected_dim:
active_message = (
f"Catalog dim {catalog_dim}d overwritten with API probe value {detected_dim}d."
)
else:
active_message = f"Active dim {detected_dim}d set from API probe."
if not run.persist_results:
active_message = f"Detected {detected_dim}d. Model settings have not been changed."
run.emit(
"capabilities",
(
f"Probe returned {detected_dim}d. "
+ (
f"Static catalog: default {default_dim}d, "
f"supported {supported or '(fixed)'}, model recognized."
if model_known
else "Static catalog: model not recognized — using probe value as the only signal."
)
),
detected_dim=detected_dim,
default_dim=default_dim,
supported_dimensions=supported,
supports_variable_dimensions=capabilities["supports_variable_dimensions"],
model_known=model_known,
active_dim=active_dim,
active_dim_source=active_source,
)
run.emit(
"response",
"Embedding vector received.",
actual_dimension=detected_dim,
expected_dimension=catalog_dim or None,
)
# Refresh the cached ``supported_dimensions`` CSV on the model entry so
# the settings page can populate the dropdown without re-running the
# test. Empty list → empty string clears any stale cache. Mutation
# happens before the persist call so a single save round-trip carries
# both fields.
new_supported_csv = ",".join(str(d) for d in supported)
if (model.get("supported_dimensions") or "") != new_supported_csv:
model["supported_dimensions"] = new_supported_csv
run.emit(
"info",
active_message,
active_dim=active_dim,
active_dim_source=active_source,
)
# Existing live probes may refresh the cached dimensions. A draft probe
# must never promote its catalog (including provider/model selection).
if run.persist_results:
saved_catalog = self._persist_embedding_dimension(catalog, model, detected_dim)
run.emit(
"catalog",
"Saved detected embedding dimension to model_catalog.json.",
catalog=saved_catalog,
)
else:
run.emit(
"info", "Draft tested. Select the detected dimension in settings before applying."
)
def _test_search(self, run: TestRun, catalog: dict[str, Any]) -> None:
from deeptutor.services.settings.provider_probe import test_search_access
resolved = resolve_search_runtime_config(catalog=catalog)
# Probe the requested engine and draft credentials; ordinary search's
# graceful fallbacks must not turn a broken configuration into success.
result = test_search_access(
resolved.requested_provider,
resolved.base_url,
resolved.api_key,
proxy=resolved.proxy or "",
max_results=resolved.max_results,
)
run.emit(
"response",
"Search result received.",
answer_preview=result.answer[:240],
citation_count=len(result.citations),
search_result_count=len(result.search_results),
)
async def _test_tts(self, run: TestRun, catalog: dict[str, Any]) -> None:
import base64
from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config
from deeptutor.services.voice import synthesize_speech
run.emit("info", "Loading TTS config from the active catalog selection.")
resolved = resolve_tts_runtime_config(catalog=catalog)
run.emit(
"info",
f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
f"voice `{resolved.voice or '(default)'}`).",
)
run.emit("info", f"Request target: {resolved.base_url}")
sample = "DeepTutor voice check. 这是一段语音合成测试。"
run.emit("info", "Synthesizing a short sample clip.")
audio, content_type = await synthesize_speech(sample, catalog=catalog)
run.emit(
"response",
f"Received {len(audio)} bytes of {content_type}.",
audio_base64=base64.b64encode(audio).decode("ascii"),
content_type=content_type,
bytes=len(audio),
)
async def _test_stt(self, run: TestRun, catalog: dict[str, Any]) -> None:
import io
import wave
from deeptutor.services.config.provider_runtime import resolve_stt_runtime_config
from deeptutor.services.voice import transcribe_audio
run.emit("info", "Loading STT config from the active catalog selection.")
resolved = resolve_stt_runtime_config(catalog=catalog)
run.emit(
"info",
f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
f"style `{resolved.request_style}`).",
)
run.emit("info", f"Request target: {resolved.base_url}")
# One second of 16 kHz mono silence — a valid WAV that exercises the
# full upload + auth + model path. Most providers return empty/near-empty
# text; a clean HTTP 200 confirms the connection is configured correctly.
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16000)
wav.writeframes(b"\x00\x00" * 16000)
run.emit("info", "Uploading a 1s silent probe clip to validate the endpoint.")
transcript = await transcribe_audio(
buffer.getvalue(),
catalog=catalog,
filename="probe.wav",
content_type="audio/wav",
)
run.emit(
"response",
"Transcription endpoint responded successfully.",
snippet=(transcript or "(empty — expected for a silent clip)")[:200],
)
async def _test_imagegen(self, run: TestRun, catalog: dict[str, Any]) -> None:
import base64
from deeptutor.services.config.provider_runtime import resolve_imagegen_runtime_config
from deeptutor.services.imagegen import generate_image
run.emit("info", "Loading image-generation config from the active catalog selection.")
resolved = resolve_imagegen_runtime_config(catalog=catalog)
run.emit(
"info",
f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
f"size `{resolved.size or '(default)'}`).",
)
run.emit("info", f"Request target: {resolved.base_url}")
run.emit("info", "Generating a single test image (this is a billable call).")
images = await generate_image(
"A small minimalist test icon of a blue book on a white background.",
catalog=catalog,
n=1,
)
if not images:
raise ValueError("Image provider returned no images.")
image_bytes, content_type = images[0]
run.emit(
"response",
f"Received {len(image_bytes)} bytes of {content_type}.",
image_base64=base64.b64encode(image_bytes).decode("ascii"),
content_type=content_type,
bytes=len(image_bytes),
)
async def _test_videogen(self, run: TestRun, catalog: dict[str, Any]) -> None:
from deeptutor.services.config.provider_runtime import resolve_videogen_runtime_config
from deeptutor.services.videogen import probe_video
run.emit("info", "Loading video-generation config from the active catalog selection.")
resolved = resolve_videogen_runtime_config(catalog=catalog)
run.emit(
"info",
f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
f"adapter `{resolved.adapter}`).",
)
run.emit("info", f"Request target: {resolved.base_url}")
run.emit(
"info",
"Submitting a probe task to validate endpoint + auth + model. "
"The render is not awaited (it is slow and billable).",
)
task_id = await probe_video("A short test clip of a calm ocean wave.", catalog=catalog)
run.emit(
"response",
"Video task accepted — connection is valid.",
task_id=task_id,
)
def get_config_test_runner() -> ConfigTestRunner:
return ConfigTestRunner.get_instance()
__all__ = ["ConfigTestRunner", "TestRun", "get_config_test_runner"]