Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

25.4 KB
Raw Blame
from __future__ import annotations

import asyncio
from dataclasses import dataclass, field
import json
import threading
from threading import Lock
import time
from typing import Any
from uuid import uuid4

from .context_window_detection import detect_context_window
from .embedding_endpoint import redact_embedding_endpoint_for_display
from .model_catalog import get_model_catalog_service, redact_catalog_secrets
from .provider_runtime import (
    resolve_embedding_runtime_config,
    resolve_llm_runtime_config,
    resolve_search_runtime_config,
    supported_search_providers_hint,
)


def _redact(value: str) -> str:
    if not value:
        return "(empty)"
    if len(value) <= 8:
        return "****"
    return f"{value[:4]}...{value[-4:]}"


def _coerce_int(value: Any, default: int, *, minimum: int = 1) -> int:
    try:
        parsed = int(value)
    except (TypeError, ValueError):
        return default
    return max(minimum, parsed)


def _coerce_float(value: Any, default: float) -> float:
    try:
        return float(value)
    except (TypeError, ValueError):
        return default


@dataclass
class TestRun:
    id: str
    service: str
    status: str = "running"
    events: list[dict[str, Any]] = field(default_factory=list)
    lock: Lock = field(default_factory=Lock)
    cancelled: bool = False
    # A caller-supplied catalog is a draft, not authorization to apply it.
    persist_results: bool = True

    def emit(self, kind: str, message: str, **extra: Any) -> None:
        payload = {
            "type": kind,
            "message": message,
            "timestamp": time.time(),
            **extra,
        }
        with self.lock:
            self.events.append(payload)

    def snapshot(self, start: int) -> list[dict[str, Any]]:
        with self.lock:
            return self.events[start:]


class ConfigTestRunner:
    _instance: "ConfigTestRunner | None" = None

    def __init__(self) -> None:
        self._runs: dict[str, TestRun] = {}
        self._lock = Lock()

    @classmethod
    def get_instance(cls) -> "ConfigTestRunner":
        if cls._instance is None:
            cls._instance = cls()
        return cls._instance

    def start(self, service: str, catalog: dict[str, Any] | None = None) -> TestRun:
        run = TestRun(
            id=f"{service}-{uuid4().hex[:10]}",
            service=service,
            persist_results=catalog is None,
        )
        with self._lock:
            self._runs[run.id] = run
        resolved = catalog or get_model_catalog_service().load()
        thread = threading.Thread(target=self._run_sync, args=(run, resolved), daemon=True)
        thread.start()
        return run

    def get(self, run_id: str) -> TestRun:
        return self._runs[run_id]

    def cancel(self, run_id: str) -> None:
        self.get(run_id).cancelled = True

    def _run_sync(self, run: TestRun, catalog: dict[str, Any]) -> None:
        try:
            service = run.service
            profile = get_model_catalog_service().get_active_profile(catalog, service)
            model = get_model_catalog_service().get_active_model(catalog, service)

            run.emit("info", "Preparing configuration snapshot.")
            if profile:
                run.emit(
                    "config",
                    "Using active profile.",
                    profile={
                        "name": profile.get("name", ""),
                        "base_url": profile.get("base_url", ""),
                        "binding": profile.get("binding") or profile.get("provider"),
                        "api_key": _redact(str(profile.get("api_key", ""))),
                        "api_version": profile.get("api_version", ""),
                    },
                    model=model,
                )

            if service == "llm":
                asyncio.run(self._test_llm(run, catalog))
            elif service == "task":
                asyncio.run(self._test_task(run, catalog))
            elif service == "embedding":
                asyncio.run(self._test_embedding(run, model or {}, catalog))
            elif service == "search":
                self._test_search(run, catalog)
            elif service == "tts":
                asyncio.run(self._test_tts(run, catalog))
            elif service == "stt":
                asyncio.run(self._test_stt(run, catalog))
            elif service == "imagegen":
                asyncio.run(self._test_imagegen(run, catalog))
            elif service == "videogen":
                asyncio.run(self._test_videogen(run, catalog))
            else:
                raise ValueError(f"Unsupported service: {service}")
            if not run.cancelled and run.status == "running":
                run.status = "completed"
                message = {
                    "stt": "Speech endpoint responded to a silent connection probe. Recognition quality was not tested.",
                    "videogen": "Video task submitted. Generation and playback were not verified.",
                }.get(service, f"{service.upper()} test completed successfully.")
                run.emit("completed", message)
        except Exception as exc:
            run.status = "failed"
            run.emit("failed", str(exc))

    def _persist_embedding_dimension(
        self,
        catalog: dict[str, Any],
        model: dict[str, Any],
        actual_dimension: int,
    ) -> dict[str, Any]:
        """Write the probe-detected dim onto the active embedding model entry.

        Called after every successful "Test connection" — the probe is the
        single source of truth, so any prior catalog dim is overwritten.
        Refreshes the embedding client singleton so subsequent embed calls
        use the new dim.
        """
        from deeptutor.services.embedding.client import reset_embedding_client

        service = get_model_catalog_service()
        if model is None:
            return catalog
        model["dimension"] = str(actual_dimension)
        saved = service.save(catalog)
        reset_embedding_client()
        return redact_catalog_secrets(saved)

    @staticmethod
    def _capabilities_from_adapter(adapter: Any, model_name: str) -> dict[str, Any]:
        """Normalize an adapter's static-model knowledge into a uniform shape.

        Adapters disagree on which keys they expose from ``get_model_info()``
        (Cohere/Ollama omit ``supported_dimensions`` even though the data is
        in their ``MODELS_INFO``). This helper folds both sources together
        so the SSE event payload is always the same shape.
        """
        info: dict[str, Any] = {}
        try:
            info = adapter.get_model_info() or {}
        except Exception:
            info = {}
        models_info = getattr(adapter, "MODELS_INFO", {}) or {}
        model_known = bool(model_name and model_name in models_info)

        raw_supported = info.get("supported_dimensions")
        if not isinstance(raw_supported, list):
            entry = models_info.get(model_name) if model_known else None
            if isinstance(entry, dict):
                raw_supported = entry.get("dimensions")
            else:
                raw_supported = None
        supported: list[int] = []
        if isinstance(raw_supported, list):
            for value in raw_supported:
                try:
                    supported.append(int(value))
                except (TypeError, ValueError):
                    continue

        default_raw = info.get("dimensions")
        try:
            default_dim = int(default_raw) if default_raw is not None else 0
        except (TypeError, ValueError):
            default_dim = 0

        return {
            "default_dim": default_dim,
            "supported_dimensions": supported,
            "supports_variable_dimensions": bool(info.get("supports_variable_dimensions")),
            "model_known": model_known,
        }

    async def _test_llm(self, run: TestRun, catalog: dict[str, Any]) -> None:
        from deeptutor.services.llm import get_token_limit_kwargs
        from deeptutor.services.llm.config import LLMConfig
        from deeptutor.services.llm.factory import complete_with_config

        run.emit("info", "Loading LLM config from the active catalog selection.")
        resolved = resolve_llm_runtime_config(catalog=catalog, service_name=run.service)
        llm_config = LLMConfig(
            model=resolved.model,
            api_key=resolved.api_key,
            base_url=resolved.base_url,
            effective_url=resolved.effective_url,
            binding=resolved.binding,
            provider_name=resolved.provider_name,
            provider_mode=resolved.provider_mode,
            api_version=resolved.api_version,
            extra_headers=resolved.extra_headers,
            wire_api=resolved.wire_api,
            api_format=resolved.api_format,
            reasoning_effort=resolved.reasoning_effort,
        )
        run.emit(
            "info", f"Resolved model `{llm_config.model}` with binding `{llm_config.binding}`."
        )
        run.emit("info", f"Request target: {llm_config.base_url}")
        # Reasoning models spend part of the budget on internal thinking;
        # too tight a cap makes them return empty content. Configurable
        # via diagnostics.llm_probe.max_tokens in agents.yaml.
        from .loader import get_agent_params

        probe_params = get_agent_params("llm_probe")
        max_tokens = _coerce_int(probe_params.get("max_tokens"), 4096)
        temperature = _coerce_float(probe_params.get("temperature"), 0.1)
        token_kwargs: dict[str, Any] = get_token_limit_kwargs(
            llm_config.model, max_tokens=max_tokens
        )
        run.emit("info", f"Token options: {json.dumps(token_kwargs)}")
        if llm_config.reasoning_effort:
            run.emit("info", f"Reasoning effort: {llm_config.reasoning_effort}")
        response = await complete_with_config(
            llm_config,
            prompt="Reply with OK.",
            system_prompt="This is a connection test. Respond briefly.",
            temperature=temperature,
            **token_kwargs,
        )
        snippet = (response or "").strip()
        run.emit("response", "Received LLM response.", snippet=snippet[:400])
        if not snippet:
            raise ValueError("LLM returned an empty response.")
        run.emit(
            "info",
            (
                "Basic LLM completion succeeded. Chat additionally validates "
                "streaming and provider tool compatibility at runtime."
            ),
        )

        run.emit("info", "Detecting model context window.")
        detection = await detect_context_window(
            llm_config,
            on_log=lambda message: run.emit("info", message),
        )
        run.emit(
            "context_window",
            (
                f"Context capacity unknown; conservative budget {detection.context_window} tokens."
                if detection.source == "default"
                else f"Detected context window {detection.context_window} tokens ({detection.source})."
            ),
            context_window=detection.context_window,
            source=detection.source,
            detail=detection.detail,
            detected_at=detection.detected_at,
        )
        run.emit(
            "info",
            "Context window detection is available in Settings and was not written automatically.",
        )

    async def _test_task(self, run: TestRun, catalog: dict[str, Any]) -> None:
        from deeptutor.services.model_selection.tasks import task_service_configured

        if not task_service_configured(catalog):
            run.status = "completed"
            run.emit(
                "completed",
                "No global task model is configured; background tasks inherit the main LLM.",
            )
            return

        from deeptutor.services.llm.config import get_token_limit_kwargs
        from deeptutor.services.llm.factory import complete_with_config
        from deeptutor.services.model_selection.runtime import llm_config_from_resolved

        run.emit("info", "Loading task model config from the active catalog selection.")
        resolved = resolve_llm_runtime_config(catalog=catalog, service_name="task")
        llm_config = llm_config_from_resolved(resolved)
        run.emit(
            "info",
            f"Resolved task model `{llm_config.model}` with binding `{llm_config.binding}`.",
        )
        run.emit("info", f"Request target: {llm_config.base_url}")
        # Mirror the production calls (conversation titles, composer starting
        # points): short prompt, tiny token budget, no context-window probe.
        run.emit("info", "Generating a title-style probe response (short, bounded).")
        token_kwargs: dict[str, Any] = get_token_limit_kwargs(llm_config.model, max_tokens=80)
        response = await complete_with_config(
            llm_config,
            prompt=(
                "Write a title of at most four words for a conversation about: "
                '"DeepTutor configuration health check".'
            ),
            system_prompt="You write very short conversation titles.",
            **token_kwargs,
        )
        snippet = (response or "").strip()
        run.emit("response", "Received task model response.", snippet=snippet[:400])
        if not snippet:
            raise ValueError("Task model returned an empty response.")
        run.emit(
            "info",
            "Task model completion succeeded. Conversation titles and composer "
            "starting points use this same path at runtime.",
        )

    async def _test_embedding(
        self, run: TestRun, model: dict[str, Any], catalog: dict[str, Any]
    ) -> None:
        from deeptutor.services.embedding.client import EmbeddingClient
        from deeptutor.services.embedding.config import EmbeddingConfig

        run.emit("info", "Loading embedding config from the active catalog selection.")
        resolved = resolve_embedding_runtime_config(catalog=catalog)
        catalog_dim = _coerce_int(model.get("dimension"), 0, minimum=0)
        # Force the smoke probe to send NO `dimensions=` parameter so we get
        # the model's native max dim back. If we used the configured dim,
        # Matryoshka models (OpenAI text-embedding-3-*, Cohere embed-v4,
        # Jina v3/v4, DashScope qwen3-vl-embedding) would just truncate and
        # return whatever we asked for — making "detected_dim" meaningless.
        config = EmbeddingConfig(
            model=resolved.model,
            api_key=resolved.api_key,
            base_url=resolved.base_url,
            effective_url=resolved.effective_url,
            binding=resolved.binding,
            provider_name=resolved.provider_name,
            provider_mode=resolved.provider_mode,
            api_version=resolved.api_version,
            extra_headers=resolved.extra_headers,
            dim=0,
            send_dimensions=False,
            request_timeout=max(1, resolved.request_timeout),
            batch_size=max(1, resolved.batch_size),
            batch_delay=max(0.0, resolved.batch_delay),
        )
        run.emit(
            "info", f"Resolved embedding model `{config.model}` with binding `{config.binding}`."
        )
        run.emit(
            "info",
            "Request target (POSTed exactly as shown in Settings): "
            f"{redact_embedding_endpoint_for_display(config.base_url)}",
        )
        run.emit(
            "info",
            "Probing native max dimension with a small batch (sending no `dimensions=` param).",
        )
        client = EmbeddingClient(config)
        probe_texts = [
            "DeepTutor embedding smoke test",
            "DeepTutor retrieval batch probe",
        ]
        vectors = await client.embed(probe_texts)
        if len(vectors) != len(probe_texts):
            raise ValueError(
                "Embedding service returned an unexpected number of vectors "
                f"(expected {len(probe_texts)}, got {len(vectors)})."
            )
        if any(not vector for vector in vectors):
            raise ValueError("Embedding service returned an empty vector.")
        detected_dim = len(vectors[0])
        if any(len(vector) != detected_dim for vector in vectors):
            raise ValueError("Embedding service returned inconsistent vector dimensions.")

        capabilities = self._capabilities_from_adapter(client.adapter, config.model)
        supported = capabilities["supported_dimensions"]
        default_dim = capabilities["default_dim"]
        model_known = capabilities["model_known"]

        # Probe is the source of truth: always overwrite the catalog dim with
        # the detected value. Matryoshka users who want a truncated variant
        # can edit the field manually after the test. Source code stays
        # ``"detected"`` so the UI shows "Source: detected from API probe".
        active_dim = detected_dim
        active_source = "detected"
        if catalog_dim and catalog_dim != detected_dim:
            active_message = (
                f"Catalog dim {catalog_dim}d overwritten with API probe value {detected_dim}d."
            )
        else:
            active_message = f"Active dim {detected_dim}d set from API probe."
        if not run.persist_results:
            active_message = f"Detected {detected_dim}d. Model settings have not been changed."

        run.emit(
            "capabilities",
            (
                f"Probe returned {detected_dim}d. "
                + (
                    f"Static catalog: default {default_dim}d, "
                    f"supported {supported or '(fixed)'}, model recognized."
                    if model_known
                    else "Static catalog: model not recognized — using probe value as the only signal."
                )
            ),
            detected_dim=detected_dim,
            default_dim=default_dim,
            supported_dimensions=supported,
            supports_variable_dimensions=capabilities["supports_variable_dimensions"],
            model_known=model_known,
            active_dim=active_dim,
            active_dim_source=active_source,
        )

        run.emit(
            "response",
            "Embedding vector received.",
            actual_dimension=detected_dim,
            expected_dimension=catalog_dim or None,
        )

        # Refresh the cached ``supported_dimensions`` CSV on the model entry so
        # the settings page can populate the dropdown without re-running the
        # test. Empty list → empty string clears any stale cache. Mutation
        # happens before the persist call so a single save round-trip carries
        # both fields.
        new_supported_csv = ",".join(str(d) for d in supported)
        if (model.get("supported_dimensions") or "") != new_supported_csv:
            model["supported_dimensions"] = new_supported_csv

        run.emit(
            "info",
            active_message,
            active_dim=active_dim,
            active_dim_source=active_source,
        )

        # Existing live probes may refresh the cached dimensions. A draft probe
        # must never promote its catalog (including provider/model selection).
        if run.persist_results:
            saved_catalog = self._persist_embedding_dimension(catalog, model, detected_dim)
            run.emit(
                "catalog",
                "Saved detected embedding dimension to model_catalog.json.",
                catalog=saved_catalog,
            )
        else:
            run.emit(
                "info", "Draft tested. Select the detected dimension in settings before applying."
            )

    def _test_search(self, run: TestRun, catalog: dict[str, Any]) -> None:
        from deeptutor.services.settings.provider_probe import test_search_access

        resolved = resolve_search_runtime_config(catalog=catalog)
        # Probe the requested engine and draft credentials; ordinary search's
        # graceful fallbacks must not turn a broken configuration into success.
        result = test_search_access(
            resolved.requested_provider,
            resolved.base_url,
            resolved.api_key,
            proxy=resolved.proxy or "",
            max_results=resolved.max_results,
        )
        run.emit(
            "response",
            "Search result received.",
            answer_preview=result.answer[:240],
            citation_count=len(result.citations),
            search_result_count=len(result.search_results),
        )

    async def _test_tts(self, run: TestRun, catalog: dict[str, Any]) -> None:
        import base64

        from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config
        from deeptutor.services.voice import synthesize_speech

        run.emit("info", "Loading TTS config from the active catalog selection.")
        resolved = resolve_tts_runtime_config(catalog=catalog)
        run.emit(
            "info",
            f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
            f"voice `{resolved.voice or '(default)'}`).",
        )
        run.emit("info", f"Request target: {resolved.base_url}")
        sample = "DeepTutor voice check. 这是一段语音合成测试。"
        run.emit("info", "Synthesizing a short sample clip.")
        audio, content_type = await synthesize_speech(sample, catalog=catalog)
        run.emit(
            "response",
            f"Received {len(audio)} bytes of {content_type}.",
            audio_base64=base64.b64encode(audio).decode("ascii"),
            content_type=content_type,
            bytes=len(audio),
        )

    async def _test_stt(self, run: TestRun, catalog: dict[str, Any]) -> None:
        import io
        import wave

        from deeptutor.services.config.provider_runtime import resolve_stt_runtime_config
        from deeptutor.services.voice import transcribe_audio

        run.emit("info", "Loading STT config from the active catalog selection.")
        resolved = resolve_stt_runtime_config(catalog=catalog)
        run.emit(
            "info",
            f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
            f"style `{resolved.request_style}`).",
        )
        run.emit("info", f"Request target: {resolved.base_url}")

        # One second of 16 kHz mono silence — a valid WAV that exercises the
        # full upload + auth + model path. Most providers return empty/near-empty
        # text; a clean HTTP 200 confirms the connection is configured correctly.
        buffer = io.BytesIO()
        with wave.open(buffer, "wb") as wav:
            wav.setnchannels(1)
            wav.setsampwidth(2)
            wav.setframerate(16000)
            wav.writeframes(b"\x00\x00" * 16000)
        run.emit("info", "Uploading a 1s silent probe clip to validate the endpoint.")
        transcript = await transcribe_audio(
            buffer.getvalue(),
            catalog=catalog,
            filename="probe.wav",
            content_type="audio/wav",
        )
        run.emit(
            "response",
            "Transcription endpoint responded successfully.",
            snippet=(transcript or "(empty — expected for a silent clip)")[:200],
        )

    async def _test_imagegen(self, run: TestRun, catalog: dict[str, Any]) -> None:
        import base64

        from deeptutor.services.config.provider_runtime import resolve_imagegen_runtime_config
        from deeptutor.services.imagegen import generate_image

        run.emit("info", "Loading image-generation config from the active catalog selection.")
        resolved = resolve_imagegen_runtime_config(catalog=catalog)
        run.emit(
            "info",
            f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
            f"size `{resolved.size or '(default)'}`).",
        )
        run.emit("info", f"Request target: {resolved.base_url}")
        run.emit("info", "Generating a single test image (this is a billable call).")
        images = await generate_image(
            "A small minimalist test icon of a blue book on a white background.",
            catalog=catalog,
            n=1,
        )
        if not images:
            raise ValueError("Image provider returned no images.")
        image_bytes, content_type = images[0]
        run.emit(
            "response",
            f"Received {len(image_bytes)} bytes of {content_type}.",
            image_base64=base64.b64encode(image_bytes).decode("ascii"),
            content_type=content_type,
            bytes=len(image_bytes),
        )

    async def _test_videogen(self, run: TestRun, catalog: dict[str, Any]) -> None:
        from deeptutor.services.config.provider_runtime import resolve_videogen_runtime_config
        from deeptutor.services.videogen import probe_video

        run.emit("info", "Loading video-generation config from the active catalog selection.")
        resolved = resolve_videogen_runtime_config(catalog=catalog)
        run.emit(
            "info",
            f"Resolved model `{resolved.model}` (provider `{resolved.provider_name}`, "
            f"adapter `{resolved.adapter}`).",
        )
        run.emit("info", f"Request target: {resolved.base_url}")
        run.emit(
            "info",
            "Submitting a probe task to validate endpoint + auth + model. "
            "The render is not awaited (it is slow and billable).",
        )
        task_id = await probe_video("A short test clip of a calm ocean wave.", catalog=catalog)
        run.emit(
            "response",
            "Video task accepted — connection is valid.",
            task_id=task_id,
        )


def get_config_test_runner() -> ConfigTestRunner:
    return ConfigTestRunner.get_instance()


__all__ = ["ConfigTestRunner", "TestRun", "get_config_test_runner"]