Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(settings): send images for a validated image-description model

cgycorey731b5159a0cc
diff --git a/deeptutor/services/llm/image_description.py b/deeptutor/services/llm/image_description.py
index 8722ad8c8..299b2d11e 100644
--- a/deeptutor/services/llm/image_description.py
+++ b/deeptutor/services/llm/image_description.py
@@ -38,6 +38,15 @@ def resolve_image_description_config(selection: dict[str, str]) -> LLMConfig:
class _ImageDescriptionClient(LLMClient):
+ def supports_multimodal_images(self) -> bool:
+ """Trust the gate: this client only exists for a validated selection.
+
+ ``self.config.binding`` is the provider detected from ``base_url``, which
+ can differ from the declared binding ``resolve_image_description_config``
+ validated, so re-deriving capability here would veto a valid selection.
+ """
+ return True
+
async def complete(
self,
prompt: str,
@@ -52,6 +61,9 @@ class _ImageDescriptionClient(LLMClient):
prompt=prompt,
system_prompt=system_prompt or "You are a helpful assistant.",
messages=history or None,
+ # The validated selection decided images are supported; the factory
+ # would otherwise strip them for the model it detects on its own.
+ allow_image_fallback=False,
**kwargs,
)
diff --git a/tests/services/llm/test_image_description_selection.py b/tests/services/llm/test_image_description_selection.py
index 6254d0ff4..4a8582dff 100644
--- a/tests/services/llm/test_image_description_selection.py
+++ b/tests/services/llm/test_image_description_selection.py
@@ -5,13 +5,18 @@ import os
import pytest
+from deeptutor.services.config.model_catalog import ModelCatalogService
from deeptutor.services.llm import image_description as mod
from deeptutor.services.llm.config import LLMConfig
from deeptutor.services.model_selection.llm import LLMSelection
@pytest.fixture
-def selected(monkeypatch):
+def selected(monkeypatch, tmp_path):
+ # An empty catalog of our own: the gate must not read the developer's file.
+ monkeypatch.setattr(
+ mod, "get_model_catalog_service", lambda: ModelCatalogService(tmp_path / "catalog.json")
+ )
config = LLMConfig(
model="vision-model",
binding="openai",
diff --git a/tests/services/llm/test_image_description_vision_gate.py b/tests/services/llm/test_image_description_vision_gate.py
index dff9815e5..ec5831890 100644
--- a/tests/services/llm/test_image_description_vision_gate.py
+++ b/tests/services/llm/test_image_description_vision_gate.py
@@ -91,3 +91,36 @@ def test_declared_capability_beats_the_provider_table(tmp_path: Path, monkeypatc
assert _offered(service) == [False]
with pytest.raises(ValueError, match="must support image input"):
resolve_image_description_config(SELECTION)
+
+
+def test_validated_selection_accepts_images_despite_the_detected_binding(
+ tmp_path: Path, monkeypatch
+) -> None:
+ service = _service(tmp_path, [{"id": "m", "model": "llama3.2-vision"}])
+ _patch_catalog(monkeypatch, service)
+
+ config = resolve_image_description_config(SELECTION)
+ # The gateway URL makes the runtime detect ollama although ``openai`` was declared.
+ assert config.binding == "ollama"
+ client = image_description._ImageDescriptionClient(config, configure_env=False)
+ assert client.supports_multimodal_images() is True
+
+
+async def test_complete_sends_images_for_the_validated_selection(
+ tmp_path: Path, monkeypatch
+) -> None:
+ service = _service(tmp_path, [{"id": "m", "model": "llama3.2-vision"}])
+ _patch_catalog(monkeypatch, service)
+ config = resolve_image_description_config(SELECTION)
+ calls: list[dict[str, Any]] = []
+
+ async def complete(**kwargs: Any) -> str:
+ calls.append(kwargs)
+ return "a labelled diagram"
+
+ monkeypatch.setattr(image_description, "complete_with_config", complete)
+ client = image_description._ImageDescriptionClient(config, configure_env=False)
+
+ assert await client.complete("Describe", image_data="synthetic-image") == "a labelled diagram"
+ assert calls[0]["allow_image_fallback"] is False