Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(settings): judge vision support from the declared provider

cgycorey0a4dc9ad4d15
diff --git a/deeptutor/services/llm/capabilities.py b/deeptutor/services/llm/capabilities.py
index 927eea0f5..def76e36a 100644
--- a/deeptutor/services/llm/capabilities.py
+++ b/deeptutor/services/llm/capabilities.py
@@ -630,6 +630,22 @@ def supports_vision(binding: str, model: str | None = None) -> bool:
return bool(value)
+def supports_vision_declared(
+ declared: object, binding: str | None, model: str | None = None
+) -> bool:
+ """Vision support, with a model's own declaration winning over the tables.
+
+ Catalog capability overrides register against the provider the runtime may
+ detect from ``base_url``, which is not always the profile's declared
+ binding, so the declaration has to be honoured before the lookup. The
+ settings picker and the save-time gate both use this rule, otherwise they
+ disagree about the same model (#2037 follow-up).
+ """
+ if type(declared) is bool:
+ return declared
+ return supports_vision(binding, model)
+
+
def supports_vision_url(binding: str, model: str | None = None) -> bool:
"""Whether the provider accepts remote URL image references.
diff --git a/deeptutor/services/llm/image_description.py b/deeptutor/services/llm/image_description.py
index 31472f876..8722ad8c8 100644
--- a/deeptutor/services/llm/image_description.py
+++ b/deeptutor/services/llm/image_description.py
@@ -2,20 +2,37 @@
from typing import Any
+from deeptutor.services.config.model_catalog import get_model_catalog_service
from deeptutor.services.config.runtime_settings import load_document_parsing_settings
-from deeptutor.services.model_selection.llm import LLMSelection
+from deeptutor.services.model_selection.llm import LLMSelection, list_llm_options
from deeptutor.services.model_selection.runtime import resolve_llm_config_for_selection
-from .capabilities import supports_vision
+from .capabilities import supports_vision_declared
from .client import LLMClient, get_llm_client
from .config import LLMConfig
from .factory import complete_with_config
+def _vision_support(selection: LLMSelection, config: LLMConfig) -> bool:
+ """Take the verdict from the picker so the two can never disagree.
+
+ Falls back to the resolved binding for selections the global catalog does
+ not list, such as a per-user model.
+ """
+ for option in list_llm_options(get_model_catalog_service().load())["options"]:
+ if (option["profile_id"], option["model_id"]) == (
+ selection.profile_id,
+ selection.model_id,
+ ):
+ return option["supports_vision"]
+ return supports_vision_declared(None, config.binding, config.model)
+
+
def resolve_image_description_config(selection: dict[str, str]) -> LLMConfig:
"""Validate an explicit selection without installing a global override."""
- config = resolve_llm_config_for_selection(LLMSelection.from_payload(selection))
- if not supports_vision(config.binding, config.model):
+ payload = LLMSelection.from_payload(selection)
+ config = resolve_llm_config_for_selection(payload)
+ if not _vision_support(payload, config):
raise ValueError("The image description model must support image input.")
return config
diff --git a/deeptutor/services/model_selection/llm.py b/deeptutor/services/model_selection/llm.py
index 910ea3fa6..3cdb63256 100644
--- a/deeptutor/services/model_selection/llm.py
+++ b/deeptutor/services/model_selection/llm.py
@@ -100,7 +100,7 @@ def list_llm_options(catalog: dict[str, Any]) -> dict[str, Any]:
options: list[dict[str, Any]] = []
# Local import: the llm package imports back into this module at load time.
- from deeptutor.services.llm.capabilities import supports_vision
+ from deeptutor.services.llm.capabilities import supports_vision_declared
for profile in service.get("profiles", []) or []:
if not isinstance(profile, dict):
@@ -159,13 +159,13 @@ def list_llm_options(catalog: dict[str, Any]) -> dict[str, Any]:
for capability in ("reasoning", "vision"):
if type(declared.get(capability)) is bool:
option[f"declared_{capability}"] = declared[capability]
- # The settings UI filters vision models on this field. The catalog's
- # own declaration wins over the process-global override table, which
+ # The settings UI filters vision models on this field. A model's own
+ # declaration wins over the process-global override table, which
# only refreshes on the next runtime resolution -- otherwise a
# freshly declared model stays hidden until something else resolves
- # (#2037). Same rule as `lightrag/roles.py`.
- option["supports_vision"] = option.get(
- "declared_vision", supports_vision(model_provider, model_value)
+ # (#2037). Same rule as `lightrag/roles.py` and the save-time gate.
+ option["supports_vision"] = supports_vision_declared(
+ declared.get("vision"), model_provider, model_value
)
options.append(option)
diff --git a/tests/services/llm/test_image_description_selection.py b/tests/services/llm/test_image_description_selection.py
index ff72b7c12..6254d0ff4 100644
--- a/tests/services/llm/test_image_description_selection.py
+++ b/tests/services/llm/test_image_description_selection.py
@@ -32,7 +32,7 @@ def selected(monkeypatch):
return config
monkeypatch.setattr(mod, "resolve_llm_config_for_selection", resolve)
- monkeypatch.setattr(mod, "supports_vision", lambda *_: True)
+ monkeypatch.setattr(mod, "supports_vision_declared", lambda *_: True)
return config
@@ -74,7 +74,7 @@ async def test_explicit_model_preserves_full_config_and_process_state(monkeypatc
def test_explicit_text_model_is_rejected(monkeypatch, selected):
- monkeypatch.setattr(mod, "supports_vision", lambda *_: False)
+ monkeypatch.setattr(mod, "supports_vision_declared", lambda *_: False)
with pytest.raises(ValueError, match="must support image input"):
mod.get_image_description_client()
diff --git a/tests/services/llm/test_image_description_vision_gate.py b/tests/services/llm/test_image_description_vision_gate.py
new file mode 100644
index 000000000..dff9815e5
--- /dev/null
+++ b/tests/services/llm/test_image_description_vision_gate.py
@@ -0,0 +1,93 @@
+"""The image-description gate and the model picker must reach the same verdict.
+
+Both read the same catalog, so a model the picker offers has to be savable and a
+model it hides has to be refused. They disagreed whenever the profile's declared
+binding differed from the provider the runtime detects from ``base_url``: the
+picker judged capability from the declared provider, the gate from the detected
+one (#2037 follow-up).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from deeptutor.services.config import model_catalog, provider_runtime
+from deeptutor.services.config.model_catalog import ModelCatalogService
+from deeptutor.services.llm import image_description
+from deeptutor.services.llm.capabilities import set_catalog_capability_overrides
+from deeptutor.services.llm.image_description import resolve_image_description_config
+from deeptutor.services.model_selection.llm import list_llm_options
+
+GATEWAY_BASE_URL = "http://localhost:11434/v1"
+SELECTION = {"profile_id": "gateway", "model_id": "m"}
+
+
[email protected](autouse=True)
+def _clean_capability_overrides():
+ """The override table is process-global; keep these tests independent."""
+ set_catalog_capability_overrides([])
+ yield
+ set_catalog_capability_overrides([])
+
+
+def _service(
+ tmp_path: Path, models: list[dict[str, Any]], *, binding: str = "openai"
+) -> ModelCatalogService:
+ service = ModelCatalogService(path=tmp_path / "model_catalog.json")
+ catalog = service.load()
+ catalog["services"]["llm"]["profiles"] = [
+ {
+ "id": "gateway",
+ "name": "OpenAI binding pointed at a local gateway",
+ "binding": binding,
+ "base_url": GATEWAY_BASE_URL,
+ "api_key": "synthetic-key",
+ "models": models,
+ }
+ ]
+ service.save(catalog)
+ return service
+
+
+def _patch_catalog(monkeypatch, service: ModelCatalogService) -> None:
+ """Point every catalog lookup on this path at the fixture."""
+ monkeypatch.setattr(model_catalog, "get_model_catalog_service", lambda: service)
+ monkeypatch.setattr(provider_runtime, "get_model_catalog_service", lambda: service)
+ monkeypatch.setattr(
+ image_description, "get_model_catalog_service", lambda: service, raising=False
+ )
+
+
+def _offered(service: ModelCatalogService) -> list[bool]:
+ return [option["supports_vision"] for option in list_llm_options(service.load())["options"]]
+
+
+def test_gate_accepts_every_model_the_picker_offers(tmp_path: Path, monkeypatch) -> None:
+ service = _service(tmp_path, [{"id": "m", "model": "llama3.2-vision"}])
+ _patch_catalog(monkeypatch, service)
+
+ assert _offered(service) == [True]
+ assert resolve_image_description_config(SELECTION).model == "llama3.2-vision"
+
+
+def test_gate_refuses_a_model_the_tables_do_not_know(tmp_path: Path, monkeypatch) -> None:
+ service = _service(tmp_path, [{"id": "m", "model": "llama3.2"}], binding="custom")
+ _patch_catalog(monkeypatch, service)
+
+ assert _offered(service) == [False]
+ with pytest.raises(ValueError, match="must support image input"):
+ resolve_image_description_config(SELECTION)
+
+
+def test_declared_capability_beats_the_provider_table(tmp_path: Path, monkeypatch) -> None:
+ service = _service(
+ tmp_path, [{"id": "m", "model": "gpt-4o", "capabilities": {"vision": False}}]
+ )
+ _patch_catalog(monkeypatch, service)
+
+ assert _offered(service) == [False]
+ with pytest.raises(ValueError, match="must support image input"):
+ resolve_image_description_config(SELECTION)