Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(parsing): select an independent image description model

ZyPulsecbb742a141ac
diff --git a/deeptutor/api/routers/settings.py b/deeptutor/api/routers/settings.py
index 41fd591c5..af5697cb6 100644
--- a/deeptutor/api/routers/settings.py
+++ b/deeptutor/api/routers/settings.py
@@ -38,6 +38,7 @@ from deeptutor.services.config import (
redact_catalog_secrets,
restore_catalog_secrets,
)
+from deeptutor.services.config.image_description import ImageDescriptionModelSelection
from deeptutor.services.config.origins import normalize_origins
from deeptutor.services.config.runtime_settings import (
CHAT_ATTACHMENT_CHARS_RANGE,
@@ -428,6 +429,8 @@ class DocumentParsingUpdate(BaseModel):
engines: Optional[dict[str, dict]] = None
# Toggle for vision-model captions of embedded images (None = keep stored).
image_caption: Optional[bool] = None
+ # Omit to keep the selection; null restores the main LLM.
+ image_description_model: Optional[ImageDescriptionModelSelection] = None
class DocumentParsingTest(BaseModel):
@@ -1192,6 +1195,7 @@ def _document_parsing_payload() -> dict[str, Any]:
return {
"engine": full.get("engine"),
"image_caption": bool(full.get("image_caption", False)),
+ "image_description_model": full.get("image_description_model"),
"engines": redacted,
"available_engines": available,
"readiness": readiness,
@@ -1360,6 +1364,20 @@ async def update_document_parsing_settings(payload: DocumentParsingUpdate):
engines[name].update(merged)
new_engine = payload.engine or full.get("engine")
+ image_model = full.get("image_description_model")
+ if "image_description_model" in payload.model_fields_set:
+ image_model = (
+ payload.image_description_model.model_dump()
+ if payload.image_description_model is not None
+ else None
+ )
+ if image_model is not None:
+ from deeptutor.services.llm.image_description import resolve_image_description_config
+
+ try:
+ resolve_image_description_config(image_model)
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
image_caption = (
payload.image_caption
if payload.image_caption is not None
@@ -1369,6 +1387,7 @@ async def update_document_parsing_settings(payload: DocumentParsingUpdate):
{
"engine": new_engine,
"image_caption": image_caption,
+ "image_description_model": image_model,
"engines": engines,
}
)
diff --git a/deeptutor/reading/captions.py b/deeptutor/reading/captions.py
index f24dcdb84..17f18053b 100644
--- a/deeptutor/reading/captions.py
+++ b/deeptutor/reading/captions.py
@@ -22,7 +22,7 @@ from typing import Any
from deeptutor.reading.store import ReadingStore
from deeptutor.services.config.runtime_settings import load_document_parsing_settings
-from deeptutor.services.llm.client import get_llm_client
+from deeptutor.services.llm.image_description import get_image_description_client
from deeptutor.services.rag.pipelines.llamaindex.config import image_description_limits
logger = logging.getLogger(__name__)
@@ -72,7 +72,7 @@ async def caption_material_media(
return 0
try:
- client = get_llm_client()
+ client = get_image_description_client()
except Exception as exc: # noqa: BLE001 - the client may not be configured
logger.warning("Image captioning skipped: LLM client is unavailable (%s)", exc)
return 0
diff --git a/deeptutor/services/config/image_description.py b/deeptutor/services/config/image_description.py
new file mode 100644
index 000000000..6d26bef8f
--- /dev/null
+++ b/deeptutor/services/config/image_description.py
@@ -0,0 +1,18 @@
+"""Credential-free selection for document image descriptions."""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ImageDescriptionModelSelection(BaseModel):
+ """Reference a configured vision model without changing the chat default."""
+
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+
+ profile_id: str = Field(min_length=1)
+ model_id: str = Field(min_length=1)
+
+
+def normalize_image_description_model(value: object) -> dict[str, str] | None:
+ if value is None or value == {}:
+ return None
+ return ImageDescriptionModelSelection.model_validate(value).model_dump()
diff --git a/deeptutor/services/config/runtime_settings.py b/deeptutor/services/config/runtime_settings.py
index 03313f63e..d31e9e4d6 100644
--- a/deeptutor/services/config/runtime_settings.py
+++ b/deeptutor/services/config/runtime_settings.py
@@ -10,6 +10,7 @@ from typing import Any, Callable
from deeptutor.services.file_io import atomic_write_json as _atomic_write_json
from deeptutor.services.path_service import get_path_service
+from .image_description import normalize_image_description_model
from .origins import normalize_origins
DEFAULT_SYSTEM_SETTINGS: dict[str, Any] = {
@@ -247,6 +248,7 @@ DEFAULT_DOCUMENT_PARSING_SETTINGS: dict[str, Any] = {
# Caption embedded figures with a vision model at ingest, so a text-only
# model reading the material can still describe its images.
"image_caption": False,
+ "image_description_model": None,
"engines": {
DOCUMENT_PARSING_ENGINE_TEXT_ONLY: _DEFAULT_TEXT_ONLY_ENGINE,
DOCUMENT_PARSING_ENGINE_MINERU: _DEFAULT_MINERU_ENGINE,
@@ -1127,6 +1129,9 @@ class RuntimeSettingsService:
"version": 2,
"engine": engine,
"image_caption": _coerce_bool(settings.get("image_caption"), False),
+ "image_description_model": normalize_image_description_model(
+ settings.get("image_description_model")
+ ),
"engines": engines_out,
}
diff --git a/deeptutor/services/llm/image_description.py b/deeptutor/services/llm/image_description.py
new file mode 100644
index 000000000..31472f876
--- /dev/null
+++ b/deeptutor/services/llm/image_description.py
@@ -0,0 +1,46 @@
+"""Model selection shared by the existing document image-description passes."""
+
+from typing import Any
+
+from deeptutor.services.config.runtime_settings import load_document_parsing_settings
+from deeptutor.services.model_selection.llm import LLMSelection
+from deeptutor.services.model_selection.runtime import resolve_llm_config_for_selection
+
+from .capabilities import supports_vision
+from .client import LLMClient, get_llm_client
+from .config import LLMConfig
+from .factory import complete_with_config
+
+
+def resolve_image_description_config(selection: dict[str, str]) -> LLMConfig:
+ """Validate an explicit selection without installing a global override."""
+ config = resolve_llm_config_for_selection(LLMSelection.from_payload(selection))
+ if not supports_vision(config.binding, config.model):
+ raise ValueError("The image description model must support image input.")
+ return config
+
+
+class _ImageDescriptionClient(LLMClient):
+ async def complete(
+ self,
+ prompt: str,
+ system_prompt: str | None = None,
+ history: list[dict[str, str]] | None = None,
+ **kwargs: Any,
+ ) -> str:
+ # Preserve the full resolved profile (including wire API and endpoint)
+ # rather than rebuilding an override from a subset of its fields.
+ return await complete_with_config(
+ config=self.config,
+ prompt=prompt,
+ system_prompt=system_prompt or "You are a helpful assistant.",
+ messages=history or None,
+ **kwargs,
+ )
+
+
+def get_image_description_client() -> LLMClient:
+ selection = load_document_parsing_settings().get("image_description_model")
+ if not selection:
+ return get_llm_client()
+ return _ImageDescriptionClient(resolve_image_description_config(selection), configure_env=False)
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index 9a8cef74e..11db7fc04 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -24,7 +24,7 @@ from llama_index.core.schema import ImageNode
from deeptutor.services.config.runtime_settings import DOCUMENT_PARSING_ENGINE_LITEPARSE
from deeptutor.services.embedding import get_embedding_client
-from deeptutor.services.llm.client import get_llm_client
+from deeptutor.services.llm.image_description import get_image_description_client
from deeptutor.services.rag.file_routing import FileTypeRouter
from deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assets
from deeptutor.utils.document_validator import DocumentValidator
@@ -328,7 +328,7 @@ class LlamaIndexDocumentLoader:
# keeps text-only embedding setups independent of LLM configuration and
# reuses one client for the whole image batch.
try:
- llm_client = get_llm_client()
+ llm_client = get_image_description_client()
except Exception as exc:
self._log_skipped_images(sources, f"LLM client is unavailable ({exc})")
return []
diff --git a/tests/api/test_image_description_settings.py b/tests/api/test_image_description_settings.py
new file mode 100644
index 000000000..4d320d29c
--- /dev/null
+++ b/tests/api/test_image_description_settings.py
@@ -0,0 +1,73 @@
+"""Persist only model IDs and preserve/clear explicit caption selections."""
+
+from fastapi import HTTPException
+from pydantic import ValidationError
+import pytest
+
+from deeptutor.api.routers import settings as router
+from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+from deeptutor.services.llm import image_description
+
+SELECTION = {"profile_id": "pictures", "model_id": "vision"}
+
+
+def service(monkeypatch, tmp_path):
+ service = RuntimeSettingsService(tmp_path / "settings", process_env={})
+ monkeypatch.setattr(router, "_require_settings_admin", lambda: None)
+ monkeypatch.setattr(router, "get_runtime_settings_service", lambda: service)
+ monkeypatch.setattr(router, "_document_parsing_payload", service.load_document_parsing)
+ monkeypatch.setattr(image_description, "resolve_image_description_config", lambda _: None)
+ return service
+
+
+async def test_selection_roundtrip_omission_and_explicit_reset(service):
+ initial = service.load_document_parsing()
+ assert initial["image_description_model"] is None
+ selected = await router.update_document_parsing_settings(
+ router.DocumentParsingUpdate(image_description_model=SELECTION)
+ )
+ assert selected["image_description_model"] == SELECTION
+ assert selected["engines"] == initial["engines"]
+ assert selected["image_caption"] == initial["image_caption"]
+ preserved = await router.update_document_parsing_settings(
+ router.DocumentParsingUpdate(engine="mineru")
+ )
+ assert preserved["image_description_model"] == SELECTION
+ service.save_mineru({"language": "en"})
+ assert service.load_document_parsing()["image_description_model"] == SELECTION
+ cleared = await router.update_document_parsing_settings(
+ router.DocumentParsingUpdate(image_description_model=None)
+ )
+ assert cleared["image_description_model"] is None
+ assert service.load_document_parsing()["engine"] == "mineru"
+
+
+async def test_invalid_or_text_only_selection_does_not_save(service, monkeypatch):
+ before = service.load_document_parsing()
+
+ def invalid(_):
+ raise ValueError("The image description model must support image input.")
+
+ monkeypatch.setattr(image_description, "resolve_image_description_config", invalid)
+ with pytest.raises(HTTPException) as exc:
+ await router.update_document_parsing_settings(
+ router.DocumentParsingUpdate(engine="mineru", image_description_model=SELECTION)
+ )
+ assert exc.value.status_code == 400
+ assert service.load_document_parsing() == before
+
+
+ "selection",
+ [
+ {"profile_id": "pictures"},
+ {"profile_id": " ", "model_id": "vision"},
+ {**SELECTION, "api_key": "must-not-be-accepted"},
+ ],
+)
+def test_model_selection_rejects_incomplete_ids_and_credentials(selection):
+ with pytest.raises(ValidationError):
+ router.DocumentParsingUpdate(image_description_model=selection)
diff --git a/tests/reading/test_reading_captions.py b/tests/reading/test_reading_captions.py
index c37756456..f8dadf7b8 100644
--- a/tests/reading/test_reading_captions.py
+++ b/tests/reading/test_reading_captions.py
@@ -74,7 +74,7 @@ async def test_caption_material_media_writes_back(
store: ReadingStore, monkeypatch: pytest.MonkeyPatch
) -> None:
stub = _StubClient()
- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)
+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)
written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)
@@ -89,7 +89,7 @@ async def test_existing_captions_are_skipped(
store: ReadingStore, monkeypatch: pytest.MonkeyPatch
) -> None:
stub = _StubClient()
- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)
+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)
assert await captions_module.caption_material_media(_MATERIAL_ID, store=store) == 3
stub.calls.clear()
@@ -105,7 +105,7 @@ async def test_single_failure_keeps_the_rest(
store: ReadingStore, monkeypatch: pytest.MonkeyPatch
) -> None:
stub = _StubClient(fail={"image-01.png"})
- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)
+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)
written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)
@@ -121,7 +121,7 @@ async def test_text_only_client_makes_no_calls(
store: ReadingStore, monkeypatch: pytest.MonkeyPatch
) -> None:
stub = _StubClient(vision=False)
- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)
+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)
written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)
diff --git a/tests/services/llm/test_image_description_selection.py b/tests/services/llm/test_image_description_selection.py
new file mode 100644
index 000000000..ff72b7c12
--- /dev/null
+++ b/tests/services/llm/test_image_description_selection.py
@@ -0,0 +1,151 @@
+"""Independent image models preserve the main client and full provider config."""
+
+from copy import deepcopy
+import os
+
+import pytest
+
+from deeptutor.services.llm import image_description as mod
+from deeptutor.services.llm.config import LLMConfig
+from deeptutor.services.model_selection.llm import LLMSelection
+
+
+def selected(monkeypatch):
+ config = LLMConfig(
+ model="vision-model",
+ binding="openai",
+ api_key="synthetic-key",
+ base_url="https://example.invalid/v1",
+ effective_url="https://example.invalid/v1/responses",
+ wire_api="responses",
+ reasoning_effort="low",
+ )
+ monkeypatch.setattr(
+ mod,
+ "load_document_parsing_settings",
+ lambda: {"image_description_model": {"profile_id": "pictures", "model_id": "vision"}},
+ )
+
+ def resolve(selection):
+ assert selection == LLMSelection(profile_id="pictures", model_id="vision")
+ return config
+
+ monkeypatch.setattr(mod, "resolve_llm_config_for_selection", resolve)
+ monkeypatch.setattr(mod, "supports_vision", lambda *_: True)
+ return config
+
+
[email protected]("value", [None, {}])
+def test_unset_selection_reuses_main_client(monkeypatch, value):
+ main = object()
+ monkeypatch.setattr(
+ mod, "load_document_parsing_settings", lambda: {"image_description_model": value}
+ )
+ monkeypatch.setattr(mod, "get_llm_client", lambda: main)
+ assert mod.get_image_description_client() is main
+
+
+async def test_explicit_model_preserves_full_config_and_process_state(monkeypatch, selected):
+ monkeypatch.setenv("OPENAI_API_KEY", "main-synthetic-key")
+ monkeypatch.setenv("OPENAI_BASE_URL", "https://chat.example.invalid")
+ original = deepcopy(selected)
+ before = dict(os.environ)
+ calls = []
+
+ async def complete(**kwargs):
+ calls.append(kwargs)
+ return "a labelled diagram"
+
+ monkeypatch.setattr(mod, "complete_with_config", complete)
+ monkeypatch.setattr(
+ mod, "get_llm_client", lambda: pytest.fail("must not replace the main client")
+ )
+ client = mod.get_image_description_client()
+ assert (
+ await client.complete("Describe", image_data="synthetic-image", image_mime_type="image/png")
+ == "a labelled diagram"
+ )
+ assert calls[0]["config"] is selected
+ assert calls[0]["image_data"] == "synthetic-image"
+ assert selected == original
+ assert dict(os.environ) == before
+
+
+def test_explicit_text_model_is_rejected(monkeypatch, selected):
+ monkeypatch.setattr(mod, "supports_vision", lambda *_: False)
+ with pytest.raises(ValueError, match="must support image input"):
+ mod.get_image_description_client()
+
+
+def test_deleted_selection_does_not_silently_charge_main_model(monkeypatch, selected):
+ def missing(_):
+ raise ValueError("selected profile/model was not found")
+
+ monkeypatch.setattr(mod, "resolve_llm_config_for_selection", missing)
+ monkeypatch.setattr(mod, "get_llm_client", lambda: pytest.fail("unexpected fallback"))
+ with pytest.raises(ValueError, match="not found"):
+ mod.get_image_description_client()
+
+
+async def test_reading_captions_use_selected_model(monkeypatch, tmp_path, selected):
+ from deeptutor.reading import captions
+
+ image = tmp_path / "figure.png"
+ image.write_bytes(b"synthetic image")
+ saved = {}
+
+ class Store:
+ def media_items(self, _):
+ return [{"name": "figure.png", "mime": "image/png"}]
+
+ def media_path(self, *_):
+ return image
+
+ def update_media_captions(self, _, result):
+ saved.update(result)
+ return len(result)
+
+ async def complete(**kwargs):
+ assert kwargs["config"] is selected
+ assert kwargs["image_filename"] == "figure.png"
+ return "selected vision caption"
+
+ monkeypatch.setattr(mod, "complete_with_config", complete)
+ monkeypatch.setattr(mod.LLMClient, "supports_multimodal_images", lambda _: True)
+ assert await captions.caption_material_media("material", store=Store()) == 1
+ assert saved == {"figure.png": "selected vision caption"}
+
+
+async def test_llamaindex_image_descriptions_use_selected_model(monkeypatch, tmp_path, selected):
+ from types import SimpleNamespace
+
+ from deeptutor.services.rag.pipelines.llamaindex import document_loader
+
+ image = tmp_path / "figure.png"
+ image.write_bytes(b"synthetic image")
+
+ class Embedding:
+ config = SimpleNamespace(binding="test", model="test")
+
+ def supports_multimodal_contents(self):
+ return True
+
+ async def embed_contents(self, contents):
+ return [[0.1, 0.2] for _ in contents]
+
+ async def complete(**kwargs):
+ assert kwargs["config"] is selected
+ return "selected vision caption"
+
+ monkeypatch.setattr(mod, "complete_with_config", complete)
+ monkeypatch.setattr(mod.LLMClient, "supports_multimodal_images", lambda _: True)
+ monkeypatch.setattr(document_loader, "get_embedding_client", Embedding)
+ nodes = await document_loader.LlamaIndexDocumentLoader()._load_image_nodes(
+ [document_loader._ImageSource(path=image, origin=image)]
+ )
+ assert len(nodes) == 1
+ assert "selected vision caption" in nodes[0].text
diff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.py
index a48ed2e75..6c2908d77 100644
--- a/tests/services/rag/test_llamaindex_document_loader.py
+++ b/tests/services/rag/test_llamaindex_document_loader.py
@@ -416,7 +416,7 @@ def test_loader_indexes_images_extracted_from_parsed_document(
return "Figure showing a bar chart."
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())
- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _VisionClient())
+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())
documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(pdf_path)]))
@@ -456,7 +456,7 @@ def test_loader_skips_images_when_embedding_provider_is_text_only(
def _unexpected_llm_client():
pytest.fail("text-only embedding must not initialize the LLM client")
- monkeypatch.setattr(loader_module, "get_llm_client", _unexpected_llm_client)
+ monkeypatch.setattr(loader_module, "get_image_description_client", _unexpected_llm_client)
documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))
@@ -504,7 +504,7 @@ def test_loader_embeds_images_with_qwen38_max_vision_capability(
monkeypatch.setattr(vision_client, "complete", _complete)
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalClient())
- monkeypatch.setattr(loader_module, "get_llm_client", lambda: vision_client)
+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: vision_client)
documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))
@@ -540,7 +540,7 @@ def test_loader_skips_images_when_llm_is_text_only(
return False
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())
- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _TextOnlyLLMClient())
+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _TextOnlyLLMClient())
documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))
@@ -566,7 +566,7 @@ def test_loader_skips_images_when_llm_client_is_unavailable(
raise RuntimeError("no LLM configured")
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())
- monkeypatch.setattr(loader_module, "get_llm_client", _unavailable_llm_client)
+ monkeypatch.setattr(loader_module, "get_image_description_client", _unavailable_llm_client)
with caplog.at_level("WARNING"):
documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))
diff --git a/tests/services/rag/test_llamaindex_image_description.py b/tests/services/rag/test_llamaindex_image_description.py
index 9b99100fc..4a3a73d60 100644
--- a/tests/services/rag/test_llamaindex_image_description.py
+++ b/tests/services/rag/test_llamaindex_image_description.py
@@ -56,7 +56,7 @@ def _install_multimodal_clients(
return await complete_fn(prompt, **kwargs)
monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _EmbeddingClient())
- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _VisionClient())
+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())
@pytest.mark.asyncio
diff --git a/web/contracts/generated/api.ts b/web/contracts/generated/api.ts
index 6677f11f6..d2ca0ac0e 100644
--- a/web/contracts/generated/api.ts
+++ b/web/contracts/generated/api.ts
@@ -11802,6 +11802,8 @@ export interface components {
} | null;
/** Image Caption */
readonly image_caption?: boolean | null;
+ readonly image_description_model?:
+ components["schemas"]["ImageDescriptionModelSelection"] | null;
};
/** DocumentResponse */
readonly DocumentResponse: {
@@ -12120,6 +12122,16 @@ export interface components {
/** Client Id */
readonly client_id?: string | null;
};
+ /**
+ * ImageDescriptionModelSelection
+ * @description Reference a configured vision model without changing the chat default.
+ */
+ readonly ImageDescriptionModelSelection: {
+ /** Model Id */
+ readonly model_id: string;
+ /** Profile Id */
+ readonly profile_id: string;
+ };
/** ImaKnowledgeBaseSummary */
readonly ImaKnowledgeBaseSummary: {
/** Description */
@@ -16312,6 +16324,8 @@ export type SchemaGuardianRestrictionsPayload =
export type SchemaHttpValidationError =
components["schemas"]["HTTPValidationError"];
export type SchemaImaConfigUpdate = components["schemas"]["ImaConfigUpdate"];
+export type SchemaImageDescriptionModelSelection =
+ components["schemas"]["ImageDescriptionModelSelection"];
export type SchemaImaKnowledgeBaseSummary =
components["schemas"]["ImaKnowledgeBaseSummary"];
export type SchemaImportedMessage = components["schemas"]["ImportedMessage"];
diff --git a/web/contracts/schema/openapi.json b/web/contracts/schema/openapi.json
index 106b6ace9..e4ec42e48 100644
--- a/web/contracts/schema/openapi.json
+++ b/web/contracts/schema/openapi.json
@@ -3174,6 +3174,16 @@
}
],
"title": "Image Caption"
+ },
+ "image_description_model": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/ImageDescriptionModelSelection"
+ },
+ {
+ "type": "null"
+ }
+ ]
}
},
"title": "DocumentParsingUpdate",
@@ -3987,6 +3997,28 @@
"title": "ImaKnowledgeBaseSummary",
"type": "object"
},
+ "ImageDescriptionModelSelection": {
+ "additionalProperties": false,
+ "description": "Reference a configured vision model without changing the chat default.",
+ "properties": {
+ "model_id": {
+ "minLength": 1,
+ "title": "Model Id",
+ "type": "string"
+ },
+ "profile_id": {
+ "minLength": 1,
+ "title": "Profile Id",
+ "type": "string"
+ }
+ },
+ "required": [
+ "profile_id",
+ "model_id"
+ ],
+ "title": "ImageDescriptionModelSelection",
+ "type": "object"
+ },
"ImportFromBookRequest": {
"properties": {
"chapters": {
diff --git a/web/features/settings/sections/DocumentParsingSettingsSection.tsx b/web/features/settings/sections/DocumentParsingSettingsSection.tsx
index b050c0101..9e421462b 100644
--- a/web/features/settings/sections/DocumentParsingSettingsSection.tsx
+++ b/web/features/settings/sections/DocumentParsingSettingsSection.tsx
@@ -17,6 +17,7 @@ import {
import { MinerUEngineSettings } from "@/components/settings/MinerUEngineSettings";
import { Toggle } from "@/components/settings/Toggle";
import { apiFetch, apiUrl } from "@/lib/api";
+import { ImageDescriptionModelSetting } from "./ImageDescriptionModelSetting";
type EngineMeta = {
id: string;
@@ -30,6 +31,7 @@ type Readiness = { ready: boolean; reason: string; message: string };
type DocumentParsingPayload = {
engine: string;
+ image_description_model?: { profile_id: string; model_id: string } | null;
engines: Record<string, Record<string, unknown>>;
available_engines: EngineMeta[];
readiness: Record<string, Readiness>;
@@ -138,6 +140,11 @@ export default function DocumentParsingSettingsPage() {
{!loading && data && (
<>
+ <ImageDescriptionModelSetting
+ value={data.image_description_model ?? null}
+ disabled={busy}
+ onChange={(selection) => putDocumentParsing({ image_description_model: selection })}
+ />
<section className="mb-10">
<header className="mb-3">
<h2 className="text-[15px] font-semibold tracking-tight text-[var(--foreground)]">
diff --git a/web/features/settings/sections/ImageDescriptionModelSetting.tsx b/web/features/settings/sections/ImageDescriptionModelSetting.tsx
new file mode 100644
index 000000000..a494b8c52
--- /dev/null
+++ b/web/features/settings/sections/ImageDescriptionModelSetting.tsx
@@ -0,0 +1,83 @@
+'use client'
+
+import { useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { nativeSelectClass, selectOptionClass } from '@/components/settings/shared'
+import { listLLMOptions, type LLMOption } from '@/lib/llm-options'
+
+type Selection = { profile_id: string; model_id: string }
+
+export function ImageDescriptionModelSetting({
+ value,
+ disabled,
+ onChange,
+}: {
+ value: Selection | null
+ disabled: boolean
+ onChange: (selection: Selection | null) => void
+}) {
+ const { t } = useTranslation()
+ const [models, setModels] = useState<LLMOption[]>([])
+ const [failed, setFailed] = useState(false)
+
+ useEffect(() => {
+ let active = true
+ listLLMOptions().then(
+ result => {
+ if (active) setModels(result.options.filter(model => model.supports_vision))
+ },
+ () => {
+ if (active) setFailed(true)
+ }
+ )
+ return () => {
+ active = false
+ }
+ }, [])
+
+ const key = (selection: Selection) => JSON.stringify([selection.profile_id, selection.model_id])
+ const selected = value ? key(value) : ''
+ const missing = value && !models.some(model => key(model) === selected)
+
+ return (
+ <section className="mb-10">
+ <label htmlFor="image-description-model" className="mb-2 block text-[13px] font-medium">
+ {t('Image description model')}
+ </label>
+ <select
+ id="image-description-model"
+ className={nativeSelectClass}
+ disabled={disabled}
+ value={selected}
+ onChange={event => {
+ const model = models.find(item => key(item) === event.target.value)
+ onChange(model ? { profile_id: model.profile_id, model_id: model.model_id } : null)
+ }}
+ >
+ <option value="" className={selectOptionClass}>
+ {t('Use main LLM (fallback)')}
+ </option>
+ {missing && (
+ <option value={selected} disabled>
+ {t('Selected model unavailable')}
+ </option>
+ )}
+ {models.map(model => (
+ <option key={key(model)} value={key(model)} className={selectOptionClass}>
+ {model.profile_name} / {model.model_name}
+ </option>
+ ))}
+ </select>
+ <p className="mt-2 text-xs text-[var(--muted-foreground)]">
+ {t(
+ 'Select a vision model for image descriptions in documents.'
+ )}
+ </p>
+ {failed && (
+ <p role="alert" className="mt-2 text-xs text-red-600">
+ {t('Failed to load image description models.')}
+ </p>
+ )}
+ </section>
+ )
+}
diff --git a/web/lib/settings-extensions.ts b/web/lib/settings-extensions.ts
index 97dad24d3..484402965 100644
--- a/web/lib/settings-extensions.ts
+++ b/web/lib/settings-extensions.ts
@@ -61,9 +61,13 @@ export async function applyExtensionPayload(
if (!endpoint) throw new Error(`Unknown settings section: ${key}`);
let body = guardian?.[1] === "materials" ? { book_ids: payload } : payload;
if (key === "document-parsing") {
- const { engine, engines } = payload as { engine: string; engines: Record<string, unknown> };
+ const { engine, engines, image_description_model } = payload as {
+ engine: string;
+ engines: Record<string, unknown>;
+ image_description_model?: { profile_id: string; model_id: string } | null;
+ };
// MinerU has its own draft; never overwrite it with this page's snapshot.
- body = { engine, engines: Object.fromEntries(Object.entries(engines).filter(([name]) => name !== "mineru")) };
+ body = { engine, image_description_model, engines: Object.fromEntries(Object.entries(engines).filter(([name]) => name !== "mineru")) };
}
const response = await apiFetch(apiUrl(endpoint), {
method: "PUT",
diff --git a/web/locales/de/app.json b/web/locales/de/app.json
index 80071a718..e06557fe9 100644
--- a/web/locales/de/app.json
+++ b/web/locales/de/app.json
@@ -5270,5 +5270,9 @@
"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "Diese Knowledge Base samt Dokumenten, Einstellungen und Indizes verschieben. Bestehende Workspace-Zuweisungen werden aktualisiert.",
"Review move": "Verschiebung prüfen",
"{{count}} files · {{bytes}} bytes": "{{count}} Dateien · {{bytes}} Bytes",
- "Assignments updated: {{names}}": "Aktualisierte Zuweisungen: {{names}}"
+ "Assignments updated: {{names}}": "Aktualisierte Zuweisungen: {{names}}",
+ "Image description model": "Modell für Bildbeschreibungen",
+ "Use main LLM (fallback)": "Hauptmodell verwenden (Standard)",
+ "Select a vision model for image descriptions in documents.": "Wählen Sie ein Vision-Modell für Bildbeschreibungen in Dokumenten.",
+ "Failed to load image description models.": "Die Modelle für Bildbeschreibungen konnten nicht geladen werden."
}
diff --git a/web/locales/en/app.json b/web/locales/en/app.json
index c6167eb21..5995f9f4e 100644
--- a/web/locales/en/app.json
+++ b/web/locales/en/app.json
@@ -5270,5 +5270,9 @@
"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.",
"Review move": "Review move",
"{{count}} files · {{bytes}} bytes": "{{count}} files · {{bytes}} bytes",
- "Assignments updated: {{names}}": "Assignments updated: {{names}}"
+ "Assignments updated: {{names}}": "Assignments updated: {{names}}",
+ "Image description model": "Image description model",
+ "Use main LLM (fallback)": "Use main LLM (fallback)",
+ "Failed to load image description models.": "Failed to load image description models.",
+ "Select a vision model for image descriptions in documents.": "Select a vision model for image descriptions in documents."
}
diff --git a/web/locales/zh/app.json b/web/locales/zh/app.json
index 4312257b3..ae0de240c 100644
--- a/web/locales/zh/app.json
+++ b/web/locales/zh/app.json
@@ -5270,5 +5270,9 @@
"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "连同文档、设置和索引一起移动此知识库。现有工作区分配会同步更新。",
"Review move": "检查移动详情",
"{{count}} files · {{bytes}} bytes": "{{count}} 个文件 · {{bytes}} 字节",
- "Assignments updated: {{names}}": "将更新以下工作区的分配:{{names}}"
+ "Assignments updated: {{names}}": "将更新以下工作区的分配:{{names}}",
+ "Image description model": "图片描述模型",
+ "Use main LLM (fallback)": "使用主模型(默认)",
+ "Failed to load image description models.": "无法加载图片描述模型。",
+ "Select a vision model for image descriptions in documents.": "选择用于文档图片描述的视觉模型。"
}
diff --git a/web/tests/image-description-model.spec.tsx b/web/tests/image-description-model.spec.tsx
new file mode 100644
index 000000000..9bc4d1cce
--- /dev/null
+++ b/web/tests/image-description-model.spec.tsx
@@ -0,0 +1,103 @@
+import React, { useState } from 'react'
+import { fireEvent, render, screen, waitFor } from '@testing-library/react'
+import { beforeEach, expect, it, vi } from 'vitest'
+import { ImageDescriptionModelSetting } from '@/features/settings/sections/ImageDescriptionModelSetting'
+import { applyExtensionPayload } from '@/lib/settings-extensions'
+
+const mocks = vi.hoisted(() => ({ options: vi.fn(), fetch: vi.fn() }))
+vi.mock('@/lib/llm-options', () => ({ listLLMOptions: () => mocks.options() }))
+vi.mock('@/lib/api', () => ({
+ apiUrl: (value: string) => value,
+ apiFetch: (...args: unknown[]) => mocks.fetch(...args),
+}))
+vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }))
+
+beforeEach(() => {
+ mocks.options.mockResolvedValue({
+ options: [
+ {
+ profile_id: 'vision',
+ model_id: 'image',
+ profile_name: 'Pictures',
+ model_name: 'Vision',
+ supports_vision: true,
+ },
+ {
+ profile_id: 'chat',
+ model_id: 'text',
+ profile_name: 'Chat',
+ model_name: 'Text',
+ supports_vision: false,
+ },
+ ],
+ })
+ mocks.fetch.mockResolvedValue({ ok: true })
+})
+
+function Harness() {
+ const [value, setValue] = useState<{ profile_id: string; model_id: string } | null>(null)
+ return (
+ <>
+ <ImageDescriptionModelSetting value={value} disabled={false} onChange={setValue} />
+ <button
+ onClick={() =>
+ applyExtensionPayload('document-parsing', {
+ engine: 'text_only',
+ engines: { mineru: { api_token: 'must-not-overwrite' }, text_only: {} },
+ image_description_model: value,
+ })
+ }
+ >
+ Apply
+ </button>
+ </>
+ )
+}
+
+it('lists vision models and sends explicit IDs or null without overwriting the MinerU draft', async () => {
+ render(<Harness />)
+ await screen.findByRole('option', { name: 'Pictures / Vision' })
+ expect(screen.queryByRole('option', { name: 'Chat / Text' })).toBeNull()
+ fireEvent.change(screen.getByLabelText('Image description model'), {
+ target: { value: '["vision","image"]' },
+ })
+ fireEvent.click(screen.getByText('Apply'))
+ await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(1))
+ expect(JSON.parse(mocks.fetch.mock.calls[0][1].body)).toEqual({
+ engine: 'text_only',
+ engines: { text_only: {} },
+ image_description_model: { profile_id: 'vision', model_id: 'image' },
+ })
+ fireEvent.change(screen.getByLabelText('Image description model'), { target: { value: '' } })
+ fireEvent.click(screen.getByText('Apply'))
+ await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2))
+ expect(JSON.parse(mocks.fetch.mock.calls[1][1].body).image_description_model).toBeNull()
+})
+
+it('keeps an unavailable saved selection visible instead of silently choosing the main model', async () => {
+ render(
+ <ImageDescriptionModelSetting
+ value={{ profile_id: 'deleted', model_id: 'gone' }}
+ disabled={false}
+ onChange={vi.fn()}
+ />
+ )
+ await screen.findByRole('option', { name: 'Pictures / Vision' })
+ expect(screen.getByLabelText('Image description model')).toHaveValue('["deleted","gone"]')
+ expect(screen.getByRole('option', { name: 'Selected model unavailable' })).toBeDisabled()
+})
+
+it('reports a model-list error while retaining the ability to reset', async () => {
+ mocks.options.mockRejectedValue(new Error('unavailable'))
+ const onChange = vi.fn()
+ render(
+ <ImageDescriptionModelSetting
+ value={{ profile_id: 'saved', model_id: 'model' }}
+ disabled={false}
+ onChange={onChange}
+ />
+ )
+ await screen.findByRole('alert')
+ fireEvent.change(screen.getByLabelText('Image description model'), { target: { value: '' } })
+ expect(onChange).toHaveBeenCalledWith(null)
+})
diff --git a/web/tests/settings-unified-draft.spec.tsx b/web/tests/settings-unified-draft.spec.tsx
index 18a272783..7fcdc393e 100644
--- a/web/tests/settings-unified-draft.spec.tsx
+++ b/web/tests/settings-unified-draft.spec.tsx
@@ -40,7 +40,10 @@ vi.mock("@/context/AppShellContext", () => ({
codeBlockWrapLongLines: false,
}),
}));
-vi.mock("@/lib/llm-options", () => ({ invalidateLLMOptionsCache: vi.fn() }));
+vi.mock("@/lib/llm-options", () => ({
+ invalidateLLMOptionsCache: vi.fn(),
+ listLLMOptions: vi.fn(async () => ({ active: null, options: [] })),
+}));
let settings: ReturnType<typeof useSettings>;
let live: ReturnType<typeof defaultCatalog>;
let stored: any;