DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(parsing): select an independent image description model
ZyPulsecbb742a141ac
diff --git a/deeptutor/api/routers/settings.py b/deeptutor/api/routers/settings.pyindex 41fd591c5..af5697cb6 100644--- a/deeptutor/api/routers/settings.py+++ b/deeptutor/api/routers/settings.py@@ -38,6 +38,7 @@ from deeptutor.services.config import (redact_catalog_secrets,restore_catalog_secrets,)+from deeptutor.services.config.image_description import ImageDescriptionModelSelectionfrom deeptutor.services.config.origins import normalize_originsfrom deeptutor.services.config.runtime_settings import (CHAT_ATTACHMENT_CHARS_RANGE,@@ -428,6 +429,8 @@ class DocumentParsingUpdate(BaseModel):engines: Optional[dict[str, dict]] = None# Toggle for vision-model captions of embedded images (None = keep stored).image_caption: Optional[bool] = None+ # Omit to keep the selection; null restores the main LLM.+ image_description_model: Optional[ImageDescriptionModelSelection] = Noneclass DocumentParsingTest(BaseModel):@@ -1192,6 +1195,7 @@ def _document_parsing_payload() -> dict[str, Any]:return {"engine": full.get("engine"),"image_caption": bool(full.get("image_caption", False)),+ "image_description_model": full.get("image_description_model"),"engines": redacted,"available_engines": available,"readiness": readiness,@@ -1360,6 +1364,20 @@ async def update_document_parsing_settings(payload: DocumentParsingUpdate):engines[name].update(merged)new_engine = payload.engine or full.get("engine")+ image_model = full.get("image_description_model")+ if "image_description_model" in payload.model_fields_set:+ image_model = (+ payload.image_description_model.model_dump()+ if payload.image_description_model is not None+ else None+ )+ if image_model is not None:+ from deeptutor.services.llm.image_description import resolve_image_description_config++ try:+ resolve_image_description_config(image_model)+ except ValueError as exc:+ raise HTTPException(status_code=400, detail=str(exc)) from excimage_caption = (payload.image_captionif payload.image_caption is not None@@ -1369,6 +1387,7 @@ async def update_document_parsing_settings(payload: DocumentParsingUpdate):{"engine": new_engine,"image_caption": image_caption,+ "image_description_model": image_model,"engines": engines,})diff --git a/deeptutor/reading/captions.py b/deeptutor/reading/captions.pyindex f24dcdb84..17f18053b 100644--- a/deeptutor/reading/captions.py+++ b/deeptutor/reading/captions.py@@ -22,7 +22,7 @@ from typing import Anyfrom deeptutor.reading.store import ReadingStorefrom deeptutor.services.config.runtime_settings import load_document_parsing_settings-from deeptutor.services.llm.client import get_llm_client+from deeptutor.services.llm.image_description import get_image_description_clientfrom deeptutor.services.rag.pipelines.llamaindex.config import image_description_limitslogger = logging.getLogger(__name__)@@ -72,7 +72,7 @@ async def caption_material_media(return 0try:- client = get_llm_client()+ client = get_image_description_client()except Exception as exc: # noqa: BLE001 - the client may not be configuredlogger.warning("Image captioning skipped: LLM client is unavailable (%s)", exc)return 0diff --git a/deeptutor/services/config/image_description.py b/deeptutor/services/config/image_description.pynew file mode 100644index 000000000..6d26bef8f--- /dev/null+++ b/deeptutor/services/config/image_description.py@@ -0,0 +1,18 @@+"""Credential-free selection for document image descriptions."""++from pydantic import BaseModel, ConfigDict, Field+++class ImageDescriptionModelSelection(BaseModel):+ """Reference a configured vision model without changing the chat default."""++ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)++ profile_id: str = Field(min_length=1)+ model_id: str = Field(min_length=1)+++def normalize_image_description_model(value: object) -> dict[str, str] | None:+ if value is None or value == {}:+ return None+ return ImageDescriptionModelSelection.model_validate(value).model_dump()diff --git a/deeptutor/services/config/runtime_settings.py b/deeptutor/services/config/runtime_settings.pyindex 03313f63e..d31e9e4d6 100644--- a/deeptutor/services/config/runtime_settings.py+++ b/deeptutor/services/config/runtime_settings.py@@ -10,6 +10,7 @@ from typing import Any, Callablefrom deeptutor.services.file_io import atomic_write_json as _atomic_write_jsonfrom deeptutor.services.path_service import get_path_service+from .image_description import normalize_image_description_modelfrom .origins import normalize_originsDEFAULT_SYSTEM_SETTINGS: dict[str, Any] = {@@ -247,6 +248,7 @@ DEFAULT_DOCUMENT_PARSING_SETTINGS: dict[str, Any] = {# Caption embedded figures with a vision model at ingest, so a text-only# model reading the material can still describe its images."image_caption": False,+ "image_description_model": None,"engines": {DOCUMENT_PARSING_ENGINE_TEXT_ONLY: _DEFAULT_TEXT_ONLY_ENGINE,DOCUMENT_PARSING_ENGINE_MINERU: _DEFAULT_MINERU_ENGINE,@@ -1127,6 +1129,9 @@ class RuntimeSettingsService:"version": 2,"engine": engine,"image_caption": _coerce_bool(settings.get("image_caption"), False),+ "image_description_model": normalize_image_description_model(+ settings.get("image_description_model")+ ),"engines": engines_out,}diff --git a/deeptutor/services/llm/image_description.py b/deeptutor/services/llm/image_description.pynew file mode 100644index 000000000..31472f876--- /dev/null+++ b/deeptutor/services/llm/image_description.py@@ -0,0 +1,46 @@+"""Model selection shared by the existing document image-description passes."""++from typing import Any++from deeptutor.services.config.runtime_settings import load_document_parsing_settings+from deeptutor.services.model_selection.llm import LLMSelection+from deeptutor.services.model_selection.runtime import resolve_llm_config_for_selection++from .capabilities import supports_vision+from .client import LLMClient, get_llm_client+from .config import LLMConfig+from .factory import complete_with_config+++def resolve_image_description_config(selection: dict[str, str]) -> LLMConfig:+ """Validate an explicit selection without installing a global override."""+ config = resolve_llm_config_for_selection(LLMSelection.from_payload(selection))+ if not supports_vision(config.binding, config.model):+ raise ValueError("The image description model must support image input.")+ return config+++class _ImageDescriptionClient(LLMClient):+ async def complete(+ self,+ prompt: str,+ system_prompt: str | None = None,+ history: list[dict[str, str]] | None = None,+ **kwargs: Any,+ ) -> str:+ # Preserve the full resolved profile (including wire API and endpoint)+ # rather than rebuilding an override from a subset of its fields.+ return await complete_with_config(+ config=self.config,+ prompt=prompt,+ system_prompt=system_prompt or "You are a helpful assistant.",+ messages=history or None,+ **kwargs,+ )+++def get_image_description_client() -> LLMClient:+ selection = load_document_parsing_settings().get("image_description_model")+ if not selection:+ return get_llm_client()+ return _ImageDescriptionClient(resolve_image_description_config(selection), configure_env=False)diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.pyindex 9a8cef74e..11db7fc04 100644--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py@@ -24,7 +24,7 @@ from llama_index.core.schema import ImageNodefrom deeptutor.services.config.runtime_settings import DOCUMENT_PARSING_ENGINE_LITEPARSEfrom deeptutor.services.embedding import get_embedding_client-from deeptutor.services.llm.client import get_llm_client+from deeptutor.services.llm.image_description import get_image_description_clientfrom deeptutor.services.rag.file_routing import FileTypeRouterfrom deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assetsfrom deeptutor.utils.document_validator import DocumentValidator@@ -328,7 +328,7 @@ class LlamaIndexDocumentLoader:# keeps text-only embedding setups independent of LLM configuration and# reuses one client for the whole image batch.try:- llm_client = get_llm_client()+ llm_client = get_image_description_client()except Exception as exc:self._log_skipped_images(sources, f"LLM client is unavailable ({exc})")return []diff --git a/tests/api/test_image_description_settings.py b/tests/api/test_image_description_settings.pynew file mode 100644index 000000000..4d320d29c--- /dev/null+++ b/tests/api/test_image_description_settings.py@@ -0,0 +1,73 @@+"""Persist only model IDs and preserve/clear explicit caption selections."""++from fastapi import HTTPException+from pydantic import ValidationError+import pytest++from deeptutor.api.routers import settings as router+from deeptutor.services.config.runtime_settings import RuntimeSettingsService+from deeptutor.services.llm import image_description++SELECTION = {"profile_id": "pictures", "model_id": "vision"}+++def service(monkeypatch, tmp_path):+ service = RuntimeSettingsService(tmp_path / "settings", process_env={})+ monkeypatch.setattr(router, "_require_settings_admin", lambda: None)+ monkeypatch.setattr(router, "get_runtime_settings_service", lambda: service)+ monkeypatch.setattr(router, "_document_parsing_payload", service.load_document_parsing)+ monkeypatch.setattr(image_description, "resolve_image_description_config", lambda _: None)+ return service+++async def test_selection_roundtrip_omission_and_explicit_reset(service):+ initial = service.load_document_parsing()+ assert initial["image_description_model"] is None+ selected = await router.update_document_parsing_settings(+ router.DocumentParsingUpdate(image_description_model=SELECTION)+ )+ assert selected["image_description_model"] == SELECTION+ assert selected["engines"] == initial["engines"]+ assert selected["image_caption"] == initial["image_caption"]+ preserved = await router.update_document_parsing_settings(+ router.DocumentParsingUpdate(engine="mineru")+ )+ assert preserved["image_description_model"] == SELECTION+ service.save_mineru({"language": "en"})+ assert service.load_document_parsing()["image_description_model"] == SELECTION+ cleared = await router.update_document_parsing_settings(+ router.DocumentParsingUpdate(image_description_model=None)+ )+ assert cleared["image_description_model"] is None+ assert service.load_document_parsing()["engine"] == "mineru"+++async def test_invalid_or_text_only_selection_does_not_save(service, monkeypatch):+ before = service.load_document_parsing()++ def invalid(_):+ raise ValueError("The image description model must support image input.")++ monkeypatch.setattr(image_description, "resolve_image_description_config", invalid)+ with pytest.raises(HTTPException) as exc:+ await router.update_document_parsing_settings(+ router.DocumentParsingUpdate(engine="mineru", image_description_model=SELECTION)+ )+ assert exc.value.status_code == 400+ assert service.load_document_parsing() == before+++ "selection",+ [+ {"profile_id": "pictures"},+ {"profile_id": " ", "model_id": "vision"},+ {**SELECTION, "api_key": "must-not-be-accepted"},+ ],+)+def test_model_selection_rejects_incomplete_ids_and_credentials(selection):+ with pytest.raises(ValidationError):+ router.DocumentParsingUpdate(image_description_model=selection)diff --git a/tests/reading/test_reading_captions.py b/tests/reading/test_reading_captions.pyindex c37756456..f8dadf7b8 100644--- a/tests/reading/test_reading_captions.py+++ b/tests/reading/test_reading_captions.py@@ -74,7 +74,7 @@ async def test_caption_material_media_writes_back(store: ReadingStore, monkeypatch: pytest.MonkeyPatch) -> None:stub = _StubClient()- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)@@ -89,7 +89,7 @@ async def test_existing_captions_are_skipped(store: ReadingStore, monkeypatch: pytest.MonkeyPatch) -> None:stub = _StubClient()- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)assert await captions_module.caption_material_media(_MATERIAL_ID, store=store) == 3stub.calls.clear()@@ -105,7 +105,7 @@ async def test_single_failure_keeps_the_rest(store: ReadingStore, monkeypatch: pytest.MonkeyPatch) -> None:stub = _StubClient(fail={"image-01.png"})- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)@@ -121,7 +121,7 @@ async def test_text_only_client_makes_no_calls(store: ReadingStore, monkeypatch: pytest.MonkeyPatch) -> None:stub = _StubClient(vision=False)- monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)+ monkeypatch.setattr(captions_module, "get_image_description_client", lambda: stub)written = await captions_module.caption_material_media(_MATERIAL_ID, store=store)diff --git a/tests/services/llm/test_image_description_selection.py b/tests/services/llm/test_image_description_selection.pynew file mode 100644index 000000000..ff72b7c12--- /dev/null+++ b/tests/services/llm/test_image_description_selection.py@@ -0,0 +1,151 @@+"""Independent image models preserve the main client and full provider config."""++from copy import deepcopy+import os++import pytest++from deeptutor.services.llm import image_description as mod+from deeptutor.services.llm.config import LLMConfig+from deeptutor.services.model_selection.llm import LLMSelection+++def selected(monkeypatch):+ config = LLMConfig(+ model="vision-model",+ binding="openai",+ api_key="synthetic-key",+ base_url="https://example.invalid/v1",+ effective_url="https://example.invalid/v1/responses",+ wire_api="responses",+ reasoning_effort="low",+ )+ monkeypatch.setattr(+ mod,+ "load_document_parsing_settings",+ lambda: {"image_description_model": {"profile_id": "pictures", "model_id": "vision"}},+ )++ def resolve(selection):+ assert selection == LLMSelection(profile_id="pictures", model_id="vision")+ return config++ monkeypatch.setattr(mod, "resolve_llm_config_for_selection", resolve)+ monkeypatch.setattr(mod, "supports_vision", lambda *_: True)+ return config++[email protected]("value", [None, {}])+def test_unset_selection_reuses_main_client(monkeypatch, value):+ main = object()+ monkeypatch.setattr(+ mod, "load_document_parsing_settings", lambda: {"image_description_model": value}+ )+ monkeypatch.setattr(mod, "get_llm_client", lambda: main)+ assert mod.get_image_description_client() is main+++async def test_explicit_model_preserves_full_config_and_process_state(monkeypatch, selected):+ monkeypatch.setenv("OPENAI_API_KEY", "main-synthetic-key")+ monkeypatch.setenv("OPENAI_BASE_URL", "https://chat.example.invalid")+ original = deepcopy(selected)+ before = dict(os.environ)+ calls = []++ async def complete(**kwargs):+ calls.append(kwargs)+ return "a labelled diagram"++ monkeypatch.setattr(mod, "complete_with_config", complete)+ monkeypatch.setattr(+ mod, "get_llm_client", lambda: pytest.fail("must not replace the main client")+ )+ client = mod.get_image_description_client()+ assert (+ await client.complete("Describe", image_data="synthetic-image", image_mime_type="image/png")+ == "a labelled diagram"+ )+ assert calls[0]["config"] is selected+ assert calls[0]["image_data"] == "synthetic-image"+ assert selected == original+ assert dict(os.environ) == before+++def test_explicit_text_model_is_rejected(monkeypatch, selected):+ monkeypatch.setattr(mod, "supports_vision", lambda *_: False)+ with pytest.raises(ValueError, match="must support image input"):+ mod.get_image_description_client()+++def test_deleted_selection_does_not_silently_charge_main_model(monkeypatch, selected):+ def missing(_):+ raise ValueError("selected profile/model was not found")++ monkeypatch.setattr(mod, "resolve_llm_config_for_selection", missing)+ monkeypatch.setattr(mod, "get_llm_client", lambda: pytest.fail("unexpected fallback"))+ with pytest.raises(ValueError, match="not found"):+ mod.get_image_description_client()+++async def test_reading_captions_use_selected_model(monkeypatch, tmp_path, selected):+ from deeptutor.reading import captions++ image = tmp_path / "figure.png"+ image.write_bytes(b"synthetic image")+ saved = {}++ class Store:+ def media_items(self, _):+ return [{"name": "figure.png", "mime": "image/png"}]++ def media_path(self, *_):+ return image++ def update_media_captions(self, _, result):+ saved.update(result)+ return len(result)++ async def complete(**kwargs):+ assert kwargs["config"] is selected+ assert kwargs["image_filename"] == "figure.png"+ return "selected vision caption"++ monkeypatch.setattr(mod, "complete_with_config", complete)+ monkeypatch.setattr(mod.LLMClient, "supports_multimodal_images", lambda _: True)+ assert await captions.caption_material_media("material", store=Store()) == 1+ assert saved == {"figure.png": "selected vision caption"}+++async def test_llamaindex_image_descriptions_use_selected_model(monkeypatch, tmp_path, selected):+ from types import SimpleNamespace++ from deeptutor.services.rag.pipelines.llamaindex import document_loader++ image = tmp_path / "figure.png"+ image.write_bytes(b"synthetic image")++ class Embedding:+ config = SimpleNamespace(binding="test", model="test")++ def supports_multimodal_contents(self):+ return True++ async def embed_contents(self, contents):+ return [[0.1, 0.2] for _ in contents]++ async def complete(**kwargs):+ assert kwargs["config"] is selected+ return "selected vision caption"++ monkeypatch.setattr(mod, "complete_with_config", complete)+ monkeypatch.setattr(mod.LLMClient, "supports_multimodal_images", lambda _: True)+ monkeypatch.setattr(document_loader, "get_embedding_client", Embedding)+ nodes = await document_loader.LlamaIndexDocumentLoader()._load_image_nodes(+ [document_loader._ImageSource(path=image, origin=image)]+ )+ assert len(nodes) == 1+ assert "selected vision caption" in nodes[0].textdiff --git a/tests/services/rag/test_llamaindex_document_loader.py b/tests/services/rag/test_llamaindex_document_loader.pyindex a48ed2e75..6c2908d77 100644--- a/tests/services/rag/test_llamaindex_document_loader.py+++ b/tests/services/rag/test_llamaindex_document_loader.py@@ -416,7 +416,7 @@ def test_loader_indexes_images_extracted_from_parsed_document(return "Figure showing a bar chart."monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _VisionClient())+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(pdf_path)]))@@ -456,7 +456,7 @@ def test_loader_skips_images_when_embedding_provider_is_text_only(def _unexpected_llm_client():pytest.fail("text-only embedding must not initialize the LLM client")- monkeypatch.setattr(loader_module, "get_llm_client", _unexpected_llm_client)+ monkeypatch.setattr(loader_module, "get_image_description_client", _unexpected_llm_client)documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))@@ -504,7 +504,7 @@ def test_loader_embeds_images_with_qwen38_max_vision_capability(monkeypatch.setattr(vision_client, "complete", _complete)monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalClient())- monkeypatch.setattr(loader_module, "get_llm_client", lambda: vision_client)+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: vision_client)documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))@@ -540,7 +540,7 @@ def test_loader_skips_images_when_llm_is_text_only(return Falsemonkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _TextOnlyLLMClient())+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _TextOnlyLLMClient())documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))@@ -566,7 +566,7 @@ def test_loader_skips_images_when_llm_client_is_unavailable(raise RuntimeError("no LLM configured")monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _MultimodalEmbeddingClient())- monkeypatch.setattr(loader_module, "get_llm_client", _unavailable_llm_client)+ monkeypatch.setattr(loader_module, "get_image_description_client", _unavailable_llm_client)with caplog.at_level("WARNING"):documents = asyncio.run(loader_module.LlamaIndexDocumentLoader().load([str(image_path)]))diff --git a/tests/services/rag/test_llamaindex_image_description.py b/tests/services/rag/test_llamaindex_image_description.pyindex 9b99100fc..4a3a73d60 100644--- a/tests/services/rag/test_llamaindex_image_description.py+++ b/tests/services/rag/test_llamaindex_image_description.py@@ -56,7 +56,7 @@ def _install_multimodal_clients(return await complete_fn(prompt, **kwargs)monkeypatch.setattr(loader_module, "get_embedding_client", lambda: _EmbeddingClient())- monkeypatch.setattr(loader_module, "get_llm_client", lambda: _VisionClient())+ monkeypatch.setattr(loader_module, "get_image_description_client", lambda: _VisionClient())@pytest.mark.asynciodiff --git a/web/contracts/generated/api.ts b/web/contracts/generated/api.tsindex 6677f11f6..d2ca0ac0e 100644--- a/web/contracts/generated/api.ts+++ b/web/contracts/generated/api.ts@@ -11802,6 +11802,8 @@ export interface components {} | null;/** Image Caption */readonly image_caption?: boolean | null;+ readonly image_description_model?:+ components["schemas"]["ImageDescriptionModelSelection"] | null;};/** DocumentResponse */readonly DocumentResponse: {@@ -12120,6 +12122,16 @@ export interface components {/** Client Id */readonly client_id?: string | null;};+ /**+ * ImageDescriptionModelSelection+ * @description Reference a configured vision model without changing the chat default.+ */+ readonly ImageDescriptionModelSelection: {+ /** Model Id */+ readonly model_id: string;+ /** Profile Id */+ readonly profile_id: string;+ };/** ImaKnowledgeBaseSummary */readonly ImaKnowledgeBaseSummary: {/** Description */@@ -16312,6 +16324,8 @@ export type SchemaGuardianRestrictionsPayload =export type SchemaHttpValidationError =components["schemas"]["HTTPValidationError"];export type SchemaImaConfigUpdate = components["schemas"]["ImaConfigUpdate"];+export type SchemaImageDescriptionModelSelection =+ components["schemas"]["ImageDescriptionModelSelection"];export type SchemaImaKnowledgeBaseSummary =components["schemas"]["ImaKnowledgeBaseSummary"];export type SchemaImportedMessage = components["schemas"]["ImportedMessage"];diff --git a/web/contracts/schema/openapi.json b/web/contracts/schema/openapi.jsonindex 106b6ace9..e4ec42e48 100644--- a/web/contracts/schema/openapi.json+++ b/web/contracts/schema/openapi.json@@ -3174,6 +3174,16 @@}],"title": "Image Caption"+ },+ "image_description_model": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ImageDescriptionModelSelection"+ },+ {+ "type": "null"+ }+ ]}},"title": "DocumentParsingUpdate",@@ -3987,6 +3997,28 @@"title": "ImaKnowledgeBaseSummary","type": "object"},+ "ImageDescriptionModelSelection": {+ "additionalProperties": false,+ "description": "Reference a configured vision model without changing the chat default.",+ "properties": {+ "model_id": {+ "minLength": 1,+ "title": "Model Id",+ "type": "string"+ },+ "profile_id": {+ "minLength": 1,+ "title": "Profile Id",+ "type": "string"+ }+ },+ "required": [+ "profile_id",+ "model_id"+ ],+ "title": "ImageDescriptionModelSelection",+ "type": "object"+ },"ImportFromBookRequest": {"properties": {"chapters": {diff --git a/web/features/settings/sections/DocumentParsingSettingsSection.tsx b/web/features/settings/sections/DocumentParsingSettingsSection.tsxindex b050c0101..9e421462b 100644--- a/web/features/settings/sections/DocumentParsingSettingsSection.tsx+++ b/web/features/settings/sections/DocumentParsingSettingsSection.tsx@@ -17,6 +17,7 @@ import {import { MinerUEngineSettings } from "@/components/settings/MinerUEngineSettings";import { Toggle } from "@/components/settings/Toggle";import { apiFetch, apiUrl } from "@/lib/api";+import { ImageDescriptionModelSetting } from "./ImageDescriptionModelSetting";type EngineMeta = {id: string;@@ -30,6 +31,7 @@ type Readiness = { ready: boolean; reason: string; message: string };type DocumentParsingPayload = {engine: string;+ image_description_model?: { profile_id: string; model_id: string } | null;engines: Record<string, Record<string, unknown>>;available_engines: EngineMeta[];readiness: Record<string, Readiness>;@@ -138,6 +140,11 @@ export default function DocumentParsingSettingsPage() {{!loading && data && (<>+ <ImageDescriptionModelSetting+ value={data.image_description_model ?? null}+ disabled={busy}+ onChange={(selection) => putDocumentParsing({ image_description_model: selection })}+ /><section className="mb-10"><header className="mb-3"><h2 className="text-[15px] font-semibold tracking-tight text-[var(--foreground)]">diff --git a/web/features/settings/sections/ImageDescriptionModelSetting.tsx b/web/features/settings/sections/ImageDescriptionModelSetting.tsxnew file mode 100644index 000000000..a494b8c52--- /dev/null+++ b/web/features/settings/sections/ImageDescriptionModelSetting.tsx@@ -0,0 +1,83 @@+'use client'++import { useEffect, useState } from 'react'+import { useTranslation } from 'react-i18next'+import { nativeSelectClass, selectOptionClass } from '@/components/settings/shared'+import { listLLMOptions, type LLMOption } from '@/lib/llm-options'++type Selection = { profile_id: string; model_id: string }++export function ImageDescriptionModelSetting({+ value,+ disabled,+ onChange,+}: {+ value: Selection | null+ disabled: boolean+ onChange: (selection: Selection | null) => void+}) {+ const { t } = useTranslation()+ const [models, setModels] = useState<LLMOption[]>([])+ const [failed, setFailed] = useState(false)++ useEffect(() => {+ let active = true+ listLLMOptions().then(+ result => {+ if (active) setModels(result.options.filter(model => model.supports_vision))+ },+ () => {+ if (active) setFailed(true)+ }+ )+ return () => {+ active = false+ }+ }, [])++ const key = (selection: Selection) => JSON.stringify([selection.profile_id, selection.model_id])+ const selected = value ? key(value) : ''+ const missing = value && !models.some(model => key(model) === selected)++ return (+ <section className="mb-10">+ <label htmlFor="image-description-model" className="mb-2 block text-[13px] font-medium">+ {t('Image description model')}+ </label>+ <select+ id="image-description-model"+ className={nativeSelectClass}+ disabled={disabled}+ value={selected}+ onChange={event => {+ const model = models.find(item => key(item) === event.target.value)+ onChange(model ? { profile_id: model.profile_id, model_id: model.model_id } : null)+ }}+ >+ <option value="" className={selectOptionClass}>+ {t('Use main LLM (fallback)')}+ </option>+ {missing && (+ <option value={selected} disabled>+ {t('Selected model unavailable')}+ </option>+ )}+ {models.map(model => (+ <option key={key(model)} value={key(model)} className={selectOptionClass}>+ {model.profile_name} / {model.model_name}+ </option>+ ))}+ </select>+ <p className="mt-2 text-xs text-[var(--muted-foreground)]">+ {t(+ 'Select a vision model for image descriptions in documents.'+ )}+ </p>+ {failed && (+ <p role="alert" className="mt-2 text-xs text-red-600">+ {t('Failed to load image description models.')}+ </p>+ )}+ </section>+ )+}diff --git a/web/lib/settings-extensions.ts b/web/lib/settings-extensions.tsindex 97dad24d3..484402965 100644--- a/web/lib/settings-extensions.ts+++ b/web/lib/settings-extensions.ts@@ -61,9 +61,13 @@ export async function applyExtensionPayload(if (!endpoint) throw new Error(`Unknown settings section: ${key}`);let body = guardian?.[1] === "materials" ? { book_ids: payload } : payload;if (key === "document-parsing") {- const { engine, engines } = payload as { engine: string; engines: Record<string, unknown> };+ const { engine, engines, image_description_model } = payload as {+ engine: string;+ engines: Record<string, unknown>;+ image_description_model?: { profile_id: string; model_id: string } | null;+ };// MinerU has its own draft; never overwrite it with this page's snapshot.- body = { engine, engines: Object.fromEntries(Object.entries(engines).filter(([name]) => name !== "mineru")) };+ body = { engine, image_description_model, engines: Object.fromEntries(Object.entries(engines).filter(([name]) => name !== "mineru")) };}const response = await apiFetch(apiUrl(endpoint), {method: "PUT",diff --git a/web/locales/de/app.json b/web/locales/de/app.jsonindex 80071a718..e06557fe9 100644--- a/web/locales/de/app.json+++ b/web/locales/de/app.json@@ -5270,5 +5270,9 @@"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "Diese Knowledge Base samt Dokumenten, Einstellungen und Indizes verschieben. Bestehende Workspace-Zuweisungen werden aktualisiert.","Review move": "Verschiebung prüfen","{{count}} files · {{bytes}} bytes": "{{count}} Dateien · {{bytes}} Bytes",- "Assignments updated: {{names}}": "Aktualisierte Zuweisungen: {{names}}"+ "Assignments updated: {{names}}": "Aktualisierte Zuweisungen: {{names}}",+ "Image description model": "Modell für Bildbeschreibungen",+ "Use main LLM (fallback)": "Hauptmodell verwenden (Standard)",+ "Select a vision model for image descriptions in documents.": "Wählen Sie ein Vision-Modell für Bildbeschreibungen in Dokumenten.",+ "Failed to load image description models.": "Die Modelle für Bildbeschreibungen konnten nicht geladen werden."}diff --git a/web/locales/en/app.json b/web/locales/en/app.jsonindex c6167eb21..5995f9f4e 100644--- a/web/locales/en/app.json+++ b/web/locales/en/app.json@@ -5270,5 +5270,9 @@"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.","Review move": "Review move","{{count}} files · {{bytes}} bytes": "{{count}} files · {{bytes}} bytes",- "Assignments updated: {{names}}": "Assignments updated: {{names}}"+ "Assignments updated: {{names}}": "Assignments updated: {{names}}",+ "Image description model": "Image description model",+ "Use main LLM (fallback)": "Use main LLM (fallback)",+ "Failed to load image description models.": "Failed to load image description models.",+ "Select a vision model for image descriptions in documents.": "Select a vision model for image descriptions in documents."}diff --git a/web/locales/zh/app.json b/web/locales/zh/app.jsonindex 4312257b3..ae0de240c 100644--- a/web/locales/zh/app.json+++ b/web/locales/zh/app.json@@ -5270,5 +5270,9 @@"Move this knowledge base with its documents, settings, and indexes. Existing workspace assignments are updated.": "连同文档、设置和索引一起移动此知识库。现有工作区分配会同步更新。","Review move": "检查移动详情","{{count}} files · {{bytes}} bytes": "{{count}} 个文件 · {{bytes}} 字节",- "Assignments updated: {{names}}": "将更新以下工作区的分配:{{names}}"+ "Assignments updated: {{names}}": "将更新以下工作区的分配:{{names}}",+ "Image description model": "图片描述模型",+ "Use main LLM (fallback)": "使用主模型(默认)",+ "Failed to load image description models.": "无法加载图片描述模型。",+ "Select a vision model for image descriptions in documents.": "选择用于文档图片描述的视觉模型。"}diff --git a/web/tests/image-description-model.spec.tsx b/web/tests/image-description-model.spec.tsxnew file mode 100644index 000000000..9bc4d1cce--- /dev/null+++ b/web/tests/image-description-model.spec.tsx@@ -0,0 +1,103 @@+import React, { useState } from 'react'+import { fireEvent, render, screen, waitFor } from '@testing-library/react'+import { beforeEach, expect, it, vi } from 'vitest'+import { ImageDescriptionModelSetting } from '@/features/settings/sections/ImageDescriptionModelSetting'+import { applyExtensionPayload } from '@/lib/settings-extensions'++const mocks = vi.hoisted(() => ({ options: vi.fn(), fetch: vi.fn() }))+vi.mock('@/lib/llm-options', () => ({ listLLMOptions: () => mocks.options() }))+vi.mock('@/lib/api', () => ({+ apiUrl: (value: string) => value,+ apiFetch: (...args: unknown[]) => mocks.fetch(...args),+}))+vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }) }))++beforeEach(() => {+ mocks.options.mockResolvedValue({+ options: [+ {+ profile_id: 'vision',+ model_id: 'image',+ profile_name: 'Pictures',+ model_name: 'Vision',+ supports_vision: true,+ },+ {+ profile_id: 'chat',+ model_id: 'text',+ profile_name: 'Chat',+ model_name: 'Text',+ supports_vision: false,+ },+ ],+ })+ mocks.fetch.mockResolvedValue({ ok: true })+})++function Harness() {+ const [value, setValue] = useState<{ profile_id: string; model_id: string } | null>(null)+ return (+ <>+ <ImageDescriptionModelSetting value={value} disabled={false} onChange={setValue} />+ <button+ onClick={() =>+ applyExtensionPayload('document-parsing', {+ engine: 'text_only',+ engines: { mineru: { api_token: 'must-not-overwrite' }, text_only: {} },+ image_description_model: value,+ })+ }+ >+ Apply+ </button>+ </>+ )+}++it('lists vision models and sends explicit IDs or null without overwriting the MinerU draft', async () => {+ render(<Harness />)+ await screen.findByRole('option', { name: 'Pictures / Vision' })+ expect(screen.queryByRole('option', { name: 'Chat / Text' })).toBeNull()+ fireEvent.change(screen.getByLabelText('Image description model'), {+ target: { value: '["vision","image"]' },+ })+ fireEvent.click(screen.getByText('Apply'))+ await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(1))+ expect(JSON.parse(mocks.fetch.mock.calls[0][1].body)).toEqual({+ engine: 'text_only',+ engines: { text_only: {} },+ image_description_model: { profile_id: 'vision', model_id: 'image' },+ })+ fireEvent.change(screen.getByLabelText('Image description model'), { target: { value: '' } })+ fireEvent.click(screen.getByText('Apply'))+ await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2))+ expect(JSON.parse(mocks.fetch.mock.calls[1][1].body).image_description_model).toBeNull()+})++it('keeps an unavailable saved selection visible instead of silently choosing the main model', async () => {+ render(+ <ImageDescriptionModelSetting+ value={{ profile_id: 'deleted', model_id: 'gone' }}+ disabled={false}+ onChange={vi.fn()}+ />+ )+ await screen.findByRole('option', { name: 'Pictures / Vision' })+ expect(screen.getByLabelText('Image description model')).toHaveValue('["deleted","gone"]')+ expect(screen.getByRole('option', { name: 'Selected model unavailable' })).toBeDisabled()+})++it('reports a model-list error while retaining the ability to reset', async () => {+ mocks.options.mockRejectedValue(new Error('unavailable'))+ const onChange = vi.fn()+ render(+ <ImageDescriptionModelSetting+ value={{ profile_id: 'saved', model_id: 'model' }}+ disabled={false}+ onChange={onChange}+ />+ )+ await screen.findByRole('alert')+ fireEvent.change(screen.getByLabelText('Image description model'), { target: { value: '' } })+ expect(onChange).toHaveBeenCalledWith(null)+})diff --git a/web/tests/settings-unified-draft.spec.tsx b/web/tests/settings-unified-draft.spec.tsxindex 18a272783..7fcdc393e 100644--- a/web/tests/settings-unified-draft.spec.tsx+++ b/web/tests/settings-unified-draft.spec.tsx@@ -40,7 +40,10 @@ vi.mock("@/context/AppShellContext", () => ({codeBlockWrapLongLines: false,}),}));-vi.mock("@/lib/llm-options", () => ({ invalidateLLMOptionsCache: vi.fn() }));+vi.mock("@/lib/llm-options", () => ({+ invalidateLLMOptionsCache: vi.fn(),+ listLLMOptions: vi.fn(async () => ({ active: null, options: [] })),+}));let settings: ReturnType<typeof useSettings>;let live: ReturnType<typeof defaultCatalog>;let stored: any;