Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

perf(parsing): cache image descriptions per workspace

ZyPulse1c07ef4e4486
diff --git a/deeptutor/reading/captions.py b/deeptutor/reading/captions.py
index f24dcdb84..7cfc3222f 100644
--- a/deeptutor/reading/captions.py
+++ b/deeptutor/reading/captions.py
@@ -23,6 +23,7 @@ from typing import Any
from deeptutor.reading.store import ReadingStore
from deeptutor.services.config.runtime_settings import load_document_parsing_settings
from deeptutor.services.llm.client import get_llm_client
+from deeptutor.services.llm.image_caption_cache import complete_image_caption
from deeptutor.services.rag.pipelines.llamaindex.config import image_description_limits
logger = logging.getLogger(__name__)
@@ -97,12 +98,14 @@ async def caption_material_media(
data = await asyncio.to_thread(path.read_bytes)
encoded = base64.b64encode(data).decode("ascii")
text = await asyncio.wait_for(
- client.complete(
+ complete_image_caption(
+ client,
CAPTION_PROMPT,
system_prompt=CAPTION_SYSTEM_PROMPT,
image_data=encoded,
image_mime_type=mime,
image_filename=name,
+ force=force,
),
timeout=timeout_seconds,
)
diff --git a/deeptutor/services/llm/image_caption_cache.py b/deeptutor/services/llm/image_caption_cache.py
new file mode 100644
index 000000000..5cc909e0e
--- /dev/null
+++ b/deeptutor/services/llm/image_caption_cache.py
@@ -0,0 +1,128 @@
+"""Best-effort, workspace-scoped caching for individual image descriptions.
+
+Only successful, nonempty captions are cached. The digest covers the complete
+request and model identity; only the digest and caption are written to disk,
+never image bytes, prompts, endpoint URLs, headers, or credentials. Removing
+``<workspace>/parse_cache/image_captions`` safely discards this derived cache.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from deeptutor.services.file_io import atomic_write_json
+from deeptutor.services.path_service import get_path_service
+
+from .client import LLMClient
+from .config import LLMConfig, get_llm_config
+
+logger = logging.getLogger(__name__)
+_CACHE_VERSION = 1
+_CONFIG_FIELDS = (
+ "model",
+ "api_key",
+ "base_url",
+ "effective_url",
+ "binding",
+ "provider_name",
+ "provider_mode",
+ "api_version",
+ "extra_headers",
+ "wire_api",
+ "api_format",
+ "reasoning_effort",
+ "context_window",
+ "max_tokens",
+ "temperature",
+)
+
+
+def _cache_path(client: Any, request: dict[str, str]) -> Path | None:
+ config = getattr(client, "config", None)
+ if not isinstance(config, LLMConfig):
+ # Unknown client adapters must not share an incomplete model identity.
+ return None
+ try:
+ configs = [config]
+ if isinstance(client, LLMClient) and type(client).complete is LLMClient.complete:
+ # The legacy facade inherits protocol/header defaults from the
+ # current runtime config. Isolated clients own their full config.
+ configs.append(get_llm_config())
+ identity = [{key: getattr(item, key) for key in _CONFIG_FIELDS} for item in configs]
+ request_identity = {key: value for key, value in request.items() if key != "image_data"}
+ request_identity["image_sha256"] = hashlib.sha256(
+ request["image_data"].encode("ascii")
+ ).hexdigest()
+ payload = json.dumps(
+ {"version": _CACHE_VERSION, "config": identity, "request": request_identity},
+ sort_keys=True,
+ ensure_ascii=True,
+ separators=(",", ":"),
+ )
+ digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
+ root = get_path_service().get_parse_cache_root() / "image_captions"
+ return root / digest[:2] / f"{digest}.json"
+ except Exception: # noqa: BLE001 - inability to cache must not break captioning
+ logger.debug("Image caption cache identity unavailable; bypassing cache")
+ return None
+
+
+def _read_caption(path: Path) -> str | None:
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(payload, dict) or payload.get("version") != _CACHE_VERSION:
+ return None
+ caption = payload.get("caption")
+ if isinstance(caption, str) and caption.strip():
+ return caption.strip()
+ except (OSError, ValueError, UnicodeError):
+ # Missing, unreadable, or malformed entries are ordinary cache misses.
+ pass
+ return None
+
+
+def _write_caption(path: Path, caption: str) -> None:
+ try:
+ atomic_write_json(path, {"version": _CACHE_VERSION, "caption": caption})
+ except OSError:
+ logger.debug("Image caption cache write failed; retaining generated caption")
+
+
+async def complete_image_caption(
+ client: Any,
+ prompt: str,
+ *,
+ system_prompt: str,
+ image_data: str,
+ image_mime_type: str,
+ image_filename: str,
+ force: bool = False,
+) -> str:
+ """Reuse a matching caption, or call the client and cache successful text.
+
+ ``force`` bypasses reads and replaces the matching entry on success.
+ Provider errors and cancellation propagate to the caller's existing
+ per-image timeout/failure handling. Concurrent misses may both call the
+ provider; atomic replacement prevents partially written cache entries.
+ """
+ request = {
+ "prompt": prompt,
+ "system_prompt": system_prompt,
+ "image_data": image_data,
+ "image_mime_type": image_mime_type,
+ "image_filename": image_filename,
+ }
+ path = await asyncio.to_thread(_cache_path, client, request)
+ if path is not None and not force:
+ cached = await asyncio.to_thread(_read_caption, path)
+ if cached is not None:
+ return cached
+ caption = (await client.complete(**request) or "").strip()
+ if path is not None and caption:
+ await asyncio.to_thread(_write_caption, path, caption)
+ return caption
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index 9a8cef74e..541c4c8b1 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -25,6 +25,7 @@ from llama_index.core.schema import ImageNode
from deeptutor.services.config.runtime_settings import DOCUMENT_PARSING_ENGINE_LITEPARSE
from deeptutor.services.embedding import get_embedding_client
from deeptutor.services.llm.client import get_llm_client
+from deeptutor.services.llm.image_caption_cache import complete_image_caption
from deeptutor.services.rag.file_routing import FileTypeRouter
from deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assets
from deeptutor.utils.document_validator import DocumentValidator
@@ -458,7 +459,8 @@ class LlamaIndexDocumentLoader:
async def _describe_image(
self, llm_client: Any, file_path: Path, image_base64: str, mimetype: str
) -> str:
- response = await llm_client.complete(
+ response = await complete_image_caption(
+ llm_client,
IMAGE_DESCRIPTION_PROMPT,
system_prompt=IMAGE_DESCRIPTION_SYSTEM_PROMPT,
image_data=image_base64,
diff --git a/tests/reading/test_reading_captions.py b/tests/reading/test_reading_captions.py
index c37756456..d32f3efb9 100644
--- a/tests/reading/test_reading_captions.py
+++ b/tests/reading/test_reading_captions.py
@@ -100,6 +100,32 @@ async def test_existing_captions_are_skipped(
assert stub.calls == []
+async def test_missing_captions_reuse_cache_but_force_refreshes(store, monkeypatch, tmp_path):
+ from types import SimpleNamespace
+
+ from deeptutor.services.llm import image_caption_cache
+ from deeptutor.services.llm.config import LLMConfig
+
+ stub = _StubClient()
+ stub.config = LLMConfig(model="vision-test", api_key="test-key")
+ monkeypatch.setattr(captions_module, "get_llm_client", lambda: stub)
+ monkeypatch.setattr(
+ image_caption_cache,
+ "get_path_service",
+ lambda: SimpleNamespace(get_parse_cache_root=lambda: tmp_path / "cache"),
+ )
+ rows = store.media_items(_MATERIAL_ID)
+ index = tmp_path / "reading" / _MATERIAL_ID / "media.json"
+ assert await captions_module.caption_material_media(_MATERIAL_ID, store=store) == 3
+ index.write_text(json.dumps(rows)) # Retry ingest with the same source images.
+ assert await captions_module.caption_material_media(_MATERIAL_ID, store=store) == 3
+ assert len(stub.calls) == 3
+ # The stub returns identical text, so the store reports no changed rows.
+ assert await captions_module.caption_material_media(_MATERIAL_ID, store=store, force=True) == 0
+ assert len(stub.calls) == 6
+
+
@pytest.mark.asyncio
async def test_single_failure_keeps_the_rest(
store: ReadingStore, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/services/llm/test_image_caption_cache.py b/tests/services/llm/test_image_caption_cache.py
new file mode 100644
index 000000000..8af82ffac
--- /dev/null
+++ b/tests/services/llm/test_image_caption_cache.py
@@ -0,0 +1,275 @@
+"""Cache behavior without provider calls or user workspace data."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from deeptutor.services.llm import image_caption_cache as cache
+from deeptutor.services.llm.client import LLMClient
+from deeptutor.services.llm.config import LLMConfig
+
+
+class Client:
+ def __init__(self) -> None:
+ self.config = LLMConfig(
+ model="vision-test", api_key="test-secret", base_url="https://example.test/v1"
+ )
+ self.calls = 0
+ self.result = " visible chart "
+ self.error: BaseException | None = None
+
+ async def complete(self, **kwargs: str) -> str:
+ self.calls += 1
+ if self.error is not None:
+ raise self.error
+ return self.result
+
+
+def cache_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ root = tmp_path / "parse_cache"
+ monkeypatch.setattr(
+ cache, "get_path_service", lambda: SimpleNamespace(get_parse_cache_root=lambda: root)
+ )
+ return root
+
+
+async def describe(client, **overrides) -> str:
+ request = {
+ "prompt": "Describe the figure",
+ "system_prompt": "Only visible facts",
+ "image_data": "cG5nLWJ5dGVz",
+ "image_mime_type": "image/png",
+ "image_filename": "figure.png",
+ }
+ request.update(overrides)
+ return await cache.complete_image_caption(client, **request)
+
+
+async def test_reuses_caption_and_persists_only_derived_text(cache_root):
+ client = Client()
+ client.config.extra_headers = {"Authorization": "header-secret"}
+ assert await describe(client) == "visible chart"
+ assert await describe(client) == "visible chart"
+ assert client.calls == 1
+ entries = list(cache_root.rglob("*.json"))
+ assert len(entries) == 1
+ assert json.loads(entries[0].read_text()) == {"version": 1, "caption": "visible chart"}
+ for secret in ("test-secret", "header-secret", "example.test", "cG5nLWJ5dGVz"):
+ assert secret not in entries[0].read_text()
+ assert secret not in str(entries[0])
+
+
+ ("field", "value"),
+ [
+ ("model", "other-model"),
+ ("base_url", "https://other.test"),
+ ("effective_url", "https://other.test/v1/responses"),
+ ("binding", "anthropic"),
+ ("provider_name", "other-provider"),
+ ("provider_mode", "other-mode"),
+ ("api_version", "new-version"),
+ ("api_key", "other-account"),
+ ("extra_headers", {"x-routing": "other"}),
+ ("wire_api", "responses"),
+ ("api_format", "anthropic"),
+ ("reasoning_effort", "high"),
+ ("max_tokens", 200),
+ ("temperature", 0.1),
+ ],
+)
+async def test_model_identity_changes_invalidate(cache_root, field, value):
+ client = Client()
+ await describe(client)
+ setattr(client.config, field, value)
+ await describe(client)
+ assert client.calls == 2
+
+
+ ("field", "value"),
+ [
+ ("prompt", "Read all text"),
+ ("system_prompt", "New instructions"),
+ ("image_data", "ZGlmZmVyZW50"),
+ ("image_mime_type", "image/jpeg"),
+ ("image_filename", "other.png"),
+ ],
+)
+async def test_request_changes_invalidate(cache_root, field, value):
+ client = Client()
+ await describe(client)
+ await describe(client, **{field: value})
+ assert client.calls == 2
+
+
+async def test_force_refresh_replaces_only_on_success(cache_root):
+ client = Client()
+ await describe(client)
+ client.result = "new chart"
+ assert await describe(client, force=True) == "new chart"
+ client.error = RuntimeError("provider unavailable")
+ with pytest.raises(RuntimeError):
+ await describe(client, force=True)
+ assert await describe(client) == "new chart"
+ assert client.calls == 3
+
+
[email protected]("result", ["", " ", None])
+async def test_empty_responses_are_not_cached(cache_root, result):
+ client = Client()
+ client.result = result
+ assert await describe(client) == ""
+ assert await describe(client) == ""
+ assert client.calls == 2
+ assert not list(cache_root.rglob("*.json"))
+
+
[email protected]("error", [RuntimeError("failed"), asyncio.CancelledError()])
+async def test_failures_propagate_and_are_not_cached(cache_root, error):
+ client = Client()
+ client.error = error
+ with pytest.raises(type(error)):
+ await describe(client)
+ assert not list(cache_root.rglob("*.json"))
+ client.error = None
+ assert await describe(client) == "visible chart"
+ assert client.calls == 2
+
+
+ "payload", ["broken", "null", "[]", '{"version": 99}', '{"version": 1, "caption": 7}']
+)
+async def test_corrupt_entries_are_replaced(cache_root, payload):
+ client = Client()
+ await describe(client)
+ entry = next(cache_root.rglob("*.json"))
+ entry.write_text(payload)
+ assert await describe(client) == "visible chart"
+ assert client.calls == 2
+ assert json.loads(entry.read_text())["caption"] == "visible chart"
+
+
+async def test_write_failure_keeps_successful_caption(cache_root, monkeypatch):
+ def fail(*args):
+ raise PermissionError("read-only filesystem")
+
+ monkeypatch.setattr(cache, "atomic_write_json", fail)
+ client = Client()
+ assert await describe(client) == "visible chart"
+ assert await describe(client) == "visible chart"
+ assert client.calls == 2
+
+
+async def test_workspace_switch_does_not_reuse_other_users_caption(cache_root, monkeypatch):
+ client = Client()
+ await describe(client)
+ other = cache_root.parent / "other_user" / "parse_cache"
+ monkeypatch.setattr(
+ cache, "get_path_service", lambda: SimpleNamespace(get_parse_cache_root=lambda: other)
+ )
+ await describe(client)
+ assert client.calls == 2
+ assert len(list(other.rglob("*.json"))) == 1
+
+
+async def test_unknown_client_identity_bypasses_cache(cache_root):
+ client = Client()
+ client.config = SimpleNamespace(model="vision-test")
+ await describe(client)
+ await describe(client)
+ assert client.calls == 2
+ assert not list(cache_root.rglob("*.json"))
+
+
+async def test_legacy_client_runtime_defaults_invalidate(cache_root, monkeypatch):
+ from deeptutor.services.llm import factory
+
+ client = LLMClient(Client().config, configure_env=False)
+ runtime = Client().config
+ monkeypatch.setattr(cache, "get_llm_config", lambda: runtime)
+ calls = []
+
+ async def complete(**kwargs):
+ calls.append(kwargs)
+ return "chart"
+
+ monkeypatch.setattr(factory, "complete", complete)
+ await describe(client)
+ await describe(client)
+ runtime.wire_api = "responses"
+ await describe(client)
+ assert len(calls) == 2
+
+
+async def test_concurrent_writes_leave_complete_json(cache_root):
+ client = Client()
+ results = await asyncio.gather(*(describe(client) for _ in range(8)))
+ assert results == ["visible chart"] * 8
+ entry = next(cache_root.rglob("*.json"))
+ assert json.loads(entry.read_text())["caption"] == "visible chart"
+ assert len(list(entry.parent.iterdir())) == 1
+
+
+async def test_unresolved_workspace_bypasses_without_global_fallback(cache_root, monkeypatch):
+ def fail():
+ raise RuntimeError("invalid workspace")
+
+ monkeypatch.setattr(cache, "get_path_service", fail)
+ client = Client()
+ assert await describe(client) == "visible chart"
+ assert not list(cache_root.rglob("*.json"))
+
+
+async def test_timeout_cancels_provider_without_caching(cache_root, monkeypatch):
+ client = Client()
+ cancelled = asyncio.Event()
+
+ async def slow(**kwargs):
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cancelled.set()
+
+ monkeypatch.setattr(client, "complete", slow)
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(describe(client), timeout=0.1)
+ assert cancelled.is_set()
+ assert not list(cache_root.rglob("*.json"))
+
+
+async def test_llamaindex_description_reuses_cache_and_preserves_source(cache_root, tmp_path):
+ pytest.importorskip("llama_index.core")
+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import (
+ LlamaIndexDocumentLoader,
+ )
+
+ path = tmp_path / "figure.png"
+ path.write_bytes(b"source image")
+ client = Client()
+ loader = LlamaIndexDocumentLoader()
+ assert await loader._describe_image(client, path, "cG5n", "image/png") == "visible chart"
+ assert await loader._describe_image(client, path, "cG5n", "image/png") == "visible chart"
+ assert client.calls == 1
+ assert path.read_bytes() == b"source image"