Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): remove corpus manifest ceiling and publish figures before indexing (#1802)

Bingxi Zhao (Frank)b0521ac3fae0
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index c1cb0ec6c..fe280e9e8 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -229,6 +229,10 @@ class LlamaIndexPipeline:
f"(chunking + embedding)..."
)
+ # #1802: validate and publish pixels before persisting nodes that
+ # reference them. Failed indexing can leave unused immutable assets;
+ # failed asset publication must not leave a partially accepted index.
+ VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)
await _run_with_stall_guard(
lambda: storage.create_index(
documents, storage_dir, show_progress=should_show_progress()
@@ -238,7 +242,6 @@ class LlamaIndexPipeline:
)
self.logger.info(f"Index persisted to {storage_dir}")
- VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)
if signature is not None:
write_version_meta(kb_dir, signature, storage_dir=storage_dir)
@@ -423,6 +426,7 @@ class LlamaIndexPipeline:
self.logger.warning("No valid documents to add")
return False
+ VisualAssetStore(kb_dir).publish(visual_candidates)
if plan.existing_storage is not None:
self.logger.info(f"Loading existing index from {plan.existing_storage}...")
num_added = await _run_with_stall_guard(
@@ -433,7 +437,6 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Added {num_added} documents to existing index")
- VisualAssetStore(kb_dir).publish(visual_candidates)
if signature is not None and plan.storage_dir != plan.existing_storage:
write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
else:
@@ -447,7 +450,6 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Created new index with {num_added} documents")
- VisualAssetStore(kb_dir).publish(visual_candidates)
if signature is not None:
write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.py
index 8ca73d58e..771a3c09b 100644
--- a/deeptutor/services/rag/visual_assets.py
+++ b/deeptutor/services/rag/visual_assets.py
@@ -11,6 +11,7 @@ from dataclasses import dataclass
from hashlib import sha256
from io import BytesIO
import json
+import logging
import os
from pathlib import Path
import re
@@ -23,7 +24,7 @@ from deeptutor.services.parsing.types import ParsedDocument
MAX_IMAGE_BYTES = 5 * 1024 * 1024
MAX_MODEL_IMAGES = 2
MAX_IMAGE_PIXELS = 30_000_000
-MAX_MANIFEST_BYTES = 16 * 1024 * 1024
+logger = logging.getLogger(__name__)
_ID_RE = re.compile(r"^[0-9a-f]{64}$")
_MARKDOWN_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
_MIME_EXT = {
@@ -231,13 +232,30 @@ class VisualAssetStore:
if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
return {}
try:
- if self.manifest_path.stat().st_size > MAX_MANIFEST_BYTES:
- return {}
+ return self._read_manifest()
+ except OSError as exc:
+ logger.warning("Cannot read source figures for KB '%s': %s", self.kb_dir.name, exc)
+ return {}
+
+ def _read_manifest(self) -> dict[str, dict[str, Any]]:
+ # #1802: the corpus grows independently of the per-image request budget.
+ # Never interpret a large or corrupt manifest as an empty store on a write.
+ try:
+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(
+ self.kb_dir.resolve()
+ ):
+ raise OSError("Visual asset directory escapes knowledge base")
+ if self.manifest_path.is_symlink():
+ raise OSError("Visual asset manifest must not be a symbolic link")
payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))
- except (OSError, ValueError):
+ except FileNotFoundError:
return {}
+ except (OSError, ValueError) as exc:
+ raise OSError(f"Cannot read visual asset manifest {self.manifest_path}: {exc}") from exc
records = payload.get("assets") if isinstance(payload, dict) else None
- return records if isinstance(records, dict) else {}
+ if not isinstance(records, dict) or any(not isinstance(v, dict) for v in records.values()):
+ raise OSError(f"Invalid visual asset manifest {self.manifest_path}")
+ return records
def publish(
self,
@@ -250,7 +268,7 @@ class VisualAssetStore:
if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
raise OSError("Visual asset directory escapes knowledge base")
self.root.mkdir(parents=True, exist_ok=True)
- prior = self.records()
+ prior = self._read_manifest()
incoming = {item.record["asset_id"]: item.record for item in candidates}
if replace:
remaining: dict[str, dict[str, Any]] = {}
@@ -275,8 +293,6 @@ class VisualAssetStore:
manifest = json.dumps(
{"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
).encode()
- if len(manifest) > MAX_MANIFEST_BYTES:
- raise OSError("Visual asset manifest exceeds its size limit")
for candidate in candidates:
record = candidate.record
loaded = _image_bytes(candidate.path)
@@ -316,7 +332,7 @@ class VisualAssetStore:
return record, data
def remove_source(self, source_relative_path: str) -> None:
- prior = self.records()
+ prior = self._read_manifest()
updated = {
key: value
for key, value in prior.items()
@@ -335,7 +351,7 @@ class VisualAssetStore:
self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)
def move_source(self, old_relative_path: str, new_relative_path: str) -> None:
- prior = self.records()
+ prior = self._read_manifest()
updated = {}
changed = False
for key, value in prior.items():
diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.py
index 48ee69ef4..9e4add08f 100644
--- a/tests/services/rag/test_llamaindex_pipeline_stall.py
+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py
@@ -298,6 +298,38 @@ async def test_add_documents_fails_bounded_when_new_index_stalls(
assert time.monotonic() - started < 10
[email protected]("operation", ["initialize", "add_new", "add_existing"])
+async def test_failed_figure_publication_does_not_modify_persisted_index(
+ tmp_path, monkeypatch: pytest.MonkeyPatch, operation: str
+) -> None:
+ """#1802: a failed asset write must happen before an index accepts the document."""
+ pipeline_module, storage_module, _ = _llamaindex_modules()
+ pipeline = _make_pipeline(tmp_path, monkeypatch)
+ writes = []
+ monkeypatch.setattr(storage_module, "create_index", lambda *a, **k: writes.append("create"))
+ monkeypatch.setattr(storage_module, "insert_documents", lambda *a, **k: writes.append("insert"))
+ if operation == "add_existing":
+ existing = tmp_path / "kb" / "llamaindex_storage"
+ existing.mkdir(parents=True)
+ monkeypatch.setattr(
+ storage_module,
+ "resolve_add_storage_plan",
+ lambda *_: storage_module.AddStoragePlan(existing, existing),
+ )
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Cannot publish source figures for KB 'kb': disk is full")
+
+ monkeypatch.setattr(pipeline_module.VisualAssetStore, "publish", fail_publication)
+ with pytest.raises(OSError, match="disk is full"):
+ if operation == "initialize":
+ await pipeline.initialize("kb", ["doc.pdf"])
+ else:
+ await pipeline.add_documents("kb", ["doc.pdf"])
+ assert writes == []
+
+
@pytest.mark.asyncio
async def test_initialize_succeeds_when_indexing_completes(
tmp_path, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.py
index a77120644..7e19084f1 100644
--- a/tests/services/rag/test_visual_assets.py
+++ b/tests/services/rag/test_visual_assets.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import base64
+import json
from pathlib import Path
from types import SimpleNamespace
import zipfile
@@ -104,24 +105,64 @@ def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: s
assert not persisted.exists()
-def test_manifest_limit_does_not_replace_existing_assets(tmp_path: Path, monkeypatch):
+def test_large_existing_manifest_remains_readable_and_accepts_more_figures(tmp_path: Path):
+ """#1802: a corpus beyond the old 16 MiB limit must retain every reference."""
import deeptutor.services.rag.visual_assets as assets_module
kb_dir, source, image, parsed = _fixture(tmp_path)
original = collect_visual_assets(parsed, source, kb_dir)[0]
store = VisualAssetStore(kb_dir)
store.publish([original])
- manifest_before = store.manifest_path.read_bytes()
- monkeypatch.setattr(assets_module, "MAX_MANIFEST_BYTES", len(manifest_before) + 10)
+ # Model a legacy manifest from a figure-heavy KB without 15,000 image copies.
+ records = {
+ f"{index:064x}": {**original.record, "asset_id": f"{index:064x}", "context": "x" * 1000}
+ for index in range(11000)
+ }
+ records[original.record["asset_id"]] = original.record
+ store.manifest_path.write_text(json.dumps({"version": 1, "assets": records}), encoding="utf-8")
+ assert store.manifest_path.stat().st_size > 16 * 1024 * 1024
+ assert store.read(original.record["asset_id"]) is not None
second = assets_module.VisualAssetCandidate(
path=image,
record={**original.record, "asset_id": "a" * 64},
)
- with pytest.raises(OSError, match="manifest exceeds"):
- store.publish([second])
- assert store.manifest_path.read_bytes() == manifest_before
+ store.publish([second])
+ assert len(store.records()) == len(records) + 1
assert store.read(original.record["asset_id"]) is not None
- assert store.read(second.record["asset_id"]) is None
+ assert store.read(second.record["asset_id"]) is not None
+
+
[email protected]("operation", ["publish", "remove_source", "move_source"])
+def test_unreadable_manifest_is_reported_and_never_overwritten(tmp_path: Path, caplog, operation):
+ kb_dir, source, _image, parsed = _fixture(tmp_path)
+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]
+ store = VisualAssetStore(kb_dir)
+ store.publish([candidate])
+ store.manifest_path.write_bytes(b"{broken manifest")
+ before = store.manifest_path.read_bytes()
+ assert store.read(candidate.record["asset_id"]) is None
+ assert "Cannot read source figures for KB 'kb'" in caplog.text
+ with pytest.raises(OSError, match="Cannot read visual asset manifest"):
+ if operation == "publish":
+ store.publish([candidate])
+ elif operation == "remove_source":
+ store.remove_source(source.name)
+ else:
+ store.move_source(source.name, "moved.pdf")
+ assert store.manifest_path.read_bytes() == before
+
+
+def test_changed_image_does_not_replace_existing_manifest(tmp_path: Path):
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]
+ store = VisualAssetStore(kb_dir)
+ store.publish([candidate])
+ before = store.manifest_path.read_bytes()
+ image.write_bytes(b"not an image anymore")
+ with pytest.raises(OSError, match="Source image changed"):
+ store.publish([candidate])
+ assert store.manifest_path.read_bytes() == before
+ assert store.read(candidate.record["asset_id"]) is not None
def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):