DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag): remove corpus manifest ceiling and publish figures before indexing (#1802)
Bingxi Zhao (Frank)b0521ac3fae0
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex c1cb0ec6c..fe280e9e8 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -229,6 +229,10 @@ class LlamaIndexPipeline:f"(chunking + embedding)...")+ # #1802: validate and publish pixels before persisting nodes that+ # reference them. Failed indexing can leave unused immutable assets;+ # failed asset publication must not leave a partially accepted index.+ VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)await _run_with_stall_guard(lambda: storage.create_index(documents, storage_dir, show_progress=should_show_progress()@@ -238,7 +242,6 @@ class LlamaIndexPipeline:)self.logger.info(f"Index persisted to {storage_dir}")- VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)if signature is not None:write_version_meta(kb_dir, signature, storage_dir=storage_dir)@@ -423,6 +426,7 @@ class LlamaIndexPipeline:self.logger.warning("No valid documents to add")return False+ VisualAssetStore(kb_dir).publish(visual_candidates)if plan.existing_storage is not None:self.logger.info(f"Loading existing index from {plan.existing_storage}...")num_added = await _run_with_stall_guard(@@ -433,7 +437,6 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Added {num_added} documents to existing index")- VisualAssetStore(kb_dir).publish(visual_candidates)if signature is not None and plan.storage_dir != plan.existing_storage:write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)else:@@ -447,7 +450,6 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Created new index with {num_added} documents")- VisualAssetStore(kb_dir).publish(visual_candidates)if signature is not None:write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.pyindex 8ca73d58e..771a3c09b 100644--- a/deeptutor/services/rag/visual_assets.py+++ b/deeptutor/services/rag/visual_assets.py@@ -11,6 +11,7 @@ from dataclasses import dataclassfrom hashlib import sha256from io import BytesIOimport json+import loggingimport osfrom pathlib import Pathimport re@@ -23,7 +24,7 @@ from deeptutor.services.parsing.types import ParsedDocumentMAX_IMAGE_BYTES = 5 * 1024 * 1024MAX_MODEL_IMAGES = 2MAX_IMAGE_PIXELS = 30_000_000-MAX_MANIFEST_BYTES = 16 * 1024 * 1024+logger = logging.getLogger(__name__)_ID_RE = re.compile(r"^[0-9a-f]{64}$")_MARKDOWN_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")_MIME_EXT = {@@ -231,13 +232,30 @@ class VisualAssetStore:if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):return {}try:- if self.manifest_path.stat().st_size > MAX_MANIFEST_BYTES:- return {}+ return self._read_manifest()+ except OSError as exc:+ logger.warning("Cannot read source figures for KB '%s': %s", self.kb_dir.name, exc)+ return {}++ def _read_manifest(self) -> dict[str, dict[str, Any]]:+ # #1802: the corpus grows independently of the per-image request budget.+ # Never interpret a large or corrupt manifest as an empty store on a write.+ try:+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(+ self.kb_dir.resolve()+ ):+ raise OSError("Visual asset directory escapes knowledge base")+ if self.manifest_path.is_symlink():+ raise OSError("Visual asset manifest must not be a symbolic link")payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))- except (OSError, ValueError):+ except FileNotFoundError:return {}+ except (OSError, ValueError) as exc:+ raise OSError(f"Cannot read visual asset manifest {self.manifest_path}: {exc}") from excrecords = payload.get("assets") if isinstance(payload, dict) else None- return records if isinstance(records, dict) else {}+ if not isinstance(records, dict) or any(not isinstance(v, dict) for v in records.values()):+ raise OSError(f"Invalid visual asset manifest {self.manifest_path}")+ return recordsdef publish(self,@@ -250,7 +268,7 @@ class VisualAssetStore:if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):raise OSError("Visual asset directory escapes knowledge base")self.root.mkdir(parents=True, exist_ok=True)- prior = self.records()+ prior = self._read_manifest()incoming = {item.record["asset_id"]: item.record for item in candidates}if replace:remaining: dict[str, dict[str, Any]] = {}@@ -275,8 +293,6 @@ class VisualAssetStore:manifest = json.dumps({"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True).encode()- if len(manifest) > MAX_MANIFEST_BYTES:- raise OSError("Visual asset manifest exceeds its size limit")for candidate in candidates:record = candidate.recordloaded = _image_bytes(candidate.path)@@ -316,7 +332,7 @@ class VisualAssetStore:return record, datadef remove_source(self, source_relative_path: str) -> None:- prior = self.records()+ prior = self._read_manifest()updated = {key: valuefor key, value in prior.items()@@ -335,7 +351,7 @@ class VisualAssetStore:self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)def move_source(self, old_relative_path: str, new_relative_path: str) -> None:- prior = self.records()+ prior = self._read_manifest()updated = {}changed = Falsefor key, value in prior.items():diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.pyindex 48ee69ef4..9e4add08f 100644--- a/tests/services/rag/test_llamaindex_pipeline_stall.py+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py@@ -298,6 +298,38 @@ async def test_add_documents_fails_bounded_when_new_index_stalls(assert time.monotonic() - started < 10[email protected]("operation", ["initialize", "add_new", "add_existing"])+async def test_failed_figure_publication_does_not_modify_persisted_index(+ tmp_path, monkeypatch: pytest.MonkeyPatch, operation: str+) -> None:+ """#1802: a failed asset write must happen before an index accepts the document."""+ pipeline_module, storage_module, _ = _llamaindex_modules()+ pipeline = _make_pipeline(tmp_path, monkeypatch)+ writes = []+ monkeypatch.setattr(storage_module, "create_index", lambda *a, **k: writes.append("create"))+ monkeypatch.setattr(storage_module, "insert_documents", lambda *a, **k: writes.append("insert"))+ if operation == "add_existing":+ existing = tmp_path / "kb" / "llamaindex_storage"+ existing.mkdir(parents=True)+ monkeypatch.setattr(+ storage_module,+ "resolve_add_storage_plan",+ lambda *_: storage_module.AddStoragePlan(existing, existing),+ )++ def fail_publication(*args, **kwargs):+ raise OSError("Cannot publish source figures for KB 'kb': disk is full")++ monkeypatch.setattr(pipeline_module.VisualAssetStore, "publish", fail_publication)+ with pytest.raises(OSError, match="disk is full"):+ if operation == "initialize":+ await pipeline.initialize("kb", ["doc.pdf"])+ else:+ await pipeline.add_documents("kb", ["doc.pdf"])+ assert writes == []++@pytest.mark.asyncioasync def test_initialize_succeeds_when_indexing_completes(tmp_path, monkeypatch: pytest.MonkeyPatchdiff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.pyindex a77120644..7e19084f1 100644--- a/tests/services/rag/test_visual_assets.py+++ b/tests/services/rag/test_visual_assets.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport asyncioimport base64+import jsonfrom pathlib import Pathfrom types import SimpleNamespaceimport zipfile@@ -104,24 +105,64 @@ def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: sassert not persisted.exists()-def test_manifest_limit_does_not_replace_existing_assets(tmp_path: Path, monkeypatch):+def test_large_existing_manifest_remains_readable_and_accepts_more_figures(tmp_path: Path):+ """#1802: a corpus beyond the old 16 MiB limit must retain every reference."""import deeptutor.services.rag.visual_assets as assets_modulekb_dir, source, image, parsed = _fixture(tmp_path)original = collect_visual_assets(parsed, source, kb_dir)[0]store = VisualAssetStore(kb_dir)store.publish([original])- manifest_before = store.manifest_path.read_bytes()- monkeypatch.setattr(assets_module, "MAX_MANIFEST_BYTES", len(manifest_before) + 10)+ # Model a legacy manifest from a figure-heavy KB without 15,000 image copies.+ records = {+ f"{index:064x}": {**original.record, "asset_id": f"{index:064x}", "context": "x" * 1000}+ for index in range(11000)+ }+ records[original.record["asset_id"]] = original.record+ store.manifest_path.write_text(json.dumps({"version": 1, "assets": records}), encoding="utf-8")+ assert store.manifest_path.stat().st_size > 16 * 1024 * 1024+ assert store.read(original.record["asset_id"]) is not Nonesecond = assets_module.VisualAssetCandidate(path=image,record={**original.record, "asset_id": "a" * 64},)- with pytest.raises(OSError, match="manifest exceeds"):- store.publish([second])- assert store.manifest_path.read_bytes() == manifest_before+ store.publish([second])+ assert len(store.records()) == len(records) + 1assert store.read(original.record["asset_id"]) is not None- assert store.read(second.record["asset_id"]) is None+ assert store.read(second.record["asset_id"]) is not None++[email protected]("operation", ["publish", "remove_source", "move_source"])+def test_unreadable_manifest_is_reported_and_never_overwritten(tmp_path: Path, caplog, operation):+ kb_dir, source, _image, parsed = _fixture(tmp_path)+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]+ store = VisualAssetStore(kb_dir)+ store.publish([candidate])+ store.manifest_path.write_bytes(b"{broken manifest")+ before = store.manifest_path.read_bytes()+ assert store.read(candidate.record["asset_id"]) is None+ assert "Cannot read source figures for KB 'kb'" in caplog.text+ with pytest.raises(OSError, match="Cannot read visual asset manifest"):+ if operation == "publish":+ store.publish([candidate])+ elif operation == "remove_source":+ store.remove_source(source.name)+ else:+ store.move_source(source.name, "moved.pdf")+ assert store.manifest_path.read_bytes() == before+++def test_changed_image_does_not_replace_existing_manifest(tmp_path: Path):+ kb_dir, source, image, parsed = _fixture(tmp_path)+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]+ store = VisualAssetStore(kb_dir)+ store.publish([candidate])+ before = store.manifest_path.read_bytes()+ image.write_bytes(b"not an image anymore")+ with pytest.raises(OSError, match="Source image changed"):+ store.publish([candidate])+ assert store.manifest_path.read_bytes() == before+ assert store.read(candidate.record["asset_id"]) is not Nonedef test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):