DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): persist reindex hashes only for indexed sources (#1481)
Bingxi Zhao (Frank)2faceb1f5a15
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex cf9f6cda4..7f9b6ec45 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -40,7 +40,11 @@ from deeptutor.api.routers.auth import require_adminfrom deeptutor.api.utils.progress_broadcaster import ProgressBroadcasterfrom deeptutor.api.utils.task_id_manager import TaskIDManagerfrom deeptutor.api.utils.task_log_stream import capture_task_logs, get_task_stream_manager-from deeptutor.knowledge.add_documents import DocumentAdder, remove_raw_document+from deeptutor.knowledge.add_documents import (+ DocumentAdder,+ hashes_for_indexed_files,+ remove_raw_document,+)from deeptutor.knowledge.initializer import KnowledgeBaseInitializerfrom deeptutor.knowledge.kb_types import is_connected_kb, supports_local_raw_filesfrom deeptutor.knowledge.manager import KnowledgeBaseManager@@ -3503,6 +3507,22 @@ async def run_reindex_task(# that provider rather than forcing the default pipeline.rag_service = RAGService(kb_base_dir=str(base_path), provider=None)+ # Only files represented in the completed index may receive a+ # duplicate-detection hash. LlamaIndex may skip one malformed file+ # while successfully indexing its siblings (#1481).+ requested_files = {str(Path(path).resolve()) for path in file_paths}+ indexed_files: set[str] | None = None+ reindexed_hashes: dict[str, str] | None = None++ def remember_indexed_files(paths: list[str]) -> None:+ nonlocal indexed_files+ confirmed: set[str] = set()+ for path in paths:+ resolved = str(Path(path).resolve())+ if resolved in requested_files:+ confirmed.add(resolved)+ indexed_files = (indexed_files or set()) | confirmed+def _on_progress(batch_num: int, total_batches: int) -> None:progress_tracker.update(ProgressStage.PROCESSING_DOCUMENTS,@@ -3518,6 +3538,9 @@ async def run_reindex_task(# return is reserved for "no documents to index" — surface that# specifically too.def persist_terminal_state(candidate_root=None):+ completed_count = (+ len(indexed_files) if indexed_files is not None else len(file_paths)+ )if candidate_root is None:completed_at = datetime.now().isoformat()metadata_file = kb_dir / "metadata.json"@@ -3530,8 +3553,10 @@ async def run_reindex_task(metadata = loaded_metadatametadata["last_updated"] = completed_atmetadata["last_indexed_at"] = completed_at- metadata["last_indexed_count"] = len(file_paths)+ metadata["last_indexed_count"] = completed_countmetadata["last_indexed_action"] = "reindex"+ if reindexed_hashes is not None:+ metadata["file_hashes"] = reindexed_hashesatomic_write_json(metadata_file, metadata)except Exception:raise@@ -3541,7 +3566,7 @@ async def run_reindex_task("Re-index complete",current=len(file_paths),total=len(file_paths),- indexed_count=len(file_paths),+ indexed_count=completed_count,index_changed=True,index_action="reindex",publication_version=candidate_root.name if candidate_root else None,@@ -3559,7 +3584,7 @@ async def run_reindex_task("progress_percent": 100,"current": len(file_paths),"total": len(file_paths),- "indexed_count": len(file_paths),+ "indexed_count": completed_count,"index_changed": True,"index_action": "reindex",}@@ -3599,6 +3624,7 @@ async def run_reindex_task(file_paths=file_paths,progress_callback=_on_progress,indexing_snapshot=indexing_snapshot,+ indexed_file_callback=remember_indexed_files,before_publish=persist_terminal_stateif signature_hash == LIGHTRAG_PROVIDERelse None,@@ -3615,8 +3641,20 @@ async def run_reindex_task(raise RuntimeError(f"Re-index found no valid documents to index in '{kb_name}'.")index_published = signature_hash == LIGHTRAG_PROVIDER+ if indexed_files is not None:+ reindexed_hashes = await asyncio.to_thread(+ hashes_for_indexed_files, sorted(indexed_files), raw_dir+ )+persist_terminal_state()+ if indexed_files is not None and len(indexed_files) < len(file_paths):+ _task_log(+ task_id,+ f"Re-index skipped {len(file_paths) - len(indexed_files)} file(s); see parser logs.",+ level="warning",+ )+_task_log(task_id, f"Re-index of '{kb_name}' complete", level="success")task_manager.update_task_status(task_id, "completed")task_stream_manager.emit_complete(task_id, f"Re-index of '{kb_name}' complete")diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.pyindex ea7e78ca4..d341d2689 100644--- a/deeptutor/knowledge/add_documents.py+++ b/deeptutor/knowledge/add_documents.py@@ -116,6 +116,22 @@ def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:return file_path.name+def _file_hash(file_path: Path) -> str:+ sha256_hash = hashlib.sha256()+ with open(file_path, "rb") as handle:+ for block in iter(lambda: handle.read(65536), b""):+ sha256_hash.update(block)+ return sha256_hash.hexdigest()+++def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:+ """Hash source files already confirmed in an index, keyed by their raw path."""+ return {+ _raw_hash_key(Path(file_path), raw_dir): _file_hash(Path(file_path))+ for file_path in file_paths+ }++def remove_raw_document(kb_dir: Path, file_path: Path) -> RawDocumentRemoval:"""Delete one staged raw file and drop its indexed-hash record.@@ -224,11 +240,7 @@ class DocumentAdder:return resolve_bound_provider(self.base_dir, self.kb_name)def _get_file_hash(self, file_path: Path) -> str:- sha256_hash = hashlib.sha256()- with open(file_path, "rb") as f:- for byte_block in iter(lambda: f.read(65536), b""):- sha256_hash.update(byte_block)- return sha256_hash.hexdigest()+ return _file_hash(file_path)def get_ingested_hashes(self) -> dict[str, str]:if self.metadata_file.exists():@@ -508,14 +520,9 @@ async def _bootstrap_index_from_files(try:# Record hashes so future syncs detect unchanged files.metadata = _read_metadata(metadata_file)- hashes = metadata.setdefault("file_hashes", {})- for fpath_str in source_files:- fpath = Path(fpath_str)- sha = hashlib.sha256()- with open(fpath, "rb") as fh:- for block in iter(lambda: fh.read(65536), b""):- sha.update(block)- hashes[_raw_hash_key(fpath, raw_dir)] = sha.hexdigest()+ metadata.setdefault("file_hashes", {}).update(+ hashes_for_indexed_files(source_files, raw_dir)+ )metadata["rag_provider"] = providermetadata["needs_reindex"] = Falsets = datetime.now().strftime("%Y-%m-%d %H:%M:%S")diff --git a/deeptutor/services/rag/pipelines/graphrag/ingestion.py b/deeptutor/services/rag/pipelines/graphrag/ingestion.pyindex b942fc142..b994b559a 100644--- a/deeptutor/services/rag/pipelines/graphrag/ingestion.py+++ b/deeptutor/services/rag/pipelines/graphrag/ingestion.py@@ -57,7 +57,12 @@ def _extract_parser_text(path: Path, parse_service=None) -> str: # noqa: ANN001return ""-async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:+async def prepare_input(+ file_paths: Iterable[str],+ root_dir: Path,+ *,+ source_for_input: dict[str, str] | None = None,+) -> int:"""Write parsed text for each supported file into ``root_dir/input``.Returns the number of non-empty text documents written. Parser-backed files@@ -78,7 +83,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:for file_path_str in classification.parser_files:path = Path(file_path_str)text = _extract_parser_text(path)- written += _write_doc(target_dir, path, text, used)+ written += _write_doc(target_dir, path, text, used, source_for_input)for file_path_str in classification.text_files:path = Path(file_path_str)@@ -87,7 +92,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:except Exception as exc: # pragma: no cover - defensivelogger.error("GraphRAG ingestion: failed to read text %s: %s", path.name, exc)text = ""- written += _write_doc(target_dir, path, text, used)+ written += _write_doc(target_dir, path, text, used, source_for_input)for file_path_str in classification.image_files:from deeptutor.services.parsing import get_parse_service@@ -97,7 +102,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:supports = getattr(parse_service, "supports", lambda _path: False)if supports(path):text = _extract_parser_text(path, parse_service)- written += _write_doc(target_dir, path, text, used)+ written += _write_doc(target_dir, path, text, used, source_for_input)else:logger.warning("GraphRAG ingestion skips image unsupported by the active parser: %s",@@ -109,12 +114,20 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:return written-def _write_doc(target_dir: Path, source: Path, text: str, used: set[str]) -> int:+def _write_doc(+ target_dir: Path,+ source: Path,+ text: str,+ used: set[str],+ source_for_input: dict[str, str] | None,+) -> int:if not text.strip():logger.warning("GraphRAG ingestion: empty document skipped: %s", source.name)return 0dest = _unique_txt_path(target_dir, source, used)dest.write_text(text, encoding="utf-8")+ if source_for_input is not None:+ source_for_input[dest.name] = str(source)logger.info("GraphRAG ingestion: wrote %s (%d chars)", dest.name, len(text))return 1diff --git a/deeptutor/services/rag/pipelines/graphrag/pipeline.py b/deeptutor/services/rag/pipelines/graphrag/pipeline.pyindex 185579b60..54f090b33 100644--- a/deeptutor/services/rag/pipelines/graphrag/pipeline.py+++ b/deeptutor/services/rag/pipelines/graphrag/pipeline.py@@ -12,6 +12,7 @@ message when it is not installed instead of an opaque ``ImportError``.from __future__ import annotations+import asyncioimport loggingfrom pathlib import Pathimport shutil@@ -93,13 +94,21 @@ class GraphRagPipeline:)try:gr_config.write_settings(root_dir)- count = await ingestion.prepare_input(file_paths, root_dir)+ source_for_input: dict[str, str] = {}+ count = await ingestion.prepare_input(+ file_paths, root_dir, source_for_input=source_for_input+ )if count == 0:self.logger.error("GraphRAG: no extractable documents for '%s'", kb_name)self._cleanup_failed_version_dir(root_dir)return Falseawait self._build(root_dir, is_update=False)storage.write_meta(root_dir)+ if indexed_file_callback := kwargs.get("indexed_file_callback"):+ names = await asyncio.to_thread(storage.indexed_input_names, root_dir)+ indexed_file_callback(+ sorted(source_for_input[name] for name in names if name in source_for_input)+ )self.logger.info("KB '%s' initialized with GraphRAG (%d docs)", kb_name, count)return Trueexcept Exception as exc:diff --git a/deeptutor/services/rag/pipelines/graphrag/storage.py b/deeptutor/services/rag/pipelines/graphrag/storage.pyindex 690de9c02..8125c4dad 100644--- a/deeptutor/services/rag/pipelines/graphrag/storage.py+++ b/deeptutor/services/rag/pipelines/graphrag/storage.py@@ -62,6 +62,34 @@ def has_output(root_dir: Path | None) -> bool:return any((out / f"{name}.parquet").exists() for name in OUTPUT_TABLES)+def indexed_input_names(root_dir: Path) -> set[str]:+ """Input filenames with persisted GraphRAG text units; empty on unknown output."""+ documents = output_dir(root_dir) / "documents.parquet"+ text_units = output_dir(root_dir) / "text_units.parquet"+ if not documents.is_file() or not text_units.is_file():+ return set()+ try:+ import pandas as pd++ rows = pd.read_parquet(documents, columns=["title", "text_unit_ids"])+ unit_ids = set(pd.read_parquet(text_units, columns=["id"])["id"])+ except Exception as exc:+ logger.warning("Could not inspect GraphRAG document receipts: %s", exc)+ return set()++ names: set[str] = set()+ for title, text_unit_ids in rows.itertuples(index=False, name=None):+ if not isinstance(title, str) or isinstance(text_unit_ids, (str, bytes)):+ continue+ try:+ has_persisted_unit = any(unit_id in unit_ids for unit_id in text_unit_ids)+ except TypeError:+ has_persisted_unit = False+ if has_persisted_unit:+ names.add(Path(title).name)+ return names++def write_meta(root_dir: Path) -> None:"""Write a flat-layout ``meta.json`` so the version is listed as ready.@@ -94,5 +122,6 @@ __all__ = ["input_dir","output_dir","has_output",+ "indexed_input_names","write_meta",]diff --git a/deeptutor/services/rag/pipelines/lightrag/pipeline.py b/deeptutor/services/rag/pipelines/lightrag/pipeline.pyindex f3f2d7456..a9b88049b 100644--- a/deeptutor/services/rag/pipelines/lightrag/pipeline.py+++ b/deeptutor/services/rag/pipelines/lightrag/pipeline.py@@ -453,6 +453,10 @@ class LightRagPipeline:kwargs.get("publish_embedding_binding") if outcome.complete else None,)self._clear_pending_policy(kb_name)+ if outcome.complete and (indexed_file_callback := kwargs.get("indexed_file_callback")):+ # Reconciliation confirmed every requested file as processed+ # before publication, so these paths can be hashed (#1481).+ indexed_file_callback(file_paths)return outcome.completeexcept asyncio.CancelledError:raisediff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex d298989dc..a6c603bb6 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -182,6 +182,17 @@ class LlamaIndexPipeline:if signature is not None:write_version_meta(kb_dir, signature, storage_dir=storage_dir)+ indexed_file_callback = kwargs.get("indexed_file_callback")+ if indexed_file_callback is not None:+ indexed_paths: set[str] = set()+ for document in documents:+ metadata = getattr(document, "metadata", None)+ if isinstance(metadata, dict):+ path = metadata.get("file_path")+ if isinstance(path, str) and path:+ indexed_paths.add(path)+ indexed_file_callback(sorted(indexed_paths))+self.logger.info(f"KB '{kb_name}' initialized successfully with LlamaIndex")return Truediff --git a/deeptutor/services/rag/pipelines/pageindex/pipeline.py b/deeptutor/services/rag/pipelines/pageindex/pipeline.pyindex b2c92e720..6403b5356 100644--- a/deeptutor/services/rag/pipelines/pageindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/pageindex/pipeline.py@@ -115,12 +115,14 @@ class PageIndexPipeline:)try:manifest = storage._empty_manifest(self.provider)+ submitted_by_name: dict[str, str] = {}count = await self._ingest(file_paths,manifest,progress_callback,storage_dir=storage_dir,mode=self._processing_mode(kb_name),+ submitted_by_name=submitted_by_name,)if count == 0:self.logger.error("PageIndex: no supported documents to index for '%s'", kb_name)@@ -128,6 +130,10 @@ class PageIndexPipeline:return Falsestorage.write_manifest(storage_dir, manifest)storage.write_meta(storage_dir, provider=self.provider)+ if indexed_file_callback := kwargs.get("indexed_file_callback"):+ # submit_document waits for processing, and the published+ # manifest retains only the last file for each basename.+ indexed_file_callback(sorted(submitted_by_name.values()))self.logger.info("KB '%s' initialized with PageIndex (%d docs)", kb_name, count)return Trueexcept Exception as exc:@@ -177,6 +183,7 @@ class PageIndexPipeline:*,storage_dir: Path,mode: str | None,+ submitted_by_name: dict[str, str] | None = None,) -> int:supported = [fp for fp in file_paths if is_supported_file(fp, self.provider)]skipped = [fp for fp in file_paths if not is_supported_file(fp, self.provider)]@@ -193,6 +200,8 @@ class PageIndexPipeline:doc_id = await client.submit_document(path, mode=mode)size = path.stat().st_size if path.exists() else Nonestorage.upsert_doc(manifest, path.name, doc_id, size=size)+ if submitted_by_name is not None:+ submitted_by_name[path.name] = str(path)if progress_callback:progress_callback(idx, total)return totaldiff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.pyindex c7a9b1a80..f1f63ac3a 100644--- a/tests/api/test_knowledge_router.py+++ b/tests/api/test_knowledge_router.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import hashlibimport importlibimport jsonfrom pathlib import Path@@ -1598,6 +1599,7 @@ def test_reindex_task_persists_completed_progress(json.dumps({"fixture": {"status": "processed"}}), encoding="utf-8")storage.write_meta(version)+ kwargs["indexed_file_callback"]([str(raw_dir / "fixture.txt")])if embedding_changed:embedding.model = "later-default"return True@@ -1654,6 +1656,73 @@ def test_reindex_task_persists_completed_progress(assert entry["last_indexed_action"] == "reindex"assert bool(entry.get("needs_reindex")) is embedding_changedassert bool(entry.get("embedding_mismatch")) is embedding_changed+ metadata = json.loads((base_dir / "kb" / "metadata.json").read_text(encoding="utf-8"))+ assert metadata["file_hashes"] == {+ "fixture.txt": hashlib.sha256(b"synthetic fixture").hexdigest()+ }+++def test_reindex_records_only_files_confirmed_in_llamaindex(monkeypatch, tmp_path: Path) -> None:+ """A reindexed file is a duplicate; one skipped by parsing remains retryable (#1481)."""+ from deeptutor.knowledge.add_documents import DocumentAdder++ base_dir = tmp_path / "knowledge_bases"+ kb_dir = base_dir / "kb"+ raw_dir = kb_dir / "raw"+ raw_dir.mkdir(parents=True)+ indexed = raw_dir / "indexed.txt"+ skipped = raw_dir / "skipped.txt"+ indexed.write_text("indexed content", encoding="utf-8")+ skipped.write_text("parse failed", encoding="utf-8")+ (kb_dir / "metadata.json").write_text(+ json.dumps({"file_hashes": {"removed.txt": "stale"}}), encoding="utf-8"+ )+ (base_dir / "kb_config.json").write_text(+ json.dumps(+ {+ "knowledge_bases": {+ "kb": {"path": "kb", "rag_provider": "llamaindex", "status": "processing"}+ }+ }+ ),+ encoding="utf-8",+ )++ class _SuccessfulRagService:+ def __init__(self, *_args, **_kwargs) -> None:+ pass++ async def initialize(self, *_args, **kwargs) -> bool:+ kwargs["indexed_file_callback"]([str(indexed)])+ _write_ready_llamaindex_version(kb_dir)+ return True++ rag_service_module = importlib.import_module("deeptutor.services.rag.service")+ monkeypatch.setattr(rag_service_module, "RAGService", _SuccessfulRagService)++ asyncio.run(+ knowledge_router_module.run_reindex_task(+ kb_name="kb",+ base_dir=str(base_dir),+ task_id="reindex-hashes-test",+ signature_hash="sig",+ )+ )++ metadata = json.loads((kb_dir / "metadata.json").read_text(encoding="utf-8"))+ assert metadata["file_hashes"] == {+ "indexed.txt": hashlib.sha256(b"indexed content").hexdigest()+ }+ assert metadata["last_indexed_count"] == 1+ progress = json.loads((kb_dir / ".progress.json").read_text(encoding="utf-8"))+ assert progress["indexed_count"] == 1+ adder = DocumentAdder("kb", base_dir=str(base_dir), rag_provider="llamaindex")+ duplicate = tmp_path / "indexed.txt"+ duplicate.write_text("indexed content", encoding="utf-8")+ retry = tmp_path / "skipped.txt"+ retry.write_text("parse failed", encoding="utf-8")+ assert adder.add_documents([str(duplicate)]) == []+ assert adder.add_documents([str(retry)]) == [skipped]def test_reindex_task_preserves_prepublication_failure(monkeypatch, tmp_path: Path) -> None:diff --git a/tests/services/rag/test_graphrag_pipeline.py b/tests/services/rag/test_graphrag_pipeline.pyindex 9b0eab9e6..7510ed5e2 100644--- a/tests/services/rag/test_graphrag_pipeline.py+++ b/tests/services/rag/test_graphrag_pipeline.py@@ -950,10 +950,14 @@ def test_ingestion_avoids_name_collisions(tmp_path) -> None:p.parent.mkdir(parents=True)p.write_text("content", encoding="utf-8")- count = asyncio.run(ingestion.prepare_input([str(a), str(b)], root))+ source_for_input: dict[str, str] = {}+ count = asyncio.run(+ ingestion.prepare_input([str(a), str(b)], root, source_for_input=source_for_input)+ )assert count == 2names = sorted(p.name for p in storage.input_dir(root).glob("*.txt"))assert names == ["doc.txt", "doc_1.txt"]+ assert source_for_input == {"doc.txt": str(a), "doc_1.txt": str(b)}# --------------------------------------------------------------------------- #@@ -1043,9 +1047,11 @@ def test_initialize_orchestrates_index(tmp_path, monkeypatch) -> None:txt.write_text("graph content", encoding="utf-8")pipe = GraphRagPipeline(kb_base_dir=str(tmp_path))- ok = asyncio.run(pipe.initialize("kb", [str(txt)]))+ receipts: list[list[str]] = []+ ok = asyncio.run(pipe.initialize("kb", [str(txt)], indexed_file_callback=receipts.append))assert ok is True+ assert receipts == [[]] # No documents.parquet: a build marker alone proves no source.assert calls == [{"root": calls[0]["root"],@@ -1059,6 +1065,45 @@ def test_initialize_orchestrates_index(tmp_path, monkeypatch) -> None:assert (root / storage.META_FILENAME).exists()+def test_initialize_receipt_requires_persisted_document_text_units(tmp_path, monkeypatch) -> None:+ pd = pytest.importorskip("pandas")+ pytest.importorskip("pyarrow")+ _force_available(monkeypatch, True)+ _stub_build(monkeypatch)+ first = tmp_path / "first" / "doc.txt"+ second = tmp_path / "second" / "doc.txt"+ empty = tmp_path / "empty.txt"+ for path, content in ((first, "indexed"), (second, "loaded but not chunked"), (empty, " ")):+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(content, encoding="utf-8")++ async def build_with_documents(root_dir, **_kwargs):+ output = storage.output_dir(Path(root_dir))+ output.mkdir(parents=True, exist_ok=True)+ pd.DataFrame(+ {+ "title": ["doc.txt", "doc_1.txt"],+ "text_unit_ids": [["unit-1"], ["missing-unit"]],+ }+ ).to_parquet(output / "documents.parquet")+ pd.DataFrame({"id": ["unit-1"]}).to_parquet(output / "text_units.parquet")+ (output / "entities.parquet").write_bytes(b"")++ monkeypatch.setattr(engine, "build", build_with_documents)+ receipts: list[list[str]] = []+ pipe = GraphRagPipeline(kb_base_dir=str(tmp_path))++ assert asyncio.run(+ pipe.initialize(+ "kb",+ [str(first), str(second), str(empty)],+ indexed_file_callback=receipts.append,+ )+ )++ assert receipts == [[str(first)]]++def test_initialize_no_text_returns_false(tmp_path, monkeypatch) -> None:_force_available(monkeypatch, True)calls = _stub_build(monkeypatch)diff --git a/tests/services/rag/test_lightrag_roles.py b/tests/services/rag/test_lightrag_roles.pyindex b8475aef5..e47281f33 100644--- a/tests/services/rag/test_lightrag_roles.py+++ b/tests/services/rag/test_lightrag_roles.py@@ -933,7 +933,16 @@ def test_task_publishes_actual_frozen_embedding_after_defaults_change()monkeypatch.setattr(pipeline, "_run_indexing", index)- assert asyncio.run(pipeline.initialize("kb", ["doc.md"], indexing_snapshot=accepted))+ receipts: list[list[str]] = []+ assert asyncio.run(+ pipeline.initialize(+ "kb",+ ["doc.md"],+ indexing_snapshot=accepted,+ indexed_file_callback=receipts.append,+ )+ )+ assert receipts == [["doc.md"]]published = storage.latest_published_root(kb)meta = json.loads((published / "meta.json").read_text())assert meta["embedding_model"] == "embed-one"diff --git a/tests/services/rag/test_llamaindex_storage_layout.py b/tests/services/rag/test_llamaindex_storage_layout.pyindex fd59f6ad7..4e94ce337 100644--- a/tests/services/rag/test_llamaindex_storage_layout.py+++ b/tests/services/rag/test_llamaindex_storage_layout.py@@ -18,6 +18,44 @@ def _signature() -> EmbeddingSignature:)+async def test_reindex_receipt_excludes_a_file_skipped_by_parsing(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from deeptutor.services.rag.pipelines.llamaindex import pipeline as pipeline_module+ from deeptutor.services.rag.pipelines.llamaindex import storage as storage_module+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline++ raw_dir = tmp_path / "kb" / "raw"+ raw_dir.mkdir(parents=True)+ indexed = raw_dir / "indexed.txt"+ skipped = raw_dir / "skipped.txt"+ indexed.write_text("indexed content", encoding="utf-8")+ skipped.write_text(" ", encoding="utf-8")++ def create_index(documents, storage_dir: Path, *, show_progress: bool) -> None:+ assert len(documents) == 1+ storage_dir.mkdir(parents=True, exist_ok=True)+ (storage_dir / "docstore.json").write_text("{}", encoding="utf-8")+ (storage_dir / "index_store.json").write_text("{}", encoding="utf-8")++ async def verify_embedding(_self) -> None:+ return None++ monkeypatch.setattr(LlamaIndexPipeline, "_configure_settings", lambda _self: None)+ monkeypatch.setattr(LlamaIndexPipeline, "_verify_embedding_connectivity", verify_embedding)+ monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)+ monkeypatch.setattr(storage_module, "create_index", create_index)+ pipeline = LlamaIndexPipeline(kb_base_dir=str(tmp_path), signature_provider=_signature)+ receipts: list[list[str]] = []++ assert await pipeline.initialize(+ "kb", [str(indexed), str(skipped)], indexed_file_callback=receipts.append+ )++ assert receipts == [[str(indexed)]]++@pytest.mark.asyncioasync def test_incremental_add_migrates_matching_legacy_index_to_flat_version(tmp_path: Path,diff --git a/tests/services/rag/test_pageindex_pipeline.py b/tests/services/rag/test_pageindex_pipeline.pyindex 22f954c7b..7f737587b 100644--- a/tests/services/rag/test_pageindex_pipeline.py+++ b/tests/services/rag/test_pageindex_pipeline.py@@ -79,6 +79,30 @@ def test_initialize_submits_supported_and_skips_others(tmp_path) -> None:assert docs["a.pdf"]["doc_id"] == "pi-a.pdf"+def test_initialize_receipt_includes_only_published_manifest_files(tmp_path: Path) -> None:+ client = FakeClient()+ first = tmp_path / "first" / "same.pdf"+ second = tmp_path / "second" / "same.pdf"+ skipped = tmp_path / "skipped.png"+ first.parent.mkdir()+ second.parent.mkdir()+ first.write_text("first", encoding="utf-8")+ second.write_text("second", encoding="utf-8")+ skipped.write_text("image", encoding="utf-8")+ receipts: list[list[str]] = []++ assert asyncio.run(+ _pipe(tmp_path, client).initialize(+ "kb",+ [str(first), str(second), str(skipped)],+ indexed_file_callback=receipts.append,+ )+ )++ assert receipts == [[str(second)]]+ assert set(_manifest(tmp_path, "kb")["docs"]) == {"same.pdf"}++def test_initialize_no_supported_returns_false(tmp_path) -> None:client = FakeClient()png = tmp_path / "c.png"