Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): persist reindex hashes only for indexed sources (#1481)

Bingxi Zhao (Frank)2faceb1f5a15
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index cf9f6cda4..7f9b6ec45 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -40,7 +40,11 @@ from deeptutor.api.routers.auth import require_admin
from deeptutor.api.utils.progress_broadcaster import ProgressBroadcaster
from deeptutor.api.utils.task_id_manager import TaskIDManager
from deeptutor.api.utils.task_log_stream import capture_task_logs, get_task_stream_manager
-from deeptutor.knowledge.add_documents import DocumentAdder, remove_raw_document
+from deeptutor.knowledge.add_documents import (
+ DocumentAdder,
+ hashes_for_indexed_files,
+ remove_raw_document,
+)
from deeptutor.knowledge.initializer import KnowledgeBaseInitializer
from deeptutor.knowledge.kb_types import is_connected_kb, supports_local_raw_files
from deeptutor.knowledge.manager import KnowledgeBaseManager
@@ -3503,6 +3507,22 @@ async def run_reindex_task(
# that provider rather than forcing the default pipeline.
rag_service = RAGService(kb_base_dir=str(base_path), provider=None)
+ # Only files represented in the completed index may receive a
+ # duplicate-detection hash. LlamaIndex may skip one malformed file
+ # while successfully indexing its siblings (#1481).
+ requested_files = {str(Path(path).resolve()) for path in file_paths}
+ indexed_files: set[str] | None = None
+ reindexed_hashes: dict[str, str] | None = None
+
+ def remember_indexed_files(paths: list[str]) -> None:
+ nonlocal indexed_files
+ confirmed: set[str] = set()
+ for path in paths:
+ resolved = str(Path(path).resolve())
+ if resolved in requested_files:
+ confirmed.add(resolved)
+ indexed_files = (indexed_files or set()) | confirmed
+
def _on_progress(batch_num: int, total_batches: int) -> None:
progress_tracker.update(
ProgressStage.PROCESSING_DOCUMENTS,
@@ -3518,6 +3538,9 @@ async def run_reindex_task(
# return is reserved for "no documents to index" — surface that
# specifically too.
def persist_terminal_state(candidate_root=None):
+ completed_count = (
+ len(indexed_files) if indexed_files is not None else len(file_paths)
+ )
if candidate_root is None:
completed_at = datetime.now().isoformat()
metadata_file = kb_dir / "metadata.json"
@@ -3530,8 +3553,10 @@ async def run_reindex_task(
metadata = loaded_metadata
metadata["last_updated"] = completed_at
metadata["last_indexed_at"] = completed_at
- metadata["last_indexed_count"] = len(file_paths)
+ metadata["last_indexed_count"] = completed_count
metadata["last_indexed_action"] = "reindex"
+ if reindexed_hashes is not None:
+ metadata["file_hashes"] = reindexed_hashes
atomic_write_json(metadata_file, metadata)
except Exception:
raise
@@ -3541,7 +3566,7 @@ async def run_reindex_task(
"Re-index complete",
current=len(file_paths),
total=len(file_paths),
- indexed_count=len(file_paths),
+ indexed_count=completed_count,
index_changed=True,
index_action="reindex",
publication_version=candidate_root.name if candidate_root else None,
@@ -3559,7 +3584,7 @@ async def run_reindex_task(
"progress_percent": 100,
"current": len(file_paths),
"total": len(file_paths),
- "indexed_count": len(file_paths),
+ "indexed_count": completed_count,
"index_changed": True,
"index_action": "reindex",
}
@@ -3599,6 +3624,7 @@ async def run_reindex_task(
file_paths=file_paths,
progress_callback=_on_progress,
indexing_snapshot=indexing_snapshot,
+ indexed_file_callback=remember_indexed_files,
before_publish=persist_terminal_state
if signature_hash == LIGHTRAG_PROVIDER
else None,
@@ -3615,8 +3641,20 @@ async def run_reindex_task(
raise RuntimeError(f"Re-index found no valid documents to index in '{kb_name}'.")
index_published = signature_hash == LIGHTRAG_PROVIDER
+ if indexed_files is not None:
+ reindexed_hashes = await asyncio.to_thread(
+ hashes_for_indexed_files, sorted(indexed_files), raw_dir
+ )
+
persist_terminal_state()
+ if indexed_files is not None and len(indexed_files) < len(file_paths):
+ _task_log(
+ task_id,
+ f"Re-index skipped {len(file_paths) - len(indexed_files)} file(s); see parser logs.",
+ level="warning",
+ )
+
_task_log(task_id, f"Re-index of '{kb_name}' complete", level="success")
task_manager.update_task_status(task_id, "completed")
task_stream_manager.emit_complete(task_id, f"Re-index of '{kb_name}' complete")
diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.py
index ea7e78ca4..d341d2689 100644
--- a/deeptutor/knowledge/add_documents.py
+++ b/deeptutor/knowledge/add_documents.py
@@ -116,6 +116,22 @@ def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:
return file_path.name
+def _file_hash(file_path: Path) -> str:
+ sha256_hash = hashlib.sha256()
+ with open(file_path, "rb") as handle:
+ for block in iter(lambda: handle.read(65536), b""):
+ sha256_hash.update(block)
+ return sha256_hash.hexdigest()
+
+
+def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:
+ """Hash source files already confirmed in an index, keyed by their raw path."""
+ return {
+ _raw_hash_key(Path(file_path), raw_dir): _file_hash(Path(file_path))
+ for file_path in file_paths
+ }
+
+
def remove_raw_document(kb_dir: Path, file_path: Path) -> RawDocumentRemoval:
"""Delete one staged raw file and drop its indexed-hash record.
@@ -224,11 +240,7 @@ class DocumentAdder:
return resolve_bound_provider(self.base_dir, self.kb_name)
def _get_file_hash(self, file_path: Path) -> str:
- sha256_hash = hashlib.sha256()
- with open(file_path, "rb") as f:
- for byte_block in iter(lambda: f.read(65536), b""):
- sha256_hash.update(byte_block)
- return sha256_hash.hexdigest()
+ return _file_hash(file_path)
def get_ingested_hashes(self) -> dict[str, str]:
if self.metadata_file.exists():
@@ -508,14 +520,9 @@ async def _bootstrap_index_from_files(
try:
# Record hashes so future syncs detect unchanged files.
metadata = _read_metadata(metadata_file)
- hashes = metadata.setdefault("file_hashes", {})
- for fpath_str in source_files:
- fpath = Path(fpath_str)
- sha = hashlib.sha256()
- with open(fpath, "rb") as fh:
- for block in iter(lambda: fh.read(65536), b""):
- sha.update(block)
- hashes[_raw_hash_key(fpath, raw_dir)] = sha.hexdigest()
+ metadata.setdefault("file_hashes", {}).update(
+ hashes_for_indexed_files(source_files, raw_dir)
+ )
metadata["rag_provider"] = provider
metadata["needs_reindex"] = False
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
diff --git a/deeptutor/services/rag/pipelines/graphrag/ingestion.py b/deeptutor/services/rag/pipelines/graphrag/ingestion.py
index b942fc142..b994b559a 100644
--- a/deeptutor/services/rag/pipelines/graphrag/ingestion.py
+++ b/deeptutor/services/rag/pipelines/graphrag/ingestion.py
@@ -57,7 +57,12 @@ def _extract_parser_text(path: Path, parse_service=None) -> str: # noqa: ANN001
return ""
-async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:
+async def prepare_input(
+ file_paths: Iterable[str],
+ root_dir: Path,
+ *,
+ source_for_input: dict[str, str] | None = None,
+) -> int:
"""Write parsed text for each supported file into ``root_dir/input``.
Returns the number of non-empty text documents written. Parser-backed files
@@ -78,7 +83,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:
for file_path_str in classification.parser_files:
path = Path(file_path_str)
text = _extract_parser_text(path)
- written += _write_doc(target_dir, path, text, used)
+ written += _write_doc(target_dir, path, text, used, source_for_input)
for file_path_str in classification.text_files:
path = Path(file_path_str)
@@ -87,7 +92,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:
except Exception as exc: # pragma: no cover - defensive
logger.error("GraphRAG ingestion: failed to read text %s: %s", path.name, exc)
text = ""
- written += _write_doc(target_dir, path, text, used)
+ written += _write_doc(target_dir, path, text, used, source_for_input)
for file_path_str in classification.image_files:
from deeptutor.services.parsing import get_parse_service
@@ -97,7 +102,7 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:
supports = getattr(parse_service, "supports", lambda _path: False)
if supports(path):
text = _extract_parser_text(path, parse_service)
- written += _write_doc(target_dir, path, text, used)
+ written += _write_doc(target_dir, path, text, used, source_for_input)
else:
logger.warning(
"GraphRAG ingestion skips image unsupported by the active parser: %s",
@@ -109,12 +114,20 @@ async def prepare_input(file_paths: Iterable[str], root_dir: Path) -> int:
return written
-def _write_doc(target_dir: Path, source: Path, text: str, used: set[str]) -> int:
+def _write_doc(
+ target_dir: Path,
+ source: Path,
+ text: str,
+ used: set[str],
+ source_for_input: dict[str, str] | None,
+) -> int:
if not text.strip():
logger.warning("GraphRAG ingestion: empty document skipped: %s", source.name)
return 0
dest = _unique_txt_path(target_dir, source, used)
dest.write_text(text, encoding="utf-8")
+ if source_for_input is not None:
+ source_for_input[dest.name] = str(source)
logger.info("GraphRAG ingestion: wrote %s (%d chars)", dest.name, len(text))
return 1
diff --git a/deeptutor/services/rag/pipelines/graphrag/pipeline.py b/deeptutor/services/rag/pipelines/graphrag/pipeline.py
index 185579b60..54f090b33 100644
--- a/deeptutor/services/rag/pipelines/graphrag/pipeline.py
+++ b/deeptutor/services/rag/pipelines/graphrag/pipeline.py
@@ -12,6 +12,7 @@ message when it is not installed instead of an opaque ``ImportError``.
from __future__ import annotations
+import asyncio
import logging
from pathlib import Path
import shutil
@@ -93,13 +94,21 @@ class GraphRagPipeline:
)
try:
gr_config.write_settings(root_dir)
- count = await ingestion.prepare_input(file_paths, root_dir)
+ source_for_input: dict[str, str] = {}
+ count = await ingestion.prepare_input(
+ file_paths, root_dir, source_for_input=source_for_input
+ )
if count == 0:
self.logger.error("GraphRAG: no extractable documents for '%s'", kb_name)
self._cleanup_failed_version_dir(root_dir)
return False
await self._build(root_dir, is_update=False)
storage.write_meta(root_dir)
+ if indexed_file_callback := kwargs.get("indexed_file_callback"):
+ names = await asyncio.to_thread(storage.indexed_input_names, root_dir)
+ indexed_file_callback(
+ sorted(source_for_input[name] for name in names if name in source_for_input)
+ )
self.logger.info("KB '%s' initialized with GraphRAG (%d docs)", kb_name, count)
return True
except Exception as exc:
diff --git a/deeptutor/services/rag/pipelines/graphrag/storage.py b/deeptutor/services/rag/pipelines/graphrag/storage.py
index 690de9c02..8125c4dad 100644
--- a/deeptutor/services/rag/pipelines/graphrag/storage.py
+++ b/deeptutor/services/rag/pipelines/graphrag/storage.py
@@ -62,6 +62,34 @@ def has_output(root_dir: Path | None) -> bool:
return any((out / f"{name}.parquet").exists() for name in OUTPUT_TABLES)
+def indexed_input_names(root_dir: Path) -> set[str]:
+ """Input filenames with persisted GraphRAG text units; empty on unknown output."""
+ documents = output_dir(root_dir) / "documents.parquet"
+ text_units = output_dir(root_dir) / "text_units.parquet"
+ if not documents.is_file() or not text_units.is_file():
+ return set()
+ try:
+ import pandas as pd
+
+ rows = pd.read_parquet(documents, columns=["title", "text_unit_ids"])
+ unit_ids = set(pd.read_parquet(text_units, columns=["id"])["id"])
+ except Exception as exc:
+ logger.warning("Could not inspect GraphRAG document receipts: %s", exc)
+ return set()
+
+ names: set[str] = set()
+ for title, text_unit_ids in rows.itertuples(index=False, name=None):
+ if not isinstance(title, str) or isinstance(text_unit_ids, (str, bytes)):
+ continue
+ try:
+ has_persisted_unit = any(unit_id in unit_ids for unit_id in text_unit_ids)
+ except TypeError:
+ has_persisted_unit = False
+ if has_persisted_unit:
+ names.add(Path(title).name)
+ return names
+
+
def write_meta(root_dir: Path) -> None:
"""Write a flat-layout ``meta.json`` so the version is listed as ready.
@@ -94,5 +122,6 @@ __all__ = [
"input_dir",
"output_dir",
"has_output",
+ "indexed_input_names",
"write_meta",
]
diff --git a/deeptutor/services/rag/pipelines/lightrag/pipeline.py b/deeptutor/services/rag/pipelines/lightrag/pipeline.py
index f3f2d7456..a9b88049b 100644
--- a/deeptutor/services/rag/pipelines/lightrag/pipeline.py
+++ b/deeptutor/services/rag/pipelines/lightrag/pipeline.py
@@ -453,6 +453,10 @@ class LightRagPipeline:
kwargs.get("publish_embedding_binding") if outcome.complete else None,
)
self._clear_pending_policy(kb_name)
+ if outcome.complete and (indexed_file_callback := kwargs.get("indexed_file_callback")):
+ # Reconciliation confirmed every requested file as processed
+ # before publication, so these paths can be hashed (#1481).
+ indexed_file_callback(file_paths)
return outcome.complete
except asyncio.CancelledError:
raise
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index d298989dc..a6c603bb6 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -182,6 +182,17 @@ class LlamaIndexPipeline:
if signature is not None:
write_version_meta(kb_dir, signature, storage_dir=storage_dir)
+ indexed_file_callback = kwargs.get("indexed_file_callback")
+ if indexed_file_callback is not None:
+ indexed_paths: set[str] = set()
+ for document in documents:
+ metadata = getattr(document, "metadata", None)
+ if isinstance(metadata, dict):
+ path = metadata.get("file_path")
+ if isinstance(path, str) and path:
+ indexed_paths.add(path)
+ indexed_file_callback(sorted(indexed_paths))
+
self.logger.info(f"KB '{kb_name}' initialized successfully with LlamaIndex")
return True
diff --git a/deeptutor/services/rag/pipelines/pageindex/pipeline.py b/deeptutor/services/rag/pipelines/pageindex/pipeline.py
index b2c92e720..6403b5356 100644
--- a/deeptutor/services/rag/pipelines/pageindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/pageindex/pipeline.py
@@ -115,12 +115,14 @@ class PageIndexPipeline:
)
try:
manifest = storage._empty_manifest(self.provider)
+ submitted_by_name: dict[str, str] = {}
count = await self._ingest(
file_paths,
manifest,
progress_callback,
storage_dir=storage_dir,
mode=self._processing_mode(kb_name),
+ submitted_by_name=submitted_by_name,
)
if count == 0:
self.logger.error("PageIndex: no supported documents to index for '%s'", kb_name)
@@ -128,6 +130,10 @@ class PageIndexPipeline:
return False
storage.write_manifest(storage_dir, manifest)
storage.write_meta(storage_dir, provider=self.provider)
+ if indexed_file_callback := kwargs.get("indexed_file_callback"):
+ # submit_document waits for processing, and the published
+ # manifest retains only the last file for each basename.
+ indexed_file_callback(sorted(submitted_by_name.values()))
self.logger.info("KB '%s' initialized with PageIndex (%d docs)", kb_name, count)
return True
except Exception as exc:
@@ -177,6 +183,7 @@ class PageIndexPipeline:
*,
storage_dir: Path,
mode: str | None,
+ submitted_by_name: dict[str, str] | None = None,
) -> int:
supported = [fp for fp in file_paths if is_supported_file(fp, self.provider)]
skipped = [fp for fp in file_paths if not is_supported_file(fp, self.provider)]
@@ -193,6 +200,8 @@ class PageIndexPipeline:
doc_id = await client.submit_document(path, mode=mode)
size = path.stat().st_size if path.exists() else None
storage.upsert_doc(manifest, path.name, doc_id, size=size)
+ if submitted_by_name is not None:
+ submitted_by_name[path.name] = str(path)
if progress_callback:
progress_callback(idx, total)
return total
diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.py
index c7a9b1a80..f1f63ac3a 100644
--- a/tests/api/test_knowledge_router.py
+++ b/tests/api/test_knowledge_router.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import hashlib
import importlib
import json
from pathlib import Path
@@ -1598,6 +1599,7 @@ def test_reindex_task_persists_completed_progress(
json.dumps({"fixture": {"status": "processed"}}), encoding="utf-8"
)
storage.write_meta(version)
+ kwargs["indexed_file_callback"]([str(raw_dir / "fixture.txt")])
if embedding_changed:
embedding.model = "later-default"
return True
@@ -1654,6 +1656,73 @@ def test_reindex_task_persists_completed_progress(
assert entry["last_indexed_action"] == "reindex"
assert bool(entry.get("needs_reindex")) is embedding_changed
assert bool(entry.get("embedding_mismatch")) is embedding_changed
+ metadata = json.loads((base_dir / "kb" / "metadata.json").read_text(encoding="utf-8"))
+ assert metadata["file_hashes"] == {
+ "fixture.txt": hashlib.sha256(b"synthetic fixture").hexdigest()
+ }
+
+
+def test_reindex_records_only_files_confirmed_in_llamaindex(monkeypatch, tmp_path: Path) -> None:
+ """A reindexed file is a duplicate; one skipped by parsing remains retryable (#1481)."""
+ from deeptutor.knowledge.add_documents import DocumentAdder
+
+ base_dir = tmp_path / "knowledge_bases"
+ kb_dir = base_dir / "kb"
+ raw_dir = kb_dir / "raw"
+ raw_dir.mkdir(parents=True)
+ indexed = raw_dir / "indexed.txt"
+ skipped = raw_dir / "skipped.txt"
+ indexed.write_text("indexed content", encoding="utf-8")
+ skipped.write_text("parse failed", encoding="utf-8")
+ (kb_dir / "metadata.json").write_text(
+ json.dumps({"file_hashes": {"removed.txt": "stale"}}), encoding="utf-8"
+ )
+ (base_dir / "kb_config.json").write_text(
+ json.dumps(
+ {
+ "knowledge_bases": {
+ "kb": {"path": "kb", "rag_provider": "llamaindex", "status": "processing"}
+ }
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ class _SuccessfulRagService:
+ def __init__(self, *_args, **_kwargs) -> None:
+ pass
+
+ async def initialize(self, *_args, **kwargs) -> bool:
+ kwargs["indexed_file_callback"]([str(indexed)])
+ _write_ready_llamaindex_version(kb_dir)
+ return True
+
+ rag_service_module = importlib.import_module("deeptutor.services.rag.service")
+ monkeypatch.setattr(rag_service_module, "RAGService", _SuccessfulRagService)
+
+ asyncio.run(
+ knowledge_router_module.run_reindex_task(
+ kb_name="kb",
+ base_dir=str(base_dir),
+ task_id="reindex-hashes-test",
+ signature_hash="sig",
+ )
+ )
+
+ metadata = json.loads((kb_dir / "metadata.json").read_text(encoding="utf-8"))
+ assert metadata["file_hashes"] == {
+ "indexed.txt": hashlib.sha256(b"indexed content").hexdigest()
+ }
+ assert metadata["last_indexed_count"] == 1
+ progress = json.loads((kb_dir / ".progress.json").read_text(encoding="utf-8"))
+ assert progress["indexed_count"] == 1
+ adder = DocumentAdder("kb", base_dir=str(base_dir), rag_provider="llamaindex")
+ duplicate = tmp_path / "indexed.txt"
+ duplicate.write_text("indexed content", encoding="utf-8")
+ retry = tmp_path / "skipped.txt"
+ retry.write_text("parse failed", encoding="utf-8")
+ assert adder.add_documents([str(duplicate)]) == []
+ assert adder.add_documents([str(retry)]) == [skipped]
def test_reindex_task_preserves_prepublication_failure(monkeypatch, tmp_path: Path) -> None:
diff --git a/tests/services/rag/test_graphrag_pipeline.py b/tests/services/rag/test_graphrag_pipeline.py
index 9b0eab9e6..7510ed5e2 100644
--- a/tests/services/rag/test_graphrag_pipeline.py
+++ b/tests/services/rag/test_graphrag_pipeline.py
@@ -950,10 +950,14 @@ def test_ingestion_avoids_name_collisions(tmp_path) -> None:
p.parent.mkdir(parents=True)
p.write_text("content", encoding="utf-8")
- count = asyncio.run(ingestion.prepare_input([str(a), str(b)], root))
+ source_for_input: dict[str, str] = {}
+ count = asyncio.run(
+ ingestion.prepare_input([str(a), str(b)], root, source_for_input=source_for_input)
+ )
assert count == 2
names = sorted(p.name for p in storage.input_dir(root).glob("*.txt"))
assert names == ["doc.txt", "doc_1.txt"]
+ assert source_for_input == {"doc.txt": str(a), "doc_1.txt": str(b)}
# --------------------------------------------------------------------------- #
@@ -1043,9 +1047,11 @@ def test_initialize_orchestrates_index(tmp_path, monkeypatch) -> None:
txt.write_text("graph content", encoding="utf-8")
pipe = GraphRagPipeline(kb_base_dir=str(tmp_path))
- ok = asyncio.run(pipe.initialize("kb", [str(txt)]))
+ receipts: list[list[str]] = []
+ ok = asyncio.run(pipe.initialize("kb", [str(txt)], indexed_file_callback=receipts.append))
assert ok is True
+ assert receipts == [[]] # No documents.parquet: a build marker alone proves no source.
assert calls == [
{
"root": calls[0]["root"],
@@ -1059,6 +1065,45 @@ def test_initialize_orchestrates_index(tmp_path, monkeypatch) -> None:
assert (root / storage.META_FILENAME).exists()
+def test_initialize_receipt_requires_persisted_document_text_units(tmp_path, monkeypatch) -> None:
+ pd = pytest.importorskip("pandas")
+ pytest.importorskip("pyarrow")
+ _force_available(monkeypatch, True)
+ _stub_build(monkeypatch)
+ first = tmp_path / "first" / "doc.txt"
+ second = tmp_path / "second" / "doc.txt"
+ empty = tmp_path / "empty.txt"
+ for path, content in ((first, "indexed"), (second, "loaded but not chunked"), (empty, " ")):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content, encoding="utf-8")
+
+ async def build_with_documents(root_dir, **_kwargs):
+ output = storage.output_dir(Path(root_dir))
+ output.mkdir(parents=True, exist_ok=True)
+ pd.DataFrame(
+ {
+ "title": ["doc.txt", "doc_1.txt"],
+ "text_unit_ids": [["unit-1"], ["missing-unit"]],
+ }
+ ).to_parquet(output / "documents.parquet")
+ pd.DataFrame({"id": ["unit-1"]}).to_parquet(output / "text_units.parquet")
+ (output / "entities.parquet").write_bytes(b"")
+
+ monkeypatch.setattr(engine, "build", build_with_documents)
+ receipts: list[list[str]] = []
+ pipe = GraphRagPipeline(kb_base_dir=str(tmp_path))
+
+ assert asyncio.run(
+ pipe.initialize(
+ "kb",
+ [str(first), str(second), str(empty)],
+ indexed_file_callback=receipts.append,
+ )
+ )
+
+ assert receipts == [[str(first)]]
+
+
def test_initialize_no_text_returns_false(tmp_path, monkeypatch) -> None:
_force_available(monkeypatch, True)
calls = _stub_build(monkeypatch)
diff --git a/tests/services/rag/test_lightrag_roles.py b/tests/services/rag/test_lightrag_roles.py
index b8475aef5..e47281f33 100644
--- a/tests/services/rag/test_lightrag_roles.py
+++ b/tests/services/rag/test_lightrag_roles.py
@@ -933,7 +933,16 @@ def test_task_publishes_actual_frozen_embedding_after_defaults_change(
)
monkeypatch.setattr(pipeline, "_run_indexing", index)
- assert asyncio.run(pipeline.initialize("kb", ["doc.md"], indexing_snapshot=accepted))
+ receipts: list[list[str]] = []
+ assert asyncio.run(
+ pipeline.initialize(
+ "kb",
+ ["doc.md"],
+ indexing_snapshot=accepted,
+ indexed_file_callback=receipts.append,
+ )
+ )
+ assert receipts == [["doc.md"]]
published = storage.latest_published_root(kb)
meta = json.loads((published / "meta.json").read_text())
assert meta["embedding_model"] == "embed-one"
diff --git a/tests/services/rag/test_llamaindex_storage_layout.py b/tests/services/rag/test_llamaindex_storage_layout.py
index fd59f6ad7..4e94ce337 100644
--- a/tests/services/rag/test_llamaindex_storage_layout.py
+++ b/tests/services/rag/test_llamaindex_storage_layout.py
@@ -18,6 +18,44 @@ def _signature() -> EmbeddingSignature:
)
+async def test_reindex_receipt_excludes_a_file_skipped_by_parsing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from deeptutor.services.rag.pipelines.llamaindex import pipeline as pipeline_module
+ from deeptutor.services.rag.pipelines.llamaindex import storage as storage_module
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+
+ raw_dir = tmp_path / "kb" / "raw"
+ raw_dir.mkdir(parents=True)
+ indexed = raw_dir / "indexed.txt"
+ skipped = raw_dir / "skipped.txt"
+ indexed.write_text("indexed content", encoding="utf-8")
+ skipped.write_text(" ", encoding="utf-8")
+
+ def create_index(documents, storage_dir: Path, *, show_progress: bool) -> None:
+ assert len(documents) == 1
+ storage_dir.mkdir(parents=True, exist_ok=True)
+ (storage_dir / "docstore.json").write_text("{}", encoding="utf-8")
+ (storage_dir / "index_store.json").write_text("{}", encoding="utf-8")
+
+ async def verify_embedding(_self) -> None:
+ return None
+
+ monkeypatch.setattr(LlamaIndexPipeline, "_configure_settings", lambda _self: None)
+ monkeypatch.setattr(LlamaIndexPipeline, "_verify_embedding_connectivity", verify_embedding)
+ monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)
+ monkeypatch.setattr(storage_module, "create_index", create_index)
+ pipeline = LlamaIndexPipeline(kb_base_dir=str(tmp_path), signature_provider=_signature)
+ receipts: list[list[str]] = []
+
+ assert await pipeline.initialize(
+ "kb", [str(indexed), str(skipped)], indexed_file_callback=receipts.append
+ )
+
+ assert receipts == [[str(indexed)]]
+
+
@pytest.mark.asyncio
async def test_incremental_add_migrates_matching_legacy_index_to_flat_version(
tmp_path: Path,
diff --git a/tests/services/rag/test_pageindex_pipeline.py b/tests/services/rag/test_pageindex_pipeline.py
index 22f954c7b..7f737587b 100644
--- a/tests/services/rag/test_pageindex_pipeline.py
+++ b/tests/services/rag/test_pageindex_pipeline.py
@@ -79,6 +79,30 @@ def test_initialize_submits_supported_and_skips_others(tmp_path) -> None:
assert docs["a.pdf"]["doc_id"] == "pi-a.pdf"
+def test_initialize_receipt_includes_only_published_manifest_files(tmp_path: Path) -> None:
+ client = FakeClient()
+ first = tmp_path / "first" / "same.pdf"
+ second = tmp_path / "second" / "same.pdf"
+ skipped = tmp_path / "skipped.png"
+ first.parent.mkdir()
+ second.parent.mkdir()
+ first.write_text("first", encoding="utf-8")
+ second.write_text("second", encoding="utf-8")
+ skipped.write_text("image", encoding="utf-8")
+ receipts: list[list[str]] = []
+
+ assert asyncio.run(
+ _pipe(tmp_path, client).initialize(
+ "kb",
+ [str(first), str(second), str(skipped)],
+ indexed_file_callback=receipts.append,
+ )
+ )
+
+ assert receipts == [[str(second)]]
+ assert set(_manifest(tmp_path, "kb")["docs"]) == {"same.pdf"}
+
+
def test_initialize_no_supported_returns_false(tmp_path) -> None:
client = FakeClient()
png = tmp_path / "c.png"