Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): track confirmed files during empty-index bootstrap (#1481)

Bingxi Zhao (Frank)ce02c1fdbbbf
diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.py
index d341d2689..accba4bb3 100644
--- a/deeptutor/knowledge/add_documents.py
+++ b/deeptutor/knowledge/add_documents.py
@@ -509,19 +509,33 @@ async def _bootstrap_index_from_files(
raw_dir = kb_dir / "raw"
metadata_file = kb_dir / "metadata.json"
- success = await rag_service.initialize(kb_name=kb_name, file_paths=source_files)
+ requested = {str(Path(path).resolve()): str(Path(path)) for path in source_files}
+ confirmed: set[str] = set()
+
+ def on_indexed_file(paths: list[str]) -> None:
+ for path in paths:
+ canonical = str(Path(path).resolve())
+ if canonical in requested:
+ confirmed.add(canonical)
+
+ success = await rag_service.initialize(
+ kb_name=kb_name,
+ file_paths=source_files,
+ indexed_file_callback=on_indexed_file,
+ )
if not success:
raise RuntimeError(
f"Failed to initialize index for KB '{kb_name}' from {len(source_files)} file(s)"
)
- indexed = len(source_files)
+ indexed_files = [requested[path] for path in sorted(confirmed)]
+ indexed = len(indexed_files)
provider = rag_service._resolve_provider(kb_name)
try:
# Record hashes so future syncs detect unchanged files.
metadata = _read_metadata(metadata_file)
metadata.setdefault("file_hashes", {}).update(
- hashes_for_indexed_files(source_files, raw_dir)
+ hashes_for_indexed_files(indexed_files, raw_dir)
)
metadata["rag_provider"] = provider
metadata["needs_reindex"] = False
diff --git a/tests/knowledge/test_document_adder_provider.py b/tests/knowledge/test_document_adder_provider.py
index f0fae7a33..8ab576d4a 100644
--- a/tests/knowledge/test_document_adder_provider.py
+++ b/tests/knowledge/test_document_adder_provider.py
@@ -124,6 +124,48 @@ def test_document_adder_allows_empty_kb_to_bootstrap(monkeypatch, tmp_path: Path
assert adder.accepted_indexing_snapshot.vlm is None
+def test_empty_kb_bootstrap_hashes_only_files_confirmed_by_index(
+ monkeypatch, tmp_path: Path
+) -> None:
+ from deeptutor.knowledge import add_documents as add_module
+
+ raw_dir = tmp_path / "kb" / "raw"
+ raw_dir.mkdir(parents=True)
+ indexed = raw_dir / "indexed.txt"
+ skipped = raw_dir / "skipped.txt"
+ indexed.write_text("searchable", encoding="utf-8")
+ skipped.write_text("", encoding="utf-8")
+
+ class PartialIndexService:
+ def __init__(self, *_args, **_kwargs) -> None:
+ pass
+
+ async def initialize(self, *_args, **kwargs) -> bool:
+ kwargs["indexed_file_callback"]([str(indexed)])
+ return True
+
+ def _resolve_provider(self, _name: str) -> str:
+ return "llamaindex"
+
+ statuses: list[dict] = []
+ manager = type(
+ "Manager", (), {"update_kb_status": lambda _self, **kwargs: statuses.append(kwargs)}
+ )()
+ monkeypatch.setattr(add_module, "RAGService", PartialIndexService)
+
+ count = asyncio.run(
+ add_module._bootstrap_index_from_files(
+ "kb", [str(indexed), str(skipped)], str(tmp_path), manager
+ )
+ )
+
+ metadata = json.loads((tmp_path / "kb" / "metadata.json").read_text(encoding="utf-8"))
+ assert count == 1
+ assert set(metadata["file_hashes"]) == {"indexed.txt"}
+ assert metadata["last_indexed_count"] == 1
+ assert statuses[-1]["progress"]["indexed_count"] == 1
+
+
def test_process_new_documents_returns_failures_without_marking_processed(
monkeypatch, tmp_path: Path
) -> None: