Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): keep same-named documents when creating a KB

ZQR11018899ae6df693
diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.py
index 3832be829..cda8062c2 100644
--- a/deeptutor/knowledge/add_documents.py
+++ b/deeptutor/knowledge/add_documents.py
@@ -7,14 +7,12 @@ import argparse
import asyncio
from dataclasses import dataclass
from datetime import datetime
-import hashlib
-import itertools
import json
import logging
from pathlib import Path
-import shutil
from typing import TYPE_CHECKING, List, Optional
+from deeptutor.knowledge.staging import content_hash, stage_source
from deeptutor.services.config import resolve_llm_runtime_config
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.rag.factory import (
@@ -122,18 +120,10 @@ def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:
return file_path.name
-def _file_hash(file_path: Path) -> str:
- sha256_hash = hashlib.sha256()
- with open(file_path, "rb") as handle:
- for block in iter(lambda: handle.read(65536), b""):
- sha256_hash.update(block)
- return sha256_hash.hexdigest()
-
-
def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:
"""Hash source files already confirmed in an index, keyed by their raw path."""
return {
- _raw_hash_key(Path(file_path), raw_dir): _file_hash(Path(file_path))
+ _raw_hash_key(Path(file_path), raw_dir): content_hash(Path(file_path))
for file_path in file_paths
}
@@ -261,7 +251,7 @@ class DocumentAdder:
return resolve_bound_provider(self.base_dir, self.kb_name)
def _get_file_hash(self, file_path: Path) -> str:
- return _file_hash(file_path)
+ return content_hash(file_path)
def get_ingested_hashes(self) -> dict[str, str]:
if self.metadata_file.exists():
@@ -273,24 +263,6 @@ class DocumentAdder:
return {}
return {}
- @staticmethod
- def _non_colliding(dest_path: Path) -> Path:
- """A free sibling name for a *different* document that wants a taken one.
-
- Two files can legitimately share a name and hold different content — a
- docs tree with a README.md per folder is the ordinary case, and batched
- syncs of sibling folders stage them against different roots, so they
- land on the same name even with structure preserved. Dropping the later
- one loses a document the user asked to index, silently.
- """
- if not dest_path.exists():
- return dest_path
- for index in itertools.count(2):
- candidate = dest_path.with_name(f"{dest_path.stem} ({index}){dest_path.suffix}")
- if not candidate.exists():
- return candidate
- raise AssertionError("unreachable") # pragma: no cover
-
def add_documents(
self,
source_files: List[str],
@@ -337,20 +309,12 @@ class DocumentAdder:
else:
dest_path = self.raw_dir / source_path.name
- if dest_path.exists():
- dest_hash = self._get_file_hash(dest_path)
- if dest_hash == current_hash:
- logger.info(f"Recovering staged file: {source_path.name}")
- files_to_process.append(dest_path)
- continue
- # Same name, different document: keep both rather than
- # silently dropping this one.
- dest_path = self._non_colliding(dest_path)
-
- dest_path.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(source_path, dest_path)
- logger.info(f"Staged to raw: {dest_path.relative_to(self.raw_dir).as_posix()}")
- files_to_process.append(dest_path)
+ staged_path, copied = stage_source(source_path, dest_path, source_hash=current_hash)
+ if copied:
+ logger.info(f"Staged to raw: {staged_path.relative_to(self.raw_dir).as_posix()}")
+ else:
+ logger.info(f"Recovering staged file: {source_path.name}")
+ files_to_process.append(staged_path)
return files_to_process
diff --git a/deeptutor/knowledge/initializer.py b/deeptutor/knowledge/initializer.py
index 364bd2ba3..172bc2e11 100644
--- a/deeptutor/knowledge/initializer.py
+++ b/deeptutor/knowledge/initializer.py
@@ -9,7 +9,6 @@ from datetime import datetime
import json
import logging
from pathlib import Path
-import shutil
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
@@ -18,6 +17,7 @@ if TYPE_CHECKING:
from deeptutor.knowledge.naming import validate_knowledge_base_name
from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker
+from deeptutor.knowledge.staging import stage_source
from deeptutor.services.config import resolve_llm_runtime_config
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.rag.factory import normalize_provider_name
@@ -153,16 +153,27 @@ class KnowledgeBaseInitializer:
self._register_to_config()
def copy_documents(self, source_files: list[str]) -> list[str]:
- """Copy source documents into raw directory."""
+ """Stage source documents into the raw directory.
+
+ Sources arrive flattened to their basename, so a docs tree holding one
+ ``full.md`` per volume wants the same path over and over. Each document
+ is kept under a distinct name instead of overwriting the one before it
+ (#2041) — the same rule the incremental add path already followed.
+ """
copied_files: list[str] = []
+ staged: set[Path] = set()
for source in source_files:
source_path = Path(source)
if not source_path.exists() or not source_path.is_file():
logger.warning(f"Source file not found: {source}")
continue
- dest_path = self.raw_dir / source_path.name
- shutil.copy2(source_path, dest_path)
- copied_files.append(str(dest_path))
+ staged_path, copied = stage_source(source_path, self.raw_dir / source_path.name)
+ if staged_path in staged:
+ continue
+ staged.add(staged_path)
+ copied_files.append(str(staged_path))
+ if copied and staged_path.name != source_path.name:
+ logger.warning(f"Name taken by another document, staged as {staged_path}")
return copied_files
async def process_documents(
diff --git a/deeptutor/knowledge/staging.py b/deeptutor/knowledge/staging.py
new file mode 100644
index 000000000..f6d9bb499
--- /dev/null
+++ b/deeptutor/knowledge/staging.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""Shared staging rule for putting source documents into a KB's ``raw/`` tree.
+
+Both ways of filling a knowledge base — creating one from a docs directory, and
+adding documents to an existing one — flatten sources to their basename, so two
+documents that share a name want the same path. Only one place should answer
+what happens then, or the two entry points drift apart: the incremental path
+kept both files while the create path copied straight onto the taken name and
+indexed only the last one, silently losing the rest (#2041).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import itertools
+from pathlib import Path
+import shutil
+
+
+def content_hash(file_path: Path) -> str:
+ """SHA-256 of a file, streamed in blocks so large documents stay off memory."""
+ sha256_hash = hashlib.sha256()
+ with open(file_path, "rb") as handle:
+ for block in iter(lambda: handle.read(65536), b""):
+ sha256_hash.update(block)
+ return sha256_hash.hexdigest()
+
+
+def non_colliding(dest_path: Path) -> Path:
+ """A free sibling name for a *different* document that wants a taken one.
+
+ Two files can legitimately share a name and hold different content — a
+ docs tree with a README.md per folder is the ordinary case, and a book
+ series saved with one ``full.md`` per volume is the same shape. Dropping
+ the later one loses a document the user asked to index, silently.
+ """
+ if not dest_path.exists():
+ return dest_path
+ for index in itertools.count(2):
+ candidate = dest_path.with_name(f"{dest_path.stem} ({index}){dest_path.suffix}")
+ if not candidate.exists():
+ return candidate
+ raise AssertionError("unreachable") # pragma: no cover
+
+
+def stage_source(
+ source_path: Path,
+ dest_path: Path,
+ *,
+ source_hash: str | None = None,
+) -> tuple[Path, bool]:
+ """Copy *source_path* to *dest_path* without replacing a different document.
+
+ Returns the path the document actually landed on, and whether a copy was
+ made. A name already holding the *same* content is reused rather than
+ written twice; a name holding a different document is left alone and the
+ newcomer takes a free sibling name, so both get indexed.
+ """
+ if dest_path.exists():
+ current = source_hash or content_hash(source_path)
+ if current == content_hash(dest_path):
+ return dest_path, False
+ staged = non_colliding(dest_path)
+ staged.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(source_path, staged)
+ return staged, True
+
+
+__all__ = ["content_hash", "non_colliding", "stage_source"]
diff --git a/tests/knowledge/test_kb_document_staging.py b/tests/knowledge/test_kb_document_staging.py
new file mode 100644
index 000000000..ce567fd53
--- /dev/null
+++ b/tests/knowledge/test_kb_document_staging.py
@@ -0,0 +1,87 @@
+"""Staging sources into ``raw/`` must never drop a document the user asked for (#2041).
+
+Creating a KB flattened every source to its basename and copied straight onto
+the taken name, so a docs tree holding one ``full.md`` per volume indexed only
+the last file — while still reporting that the knowledge base was created. The
+incremental add path already kept both, so the rule now lives in one place and
+these tests pin the create path to it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from deeptutor.knowledge.initializer import KnowledgeBaseInitializer
+from deeptutor.knowledge.staging import stage_source
+
+
+def _initializer(tmp_path: Path) -> KnowledgeBaseInitializer:
+ initializer = KnowledgeBaseInitializer(kb_name="kb", base_dir=str(tmp_path))
+ initializer.raw_dir.mkdir(parents=True, exist_ok=True)
+ return initializer
+
+
+def test_copy_documents_keeps_same_named_documents(tmp_path: Path) -> None:
+ volumes = []
+ for index, volume in enumerate(("a", "b", "c")):
+ source = tmp_path / volume / "full.md"
+ source.parent.mkdir(parents=True)
+ source.write_text(f"volume {index}", encoding="utf-8")
+ volumes.append(str(source))
+
+ staged = _initializer(tmp_path).copy_documents(volumes)
+
+ raw_dir = tmp_path / "kb" / "raw"
+ assert len(staged) == 3
+ assert sorted(path.name for path in raw_dir.iterdir()) == sorted(
+ Path(path).name for path in staged
+ )
+ assert {Path(path).read_text(encoding="utf-8") for path in staged} == {
+ "volume 0",
+ "volume 1",
+ "volume 2",
+ }
+
+
+def test_copy_documents_stages_identical_document_once(tmp_path: Path) -> None:
+ first = tmp_path / "a" / "notes.md"
+ first.parent.mkdir(parents=True)
+ first.write_text("same text", encoding="utf-8")
+ second = tmp_path / "b" / "notes.md"
+ second.parent.mkdir(parents=True)
+ second.write_text("same text", encoding="utf-8")
+
+ staged = _initializer(tmp_path).copy_documents([str(first), str(second)])
+
+ assert [Path(path).name for path in staged] == ["notes.md"]
+ assert sorted(path.name for path in (tmp_path / "kb" / "raw").iterdir()) == ["notes.md"]
+
+
+def test_copy_documents_reports_missing_source_without_indexing_it(tmp_path: Path) -> None:
+ real = tmp_path / "real.md"
+ real.write_text("content", encoding="utf-8")
+
+ staged = _initializer(tmp_path).copy_documents([str(tmp_path / "gone.md"), str(real)])
+
+ assert [Path(path).name for path in staged] == ["real.md"]
+
+
+def test_stage_source_renames_only_when_content_differs(tmp_path: Path) -> None:
+ dest = tmp_path / "raw" / "src.md"
+ first = tmp_path / "first.md"
+ first.write_text("first", encoding="utf-8")
+
+ staged, copied = stage_source(first, dest)
+ assert copied is True and staged == dest
+
+ identical = tmp_path / "copy.md"
+ identical.write_text("first", encoding="utf-8")
+ reused, copied_again = stage_source(identical, dest)
+ assert copied_again is False and reused == dest
+
+ different = tmp_path / "other.md"
+ different.write_text("second", encoding="utf-8")
+ renamed, renamed_copied = stage_source(different, dest)
+ assert renamed_copied is True and renamed != dest
+ assert renamed.read_text(encoding="utf-8") == "second"
+ assert dest.read_text(encoding="utf-8") == "first"