DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): keep same-named documents when creating a KB
ZQR11018899ae6df693
diff --git a/deeptutor/knowledge/add_documents.py b/deeptutor/knowledge/add_documents.pyindex 3832be829..cda8062c2 100644--- a/deeptutor/knowledge/add_documents.py+++ b/deeptutor/knowledge/add_documents.py@@ -7,14 +7,12 @@ import argparseimport asynciofrom dataclasses import dataclassfrom datetime import datetime-import hashlib-import itertoolsimport jsonimport loggingfrom pathlib import Path-import shutilfrom typing import TYPE_CHECKING, List, Optional+from deeptutor.knowledge.staging import content_hash, stage_sourcefrom deeptutor.services.config import resolve_llm_runtime_configfrom deeptutor.services.file_io import atomic_write_jsonfrom deeptutor.services.rag.factory import (@@ -122,18 +120,10 @@ def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:return file_path.name-def _file_hash(file_path: Path) -> str:- sha256_hash = hashlib.sha256()- with open(file_path, "rb") as handle:- for block in iter(lambda: handle.read(65536), b""):- sha256_hash.update(block)- return sha256_hash.hexdigest()--def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:"""Hash source files already confirmed in an index, keyed by their raw path."""return {- _raw_hash_key(Path(file_path), raw_dir): _file_hash(Path(file_path))+ _raw_hash_key(Path(file_path), raw_dir): content_hash(Path(file_path))for file_path in file_paths}@@ -261,7 +251,7 @@ class DocumentAdder:return resolve_bound_provider(self.base_dir, self.kb_name)def _get_file_hash(self, file_path: Path) -> str:- return _file_hash(file_path)+ return content_hash(file_path)def get_ingested_hashes(self) -> dict[str, str]:if self.metadata_file.exists():@@ -273,24 +263,6 @@ class DocumentAdder:return {}return {}- @staticmethod- def _non_colliding(dest_path: Path) -> Path:- """A free sibling name for a *different* document that wants a taken one.-- Two files can legitimately share a name and hold different content — a- docs tree with a README.md per folder is the ordinary case, and batched- syncs of sibling folders stage them against different roots, so they- land on the same name even with structure preserved. Dropping the later- one loses a document the user asked to index, silently.- """- if not dest_path.exists():- return dest_path- for index in itertools.count(2):- candidate = dest_path.with_name(f"{dest_path.stem} ({index}){dest_path.suffix}")- if not candidate.exists():- return candidate- raise AssertionError("unreachable") # pragma: no cover-def add_documents(self,source_files: List[str],@@ -337,20 +309,12 @@ class DocumentAdder:else:dest_path = self.raw_dir / source_path.name- if dest_path.exists():- dest_hash = self._get_file_hash(dest_path)- if dest_hash == current_hash:- logger.info(f"Recovering staged file: {source_path.name}")- files_to_process.append(dest_path)- continue- # Same name, different document: keep both rather than- # silently dropping this one.- dest_path = self._non_colliding(dest_path)-- dest_path.parent.mkdir(parents=True, exist_ok=True)- shutil.copy2(source_path, dest_path)- logger.info(f"Staged to raw: {dest_path.relative_to(self.raw_dir).as_posix()}")- files_to_process.append(dest_path)+ staged_path, copied = stage_source(source_path, dest_path, source_hash=current_hash)+ if copied:+ logger.info(f"Staged to raw: {staged_path.relative_to(self.raw_dir).as_posix()}")+ else:+ logger.info(f"Recovering staged file: {source_path.name}")+ files_to_process.append(staged_path)return files_to_processdiff --git a/deeptutor/knowledge/initializer.py b/deeptutor/knowledge/initializer.pyindex 364bd2ba3..172bc2e11 100644--- a/deeptutor/knowledge/initializer.py+++ b/deeptutor/knowledge/initializer.py@@ -9,7 +9,6 @@ from datetime import datetimeimport jsonimport loggingfrom pathlib import Path-import shutilfrom typing import TYPE_CHECKING, Optionalif TYPE_CHECKING:@@ -18,6 +17,7 @@ if TYPE_CHECKING:from deeptutor.knowledge.naming import validate_knowledge_base_namefrom deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker+from deeptutor.knowledge.staging import stage_sourcefrom deeptutor.services.config import resolve_llm_runtime_configfrom deeptutor.services.file_io import atomic_write_jsonfrom deeptutor.services.rag.factory import normalize_provider_name@@ -153,16 +153,27 @@ class KnowledgeBaseInitializer:self._register_to_config()def copy_documents(self, source_files: list[str]) -> list[str]:- """Copy source documents into raw directory."""+ """Stage source documents into the raw directory.++ Sources arrive flattened to their basename, so a docs tree holding one+ ``full.md`` per volume wants the same path over and over. Each document+ is kept under a distinct name instead of overwriting the one before it+ (#2041) — the same rule the incremental add path already followed.+ """copied_files: list[str] = []+ staged: set[Path] = set()for source in source_files:source_path = Path(source)if not source_path.exists() or not source_path.is_file():logger.warning(f"Source file not found: {source}")continue- dest_path = self.raw_dir / source_path.name- shutil.copy2(source_path, dest_path)- copied_files.append(str(dest_path))+ staged_path, copied = stage_source(source_path, self.raw_dir / source_path.name)+ if staged_path in staged:+ continue+ staged.add(staged_path)+ copied_files.append(str(staged_path))+ if copied and staged_path.name != source_path.name:+ logger.warning(f"Name taken by another document, staged as {staged_path}")return copied_filesasync def process_documents(diff --git a/deeptutor/knowledge/staging.py b/deeptutor/knowledge/staging.pynew file mode 100644index 000000000..f6d9bb499--- /dev/null+++ b/deeptutor/knowledge/staging.py@@ -0,0 +1,69 @@+#!/usr/bin/env python+"""Shared staging rule for putting source documents into a KB's ``raw/`` tree.++Both ways of filling a knowledge base — creating one from a docs directory, and+adding documents to an existing one — flatten sources to their basename, so two+documents that share a name want the same path. Only one place should answer+what happens then, or the two entry points drift apart: the incremental path+kept both files while the create path copied straight onto the taken name and+indexed only the last one, silently losing the rest (#2041).+"""++from __future__ import annotations++import hashlib+import itertools+from pathlib import Path+import shutil+++def content_hash(file_path: Path) -> str:+ """SHA-256 of a file, streamed in blocks so large documents stay off memory."""+ sha256_hash = hashlib.sha256()+ with open(file_path, "rb") as handle:+ for block in iter(lambda: handle.read(65536), b""):+ sha256_hash.update(block)+ return sha256_hash.hexdigest()+++def non_colliding(dest_path: Path) -> Path:+ """A free sibling name for a *different* document that wants a taken one.++ Two files can legitimately share a name and hold different content — a+ docs tree with a README.md per folder is the ordinary case, and a book+ series saved with one ``full.md`` per volume is the same shape. Dropping+ the later one loses a document the user asked to index, silently.+ """+ if not dest_path.exists():+ return dest_path+ for index in itertools.count(2):+ candidate = dest_path.with_name(f"{dest_path.stem} ({index}){dest_path.suffix}")+ if not candidate.exists():+ return candidate+ raise AssertionError("unreachable") # pragma: no cover+++def stage_source(+ source_path: Path,+ dest_path: Path,+ *,+ source_hash: str | None = None,+) -> tuple[Path, bool]:+ """Copy *source_path* to *dest_path* without replacing a different document.++ Returns the path the document actually landed on, and whether a copy was+ made. A name already holding the *same* content is reused rather than+ written twice; a name holding a different document is left alone and the+ newcomer takes a free sibling name, so both get indexed.+ """+ if dest_path.exists():+ current = source_hash or content_hash(source_path)+ if current == content_hash(dest_path):+ return dest_path, False+ staged = non_colliding(dest_path)+ staged.parent.mkdir(parents=True, exist_ok=True)+ shutil.copy2(source_path, staged)+ return staged, True+++__all__ = ["content_hash", "non_colliding", "stage_source"]diff --git a/tests/knowledge/test_kb_document_staging.py b/tests/knowledge/test_kb_document_staging.pynew file mode 100644index 000000000..ce567fd53--- /dev/null+++ b/tests/knowledge/test_kb_document_staging.py@@ -0,0 +1,87 @@+"""Staging sources into ``raw/`` must never drop a document the user asked for (#2041).++Creating a KB flattened every source to its basename and copied straight onto+the taken name, so a docs tree holding one ``full.md`` per volume indexed only+the last file — while still reporting that the knowledge base was created. The+incremental add path already kept both, so the rule now lives in one place and+these tests pin the create path to it.+"""++from __future__ import annotations++from pathlib import Path++from deeptutor.knowledge.initializer import KnowledgeBaseInitializer+from deeptutor.knowledge.staging import stage_source+++def _initializer(tmp_path: Path) -> KnowledgeBaseInitializer:+ initializer = KnowledgeBaseInitializer(kb_name="kb", base_dir=str(tmp_path))+ initializer.raw_dir.mkdir(parents=True, exist_ok=True)+ return initializer+++def test_copy_documents_keeps_same_named_documents(tmp_path: Path) -> None:+ volumes = []+ for index, volume in enumerate(("a", "b", "c")):+ source = tmp_path / volume / "full.md"+ source.parent.mkdir(parents=True)+ source.write_text(f"volume {index}", encoding="utf-8")+ volumes.append(str(source))++ staged = _initializer(tmp_path).copy_documents(volumes)++ raw_dir = tmp_path / "kb" / "raw"+ assert len(staged) == 3+ assert sorted(path.name for path in raw_dir.iterdir()) == sorted(+ Path(path).name for path in staged+ )+ assert {Path(path).read_text(encoding="utf-8") for path in staged} == {+ "volume 0",+ "volume 1",+ "volume 2",+ }+++def test_copy_documents_stages_identical_document_once(tmp_path: Path) -> None:+ first = tmp_path / "a" / "notes.md"+ first.parent.mkdir(parents=True)+ first.write_text("same text", encoding="utf-8")+ second = tmp_path / "b" / "notes.md"+ second.parent.mkdir(parents=True)+ second.write_text("same text", encoding="utf-8")++ staged = _initializer(tmp_path).copy_documents([str(first), str(second)])++ assert [Path(path).name for path in staged] == ["notes.md"]+ assert sorted(path.name for path in (tmp_path / "kb" / "raw").iterdir()) == ["notes.md"]+++def test_copy_documents_reports_missing_source_without_indexing_it(tmp_path: Path) -> None:+ real = tmp_path / "real.md"+ real.write_text("content", encoding="utf-8")++ staged = _initializer(tmp_path).copy_documents([str(tmp_path / "gone.md"), str(real)])++ assert [Path(path).name for path in staged] == ["real.md"]+++def test_stage_source_renames_only_when_content_differs(tmp_path: Path) -> None:+ dest = tmp_path / "raw" / "src.md"+ first = tmp_path / "first.md"+ first.write_text("first", encoding="utf-8")++ staged, copied = stage_source(first, dest)+ assert copied is True and staged == dest++ identical = tmp_path / "copy.md"+ identical.write_text("first", encoding="utf-8")+ reused, copied_again = stage_source(identical, dest)+ assert copied_again is False and reused == dest++ different = tmp_path / "other.md"+ different.write_text("second", encoding="utf-8")+ renamed, renamed_copied = stage_source(different, dest)+ assert renamed_copied is True and renamed != dest+ assert renamed.read_text(encoding="utf-8") == "second"+ assert dest.read_text(encoding="utf-8") == "first"