DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
#!/usr/bin/env python
"""Incrementally add documents to an existing knowledge base."""
from __future__ import annotations
import argparse
import asyncio
from dataclasses import dataclass
from datetime import datetime
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, List, Optional
from deeptutor.knowledge.staging import content_hash, stage_source
from deeptutor.services.config import resolve_llm_runtime_config
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.rag.factory import (
DEFAULT_PROVIDER,
LIGHTRAG_PROVIDER,
has_ready_provider_index,
normalize_provider_name,
)
from deeptutor.services.rag.index_probe import provider_failure_summary
from deeptutor.services.rag.index_versioning import list_kb_versions
from deeptutor.services.rag.provider_binding import resolve_bound_provider
from deeptutor.services.rag.service import RAGService
from deeptutor.services.setup.data_volume import (
DataVolumePermissionError,
ensure_data_volume_writable,
format_data_volume_permission_error,
)
from deeptutor.utils.file_types import FileTypeRouter
logger = logging.getLogger(__name__)
DEFAULT_BASE_DIR = "./data/knowledge_bases"
if TYPE_CHECKING:
from deeptutor.knowledge.manager import KnowledgeBaseManager
@dataclass(frozen=True)
class DocumentIndexFailure:
"""One file that could not be added to the provider index."""
file_path: Path
error: str
@dataclass(frozen=True)
class DocumentIndexResult:
"""Structured incremental-index result.
A task can no longer infer success from "no exception": every staged file is
explicitly accounted for as either processed or failed.
"""
processed_files: list[Path]
failures: list[DocumentIndexFailure]
@property
def processed_count(self) -> int:
return len(self.processed_files)
@property
def failed_count(self) -> int:
return len(self.failures)
@property
def has_failures(self) -> bool:
return bool(self.failures)
def failure_summary(self, *, limit: int = 3) -> str:
if not self.failures:
return ""
shown = [f"{failure.file_path.name}: {failure.error}" for failure in self.failures[:limit]]
remaining = len(self.failures) - len(shown)
if remaining > 0:
shown.append(f"... and {remaining} more file(s)")
return "; ".join(shown)
@dataclass(frozen=True)
class RawDocumentRemoval:
"""Outcome of removing one raw document from a knowledge base."""
rel_path: str
was_indexed: bool
def _read_metadata(metadata_file: Path) -> dict:
"""Load a KB's metadata.json, returning {} when absent or unreadable."""
if not metadata_file.exists():
return {}
try:
with open(metadata_file, "r", encoding="utf-8") as f:
data = json.load(f)
except Exception:
return {}
return data if isinstance(data, dict) else {}
def _write_metadata(metadata_file: Path, metadata: dict) -> None:
"""Persist a KB's metadata.json atomically (pretty-printed, non-ASCII preserved)."""
atomic_write_json(metadata_file, metadata)
def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:
"""Stable ``file_hashes`` key for a staged file.
The key is the POSIX path relative to ``raw/`` (so folder uploads keep
distinct entries), falling back to the basename for anything that resolves
outside ``raw/``. Indexing and removal MUST share this rule, otherwise a
removed file's hash record would leak and silently skip a later re-add.
"""
try:
return file_path.resolve().relative_to(raw_dir.resolve()).as_posix()
except ValueError:
return file_path.name
def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:
"""Hash source files already confirmed in an index, keyed by their raw path."""
return {
_raw_hash_key(Path(file_path), raw_dir): content_hash(Path(file_path))
for file_path in file_paths
}
def remove_raw_document(kb_dir: Path, file_path: Path) -> RawDocumentRemoval:
"""Delete one staged raw file and drop its indexed-hash record.
Deliberately decoupled from :class:`DocumentAdder` (whose constructor
requires a ready provider index): a KB stuck in an *error* state often has
no index at all, yet its raw/ files must still be removable so the user can
drop an unparseable document instead of deleting and rebuilding the whole
KB. Vectors are not touched here — the providers index whole KBs, so any
vectors from an already-indexed file are cleared by a subsequent re-index,
which the returned ``was_indexed`` flag lets the caller surface.
``file_path`` must already be sandbox-resolved under the KB's raw/ dir.
"""
raw_dir = kb_dir / "raw"
hash_key = _raw_hash_key(file_path, raw_dir)
target = file_path.resolve()
if target.exists():
target.unlink()
metadata_file = kb_dir / "metadata.json"
metadata = _read_metadata(metadata_file)
hashes = metadata.get("file_hashes")
was_indexed = isinstance(hashes, dict) and hash_key in hashes
if was_indexed:
del hashes[hash_key]
_write_metadata(metadata_file, metadata)
return RawDocumentRemoval(rel_path=hash_key, was_indexed=was_indexed)
class DocumentAdder:
"""Stage and index new files through a KB's bound RAG provider."""
def __init__(
self,
kb_name: str,
base_dir: str = DEFAULT_BASE_DIR,
api_key: str | None = None,
base_url: str | None = None,
progress_tracker=None,
rag_provider: str | None = None,
accepted_indexing_snapshot=None,
):
self.kb_name = kb_name
self.base_dir = Path(base_dir)
self.kb_dir = self.base_dir / kb_name
if not self.kb_dir.exists():
raise ValueError(f"Knowledge base does not exist: {kb_name}")
self.raw_dir = self.kb_dir / "raw"
self.llamaindex_storage_dir = self.kb_dir / "llamaindex_storage"
self.legacy_rag_storage_dir = self.kb_dir / "rag_storage"
self.metadata_file = self.kb_dir / "metadata.json"
# Fail on UID-mismatched / unwritable volumes before the "not initialized"
# check, which otherwise hides Unraid bind-mount permission errors.
ensure_data_volume_writable(self.kb_dir)
# Incremental adds must use the engine DeepTutor has bound to this KB. An
# explicit rag_provider (from the API, already matched against the KB)
# wins; otherwise use the shared binding resolver.
if rag_provider:
self.rag_provider = normalize_provider_name(rag_provider)
else:
self.rag_provider = self._provider_from_binding()
has_provider_index = has_ready_provider_index(self.kb_dir, self.rag_provider)
if (
self.rag_provider == DEFAULT_PROVIDER
and not has_provider_index
and self.legacy_rag_storage_dir.exists()
):
raise ValueError(
f"Knowledge base '{kb_name}' uses legacy index format and requires reindex before incremental add"
)
# Both pipelines create their first index on add; existing broken versions
# still require reindex instead of being silently replaced (#1458).
versions = list_kb_versions(self.kb_dir)
allows_bootstrap = (
self.rag_provider
in {
DEFAULT_PROVIDER,
LIGHTRAG_PROVIDER,
}
and not versions
)
if not has_provider_index and not allows_bootstrap:
if versions:
summary = provider_failure_summary(self.kb_dir, self.rag_provider)
raise ValueError(
f"Knowledge base has no ready {self.rag_provider} index; reindex required: "
f"{summary or 'stored index version is incomplete'}"
)
raise ValueError(f"Knowledge base not initialized ({self.rag_provider}): {kb_name}")
self.accepted_indexing_snapshot = accepted_indexing_snapshot
if self.rag_provider == LIGHTRAG_PROVIDER and self.accepted_indexing_snapshot is None:
from deeptutor.services.rag.pipelines.lightrag.indexing_policy import (
bind_target,
resolve_write_snapshot,
)
self.accepted_indexing_snapshot = bind_target(
resolve_write_snapshot(
self.kb_dir, base_dir=str(self.base_dir), kb_name=self.kb_name
),
self.kb_dir,
)
self.api_key = api_key
self.base_url = base_url
self.progress_tracker = progress_tracker
self.raw_dir.mkdir(parents=True, exist_ok=True)
def _provider_from_binding(self) -> str:
return resolve_bound_provider(self.base_dir, self.kb_name)
def _get_file_hash(self, file_path: Path) -> str:
return content_hash(file_path)
def get_ingested_hashes(self) -> dict[str, str]:
if self.metadata_file.exists():
try:
with open(self.metadata_file, "r", encoding="utf-8") as f:
data = json.load(f)
return data.get("file_hashes", {})
except Exception:
return {}
return {}
def add_documents(
self,
source_files: List[str],
allow_duplicates: bool = False,
source_root: str | Path | None = None,
) -> List[Path]:
"""Validate and stage files into raw/ before indexing.
``source_root``, when given, is the root a caller is syncing from
(e.g. a linked folder). Any source file found under it is staged at
the same path *relative to that root*, instead of being flattened to
its bare filename: without this, every externally-sourced subfolder
collapses into raw/'s top level and same-named files in different
subfolders collide.
"""
logger.info(f"Validating documents for '{self.kb_name}'...")
ingested_hashes = self.get_ingested_hashes()
files_to_process: list[Path] = []
resolved_source_root = Path(source_root).resolve() if source_root else None
for source in source_files:
source_path = Path(source)
if not source_path.exists() or not source_path.is_file():
logger.warning(f"Missing file: {source}")
continue
current_hash = self._get_file_hash(source_path)
if current_hash in ingested_hashes.values() and not allow_duplicates:
logger.info(f"Skipped (content already indexed): {source_path.name}")
continue
resolved_source = source_path.resolve()
# Files already saved under raw/ (e.g. by the upload route, possibly
# inside a folder) are indexed in place — never flattened to the
# basename — so the uploaded folder structure is preserved verbatim.
if resolved_source.is_relative_to(self.raw_dir.resolve()):
files_to_process.append(source_path)
continue
if resolved_source_root and resolved_source.is_relative_to(resolved_source_root):
dest_path = self.raw_dir / resolved_source.relative_to(resolved_source_root)
else:
dest_path = self.raw_dir / source_path.name
staged_path, copied = stage_source(source_path, dest_path, source_hash=current_hash)
if copied:
logger.info(f"Staged to raw: {staged_path.relative_to(self.raw_dir).as_posix()}")
else:
logger.info(f"Recovering staged file: {source_path.name}")
files_to_process.append(staged_path)
return files_to_process
async def process_new_documents(self, new_files: List[Path]) -> DocumentIndexResult:
"""Index staged files via the KB's bound provider."""
if not new_files:
return DocumentIndexResult(processed_files=[], failures=[])
if self.rag_provider == LIGHTRAG_PROVIDER:
return await self._process_lightrag_batch(new_files)
rag_service = RAGService(kb_base_dir=str(self.base_dir), provider=self.rag_provider)
processed_files: list[Path] = []
failures: list[DocumentIndexFailure] = []
total_files = len(new_files)
for idx, doc_file in enumerate(new_files, 1):
try:
if self.progress_tracker is not None:
from deeptutor.knowledge.progress_tracker import ProgressStage
self.progress_tracker.update(
ProgressStage.PROCESSING_FILE,
message_key="Indexing {{name}}",
message_params={"name": doc_file.name},
current=len(processed_files),
total=total_files,
)
success = await rag_service.add_documents(self.kb_name, [str(doc_file)])
if success:
processed_files.append(doc_file)
# Re-hashes the whole file, so it stays off the event loop:
# this runs inside a FastAPI BackgroundTasks entry, which is
# awaited on the loop and would otherwise stall unrelated
# requests once per indexed file (#777).
try:
await asyncio.to_thread(self._record_successful_hash, doc_file)
except Exception:
if self.rag_provider != LIGHTRAG_PROVIDER:
raise
logger.warning(
"LightRAG published '%s' before its duplicate-detection hash "
"was recorded",
doc_file.name,
exc_info=True,
)
logger.info(f"Processed: {doc_file.name}")
if self.progress_tracker is not None:
self.progress_tracker.update(
ProgressStage.PROCESSING_FILE,
message_key="Indexed {{name}}",
message_params={"name": doc_file.name},
current=len(processed_files),
total=total_files,
)
else:
error = "Provider returned failure without details."
failures.append(DocumentIndexFailure(doc_file, error))
logger.error(f"Failed to index: {doc_file.name}")
except PermissionError as e:
logger.exception("Permission denied while indexing %s: %s", doc_file.name, e)
raise DataVolumePermissionError(
format_data_volume_permission_error(self.kb_dir, cause=e)
) from e
except Exception as e:
logger.exception(f"Failed {doc_file.name}: {e}")
failures.append(DocumentIndexFailure(doc_file, str(e)))
return DocumentIndexResult(processed_files=processed_files, failures=failures)
async def _process_lightrag_batch(self, files: List[Path]) -> DocumentIndexResult:
"""Hold native write ownership across the entire accepted batch."""
from deeptutor.services.rag.pipelines.lightrag.pipeline import LightRagBatchError
def prepare_publication(root: Path) -> None:
from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker
tracker = self.progress_tracker or ProgressTracker(self.kb_name, self.base_dir)
tracker.update(
ProgressStage.COMPLETED,
message="Document indexing complete",
current=len(files),
total=len(files),
indexed_count=len(files),
index_changed=True,
index_action="upload",
publication_version=root.name,
)
tracker.verify_terminal(
current=len(files),
total=len(files),
indexed_count=len(files),
index_action="upload",
publication_version=root.name,
)
service = RAGService(kb_base_dir=str(self.base_dir), provider=LIGHTRAG_PROVIDER)
try:
success = await service.add_documents(
self.kb_name,
[str(path) for path in files],
accepted_indexing_snapshot=self.accepted_indexing_snapshot,
before_publish=prepare_publication,
)
if not success:
raise RuntimeError("LightRAG returned failure without details.")
processed = files
failures = []
except LightRagBatchError as exc:
processed = [path for path in files if path.name in exc.outcome.processed]
failures = [
DocumentIndexFailure(path, str(exc)) for path in files if path not in processed
]
except Exception as exc:
return DocumentIndexResult([], [DocumentIndexFailure(path, str(exc)) for path in files])
for path in processed:
try:
await asyncio.to_thread(self._record_successful_hash, path)
except Exception:
logger.warning(
"Indexed document hash could not be recorded: %s", path.name, exc_info=True
)
return DocumentIndexResult(processed, failures)
def _record_successful_hash(self, file_path: Path) -> None:
file_hash = self._get_file_hash(file_path)
metadata = _read_metadata(self.metadata_file)
hash_key = _raw_hash_key(file_path, self.raw_dir)
metadata.setdefault("file_hashes", {})[hash_key] = file_hash
_write_metadata(self.metadata_file, metadata)
def update_metadata(self, added_count: int) -> None:
"""Update metadata after incremental add."""
metadata: dict = {}
if self.metadata_file.exists():
try:
with open(self.metadata_file, "r", encoding="utf-8") as f:
metadata = json.load(f)
except Exception:
metadata = {}
metadata["rag_provider"] = self.rag_provider
metadata["needs_reindex"] = False
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
metadata["last_updated"] = timestamp
if added_count > 0:
metadata["last_indexed_at"] = timestamp
metadata["last_indexed_count"] = added_count
metadata["last_indexed_action"] = "upload"
history = metadata.get("update_history", [])
history.append(
{
"timestamp": metadata["last_updated"],
"action": "incremental_add",
"count": added_count,
"provider": self.rag_provider,
}
)
metadata["update_history"] = history
_write_metadata(self.metadata_file, metadata)
async def _bootstrap_index_from_files(
kb_name: str,
source_files: list[str],
base_dir: str,
manager: "KnowledgeBaseManager",
) -> int:
"""Create a fresh index for an empty KB from the given source files.
Called when :class:`DocumentAdder` rejects an add because the KB has no
existing index (it was created empty, e.g. via the no-files fast path or
a web/GitHub source sync before any documents were indexed). Uses
:meth:`RAGService.initialize` to build the index in one batch, then
records file hashes so subsequent incremental adds can detect duplicates.
"""
rag_service = RAGService(kb_base_dir=base_dir)
kb_dir = Path(base_dir) / kb_name
raw_dir = kb_dir / "raw"
metadata_file = kb_dir / "metadata.json"
requested = {str(Path(path).resolve()): str(Path(path)) for path in source_files}
confirmed: set[str] = set()
def on_indexed_file(paths: list[str]) -> None:
for path in paths:
canonical = str(Path(path).resolve())
if canonical in requested:
confirmed.add(canonical)
success = await rag_service.initialize(
kb_name=kb_name,
file_paths=source_files,
indexed_file_callback=on_indexed_file,
)
if not success:
raise RuntimeError(
f"Failed to initialize index for KB '{kb_name}' from {len(source_files)} file(s)"
)
indexed_files = [requested[path] for path in sorted(confirmed)]
indexed = len(indexed_files)
provider = rag_service._resolve_provider(kb_name)
try:
# Record hashes so future syncs detect unchanged files.
metadata = _read_metadata(metadata_file)
metadata.setdefault("file_hashes", {}).update(
hashes_for_indexed_files(indexed_files, raw_dir)
)
metadata["rag_provider"] = provider
metadata["needs_reindex"] = False
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
metadata["last_updated"] = ts
metadata["last_indexed_at"] = ts
metadata["last_indexed_count"] = indexed
metadata["last_indexed_action"] = "create"
_write_metadata(metadata_file, metadata)
manager.update_kb_status(
name=kb_name,
status="ready",
progress={
"stage": "completed",
"message": f"Initialized index with {indexed} file(s).",
"percent": 100,
"current": indexed,
"total": max(indexed, 1),
"file_name": "",
"error": None,
"timestamp": datetime.now().isoformat(),
"indexed_count": indexed,
"index_changed": True,
"index_action": "create",
},
)
except Exception:
if provider != LIGHTRAG_PROVIDER:
raise
logger.warning(
"LightRAG bootstrap for '%s' published before bookkeeping failed",
kb_name,
exc_info=True,
)
logger.info("Bootstrapped index for empty KB '%s' with %d file(s)", kb_name, indexed)
return indexed
async def add_documents(
kb_name: str,
source_files: list[str],
base_dir: str = DEFAULT_BASE_DIR,
api_key: Optional[str] = None,
base_url: Optional[str] = None,
allow_duplicates: bool = False,
accepted_indexing_snapshot=None,
) -> int:
"""Convenience function used by CLI wrappers."""
from deeptutor.knowledge.manager import KnowledgeBaseManager
manager = KnowledgeBaseManager(base_dir=base_dir)
published_count = 0
try:
manager.update_kb_status(
name=kb_name,
status="processing",
progress={
"stage": "processing_documents",
"message": "Processing uploaded documents...",
"percent": 0,
"current": 0,
"total": max(len(source_files), 1),
"file_name": "",
"error": None,
"timestamp": datetime.now().isoformat(),
},
)
try:
adder = DocumentAdder(
kb_name=kb_name,
base_dir=base_dir,
api_key=api_key,
base_url=base_url,
accepted_indexing_snapshot=accepted_indexing_snapshot,
)
except ValueError as exc:
if "not initialized" not in str(exc).lower() or not source_files:
raise
# Empty KB (created without initial documents): bootstrap the
# index from these files instead of erroring.
return await _bootstrap_index_from_files(
kb_name=kb_name,
source_files=source_files,
base_dir=base_dir,
manager=manager,
)
new_files = adder.add_documents(source_files, allow_duplicates=allow_duplicates)
if not new_files:
manager.update_kb_status(
name=kb_name,
status="ready",
progress={
"stage": "completed",
"message": "No new unique documents to process.",
"percent": 100,
"current": 1,
"total": 1,
"file_name": "",
"error": None,
"timestamp": datetime.now().isoformat(),
},
)
return 0
result = await adder.process_new_documents(new_files)
if result.has_failures:
raise RuntimeError(
f"Failed to index {result.failed_count}/{len(new_files)} file(s): "
f"{result.failure_summary()}"
)
if adder.rag_provider == LIGHTRAG_PROVIDER:
published_count = result.processed_count
adder.update_metadata(result.processed_count)
manager.update_kb_status(
name=kb_name,
status="ready",
progress={
"stage": "completed",
"message": f"Successfully processed {result.processed_count} files!",
"percent": 100,
"current": result.processed_count,
"total": max(len(new_files), 1),
"file_name": "",
"error": None,
"timestamp": datetime.now().isoformat(),
"indexed_count": result.processed_count,
"index_changed": result.processed_count > 0,
"index_action": "upload",
},
)
return result.processed_count
except Exception as exc:
if published_count:
logger.warning(
"LightRAG append for '%s' published before bookkeeping failed",
kb_name,
exc_info=True,
)
return published_count
manager.update_kb_status(
name=kb_name,
status="error",
progress={
"stage": "error",
"message": "Document upload failed",
"percent": 0,
"current": 0,
"total": max(len(source_files), 1),
"file_name": "",
"error": str(exc),
"timestamp": datetime.now().isoformat(),
},
)
raise
async def main() -> None:
try:
llm_config = resolve_llm_runtime_config()
default_api_key = llm_config.api_key
default_base_url = llm_config.effective_url
except Exception:
default_api_key = ""
default_base_url = ""
parser = argparse.ArgumentParser(description="Incrementally add documents to a KB")
parser.add_argument("kb_name", help="KB Name")
parser.add_argument("--docs", nargs="+", help="Files")
parser.add_argument("--docs-dir", help="Directory")
parser.add_argument("--base-dir", default=DEFAULT_BASE_DIR)
parser.add_argument("--api-key", default=default_api_key)
parser.add_argument("--base-url", default=default_base_url)
parser.add_argument("--allow-duplicates", action="store_true")
args = parser.parse_args()
doc_files: list[str] = []
if args.docs:
doc_files.extend(args.docs)
if args.docs_dir:
p = Path(args.docs_dir)
doc_files.extend(str(f) for f in FileTypeRouter.collect_supported_files(p))
if not doc_files:
logger.error("No documents provided.")
return
processed_count = await add_documents(
kb_name=args.kb_name,
source_files=doc_files,
base_dir=args.base_dir,
api_key=args.api_key,
base_url=args.base_url,
allow_duplicates=args.allow_duplicates,
)
if processed_count:
logger.info(f"Done! Successfully added {processed_count} documents.")
else:
logger.info("No new unique documents to add.")
if __name__ == "__main__":
asyncio.run(main())