Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

27.6 KB
Raw Blame
#!/usr/bin/env python
"""Incrementally add documents to an existing knowledge base."""

from __future__ import annotations

import argparse
import asyncio
from dataclasses import dataclass
from datetime import datetime
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, List, Optional

from deeptutor.knowledge.staging import content_hash, stage_source
from deeptutor.services.config import resolve_llm_runtime_config
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.rag.factory import (
    DEFAULT_PROVIDER,
    LIGHTRAG_PROVIDER,
    has_ready_provider_index,
    normalize_provider_name,
)
from deeptutor.services.rag.index_probe import provider_failure_summary
from deeptutor.services.rag.index_versioning import list_kb_versions
from deeptutor.services.rag.provider_binding import resolve_bound_provider
from deeptutor.services.rag.service import RAGService
from deeptutor.services.setup.data_volume import (
    DataVolumePermissionError,
    ensure_data_volume_writable,
    format_data_volume_permission_error,
)
from deeptutor.utils.file_types import FileTypeRouter

logger = logging.getLogger(__name__)

DEFAULT_BASE_DIR = "./data/knowledge_bases"

if TYPE_CHECKING:
    from deeptutor.knowledge.manager import KnowledgeBaseManager


@dataclass(frozen=True)
class DocumentIndexFailure:
    """One file that could not be added to the provider index."""

    file_path: Path
    error: str


@dataclass(frozen=True)
class DocumentIndexResult:
    """Structured incremental-index result.

    A task can no longer infer success from "no exception": every staged file is
    explicitly accounted for as either processed or failed.
    """

    processed_files: list[Path]
    failures: list[DocumentIndexFailure]

    @property
    def processed_count(self) -> int:
        return len(self.processed_files)

    @property
    def failed_count(self) -> int:
        return len(self.failures)

    @property
    def has_failures(self) -> bool:
        return bool(self.failures)

    def failure_summary(self, *, limit: int = 3) -> str:
        if not self.failures:
            return ""
        shown = [f"{failure.file_path.name}: {failure.error}" for failure in self.failures[:limit]]
        remaining = len(self.failures) - len(shown)
        if remaining > 0:
            shown.append(f"... and {remaining} more file(s)")
        return "; ".join(shown)


@dataclass(frozen=True)
class RawDocumentRemoval:
    """Outcome of removing one raw document from a knowledge base."""

    rel_path: str
    was_indexed: bool


def _read_metadata(metadata_file: Path) -> dict:
    """Load a KB's metadata.json, returning {} when absent or unreadable."""
    if not metadata_file.exists():
        return {}
    try:
        with open(metadata_file, "r", encoding="utf-8") as f:
            data = json.load(f)
    except Exception:
        return {}
    return data if isinstance(data, dict) else {}


def _write_metadata(metadata_file: Path, metadata: dict) -> None:
    """Persist a KB's metadata.json atomically (pretty-printed, non-ASCII preserved)."""
    atomic_write_json(metadata_file, metadata)


def _raw_hash_key(file_path: Path, raw_dir: Path) -> str:
    """Stable ``file_hashes`` key for a staged file.

    The key is the POSIX path relative to ``raw/`` (so folder uploads keep
    distinct entries), falling back to the basename for anything that resolves
    outside ``raw/``. Indexing and removal MUST share this rule, otherwise a
    removed file's hash record would leak and silently skip a later re-add.
    """
    try:
        return file_path.resolve().relative_to(raw_dir.resolve()).as_posix()
    except ValueError:
        return file_path.name


def hashes_for_indexed_files(file_paths: list[str], raw_dir: Path) -> dict[str, str]:
    """Hash source files already confirmed in an index, keyed by their raw path."""
    return {
        _raw_hash_key(Path(file_path), raw_dir): content_hash(Path(file_path))
        for file_path in file_paths
    }


def remove_raw_document(kb_dir: Path, file_path: Path) -> RawDocumentRemoval:
    """Delete one staged raw file and drop its indexed-hash record.

    Deliberately decoupled from :class:`DocumentAdder` (whose constructor
    requires a ready provider index): a KB stuck in an *error* state often has
    no index at all, yet its raw/ files must still be removable so the user can
    drop an unparseable document instead of deleting and rebuilding the whole
    KB. Vectors are not touched here — the providers index whole KBs, so any
    vectors from an already-indexed file are cleared by a subsequent re-index,
    which the returned ``was_indexed`` flag lets the caller surface.

    ``file_path`` must already be sandbox-resolved under the KB's raw/ dir.
    """
    raw_dir = kb_dir / "raw"
    hash_key = _raw_hash_key(file_path, raw_dir)

    target = file_path.resolve()
    if target.exists():
        target.unlink()

    metadata_file = kb_dir / "metadata.json"
    metadata = _read_metadata(metadata_file)
    hashes = metadata.get("file_hashes")
    was_indexed = isinstance(hashes, dict) and hash_key in hashes
    if was_indexed:
        del hashes[hash_key]
        _write_metadata(metadata_file, metadata)

    return RawDocumentRemoval(rel_path=hash_key, was_indexed=was_indexed)


class DocumentAdder:
    """Stage and index new files through a KB's bound RAG provider."""

    def __init__(
        self,
        kb_name: str,
        base_dir: str = DEFAULT_BASE_DIR,
        api_key: str | None = None,
        base_url: str | None = None,
        progress_tracker=None,
        rag_provider: str | None = None,
        accepted_indexing_snapshot=None,
    ):
        self.kb_name = kb_name
        self.base_dir = Path(base_dir)
        self.kb_dir = self.base_dir / kb_name

        if not self.kb_dir.exists():
            raise ValueError(f"Knowledge base does not exist: {kb_name}")

        self.raw_dir = self.kb_dir / "raw"
        self.llamaindex_storage_dir = self.kb_dir / "llamaindex_storage"
        self.legacy_rag_storage_dir = self.kb_dir / "rag_storage"
        self.metadata_file = self.kb_dir / "metadata.json"

        # Fail on UID-mismatched / unwritable volumes before the "not initialized"
        # check, which otherwise hides Unraid bind-mount permission errors.
        ensure_data_volume_writable(self.kb_dir)

        # Incremental adds must use the engine DeepTutor has bound to this KB. An
        # explicit rag_provider (from the API, already matched against the KB)
        # wins; otherwise use the shared binding resolver.
        if rag_provider:
            self.rag_provider = normalize_provider_name(rag_provider)
        else:
            self.rag_provider = self._provider_from_binding()

        has_provider_index = has_ready_provider_index(self.kb_dir, self.rag_provider)

        if (
            self.rag_provider == DEFAULT_PROVIDER
            and not has_provider_index
            and self.legacy_rag_storage_dir.exists()
        ):
            raise ValueError(
                f"Knowledge base '{kb_name}' uses legacy index format and requires reindex before incremental add"
            )

        # Both pipelines create their first index on add; existing broken versions
        # still require reindex instead of being silently replaced (#1458).
        versions = list_kb_versions(self.kb_dir)
        allows_bootstrap = (
            self.rag_provider
            in {
                DEFAULT_PROVIDER,
                LIGHTRAG_PROVIDER,
            }
            and not versions
        )
        if not has_provider_index and not allows_bootstrap:
            if versions:
                summary = provider_failure_summary(self.kb_dir, self.rag_provider)
                raise ValueError(
                    f"Knowledge base has no ready {self.rag_provider} index; reindex required: "
                    f"{summary or 'stored index version is incomplete'}"
                )
            raise ValueError(f"Knowledge base not initialized ({self.rag_provider}): {kb_name}")

        self.accepted_indexing_snapshot = accepted_indexing_snapshot
        if self.rag_provider == LIGHTRAG_PROVIDER and self.accepted_indexing_snapshot is None:
            from deeptutor.services.rag.pipelines.lightrag.indexing_policy import (
                bind_target,
                resolve_write_snapshot,
            )

            self.accepted_indexing_snapshot = bind_target(
                resolve_write_snapshot(
                    self.kb_dir, base_dir=str(self.base_dir), kb_name=self.kb_name
                ),
                self.kb_dir,
            )

        self.api_key = api_key
        self.base_url = base_url
        self.progress_tracker = progress_tracker

        self.raw_dir.mkdir(parents=True, exist_ok=True)

    def _provider_from_binding(self) -> str:
        return resolve_bound_provider(self.base_dir, self.kb_name)

    def _get_file_hash(self, file_path: Path) -> str:
        return content_hash(file_path)

    def get_ingested_hashes(self) -> dict[str, str]:
        if self.metadata_file.exists():
            try:
                with open(self.metadata_file, "r", encoding="utf-8") as f:
                    data = json.load(f)
                return data.get("file_hashes", {})
            except Exception:
                return {}
        return {}

    def add_documents(
        self,
        source_files: List[str],
        allow_duplicates: bool = False,
        source_root: str | Path | None = None,
    ) -> List[Path]:
        """Validate and stage files into raw/ before indexing.

        ``source_root``, when given, is the root a caller is syncing from
        (e.g. a linked folder). Any source file found under it is staged at
        the same path *relative to that root*, instead of being flattened to
        its bare filename: without this, every externally-sourced subfolder
        collapses into raw/'s top level and same-named files in different
        subfolders collide.
        """
        logger.info(f"Validating documents for '{self.kb_name}'...")

        ingested_hashes = self.get_ingested_hashes()
        files_to_process: list[Path] = []
        resolved_source_root = Path(source_root).resolve() if source_root else None

        for source in source_files:
            source_path = Path(source)
            if not source_path.exists() or not source_path.is_file():
                logger.warning(f"Missing file: {source}")
                continue

            current_hash = self._get_file_hash(source_path)
            if current_hash in ingested_hashes.values() and not allow_duplicates:
                logger.info(f"Skipped (content already indexed): {source_path.name}")
                continue

            resolved_source = source_path.resolve()

            # Files already saved under raw/ (e.g. by the upload route, possibly
            # inside a folder) are indexed in place — never flattened to the
            # basename — so the uploaded folder structure is preserved verbatim.
            if resolved_source.is_relative_to(self.raw_dir.resolve()):
                files_to_process.append(source_path)
                continue

            if resolved_source_root and resolved_source.is_relative_to(resolved_source_root):
                dest_path = self.raw_dir / resolved_source.relative_to(resolved_source_root)
            else:
                dest_path = self.raw_dir / source_path.name

            staged_path, copied = stage_source(source_path, dest_path, source_hash=current_hash)
            if copied:
                logger.info(f"Staged to raw: {staged_path.relative_to(self.raw_dir).as_posix()}")
            else:
                logger.info(f"Recovering staged file: {source_path.name}")
            files_to_process.append(staged_path)

        return files_to_process

    async def process_new_documents(self, new_files: List[Path]) -> DocumentIndexResult:
        """Index staged files via the KB's bound provider."""
        if not new_files:
            return DocumentIndexResult(processed_files=[], failures=[])

        if self.rag_provider == LIGHTRAG_PROVIDER:
            return await self._process_lightrag_batch(new_files)

        rag_service = RAGService(kb_base_dir=str(self.base_dir), provider=self.rag_provider)
        processed_files: list[Path] = []
        failures: list[DocumentIndexFailure] = []
        total_files = len(new_files)

        for idx, doc_file in enumerate(new_files, 1):
            try:
                if self.progress_tracker is not None:
                    from deeptutor.knowledge.progress_tracker import ProgressStage

                    self.progress_tracker.update(
                        ProgressStage.PROCESSING_FILE,
                        message_key="Indexing {{name}}",
                        message_params={"name": doc_file.name},
                        current=len(processed_files),
                        total=total_files,
                    )

                success = await rag_service.add_documents(self.kb_name, [str(doc_file)])
                if success:
                    processed_files.append(doc_file)
                    # Re-hashes the whole file, so it stays off the event loop:
                    # this runs inside a FastAPI BackgroundTasks entry, which is
                    # awaited on the loop and would otherwise stall unrelated
                    # requests once per indexed file (#777).
                    try:
                        await asyncio.to_thread(self._record_successful_hash, doc_file)
                    except Exception:
                        if self.rag_provider != LIGHTRAG_PROVIDER:
                            raise
                        logger.warning(
                            "LightRAG published '%s' before its duplicate-detection hash "
                            "was recorded",
                            doc_file.name,
                            exc_info=True,
                        )
                    logger.info(f"Processed: {doc_file.name}")
                    if self.progress_tracker is not None:
                        self.progress_tracker.update(
                            ProgressStage.PROCESSING_FILE,
                            message_key="Indexed {{name}}",
                            message_params={"name": doc_file.name},
                            current=len(processed_files),
                            total=total_files,
                        )
                else:
                    error = "Provider returned failure without details."
                    failures.append(DocumentIndexFailure(doc_file, error))
                    logger.error(f"Failed to index: {doc_file.name}")
            except PermissionError as e:
                logger.exception("Permission denied while indexing %s: %s", doc_file.name, e)
                raise DataVolumePermissionError(
                    format_data_volume_permission_error(self.kb_dir, cause=e)
                ) from e
            except Exception as e:
                logger.exception(f"Failed {doc_file.name}: {e}")
                failures.append(DocumentIndexFailure(doc_file, str(e)))

        return DocumentIndexResult(processed_files=processed_files, failures=failures)

    async def _process_lightrag_batch(self, files: List[Path]) -> DocumentIndexResult:
        """Hold native write ownership across the entire accepted batch."""
        from deeptutor.services.rag.pipelines.lightrag.pipeline import LightRagBatchError

        def prepare_publication(root: Path) -> None:
            from deeptutor.knowledge.progress_tracker import ProgressStage, ProgressTracker

            tracker = self.progress_tracker or ProgressTracker(self.kb_name, self.base_dir)
            tracker.update(
                ProgressStage.COMPLETED,
                message="Document indexing complete",
                current=len(files),
                total=len(files),
                indexed_count=len(files),
                index_changed=True,
                index_action="upload",
                publication_version=root.name,
            )
            tracker.verify_terminal(
                current=len(files),
                total=len(files),
                indexed_count=len(files),
                index_action="upload",
                publication_version=root.name,
            )

        service = RAGService(kb_base_dir=str(self.base_dir), provider=LIGHTRAG_PROVIDER)
        try:
            success = await service.add_documents(
                self.kb_name,
                [str(path) for path in files],
                accepted_indexing_snapshot=self.accepted_indexing_snapshot,
                before_publish=prepare_publication,
            )
            if not success:
                raise RuntimeError("LightRAG returned failure without details.")
            processed = files
            failures = []
        except LightRagBatchError as exc:
            processed = [path for path in files if path.name in exc.outcome.processed]
            failures = [
                DocumentIndexFailure(path, str(exc)) for path in files if path not in processed
            ]
        except Exception as exc:
            return DocumentIndexResult([], [DocumentIndexFailure(path, str(exc)) for path in files])
        for path in processed:
            try:
                await asyncio.to_thread(self._record_successful_hash, path)
            except Exception:
                logger.warning(
                    "Indexed document hash could not be recorded: %s", path.name, exc_info=True
                )
        return DocumentIndexResult(processed, failures)

    def _record_successful_hash(self, file_path: Path) -> None:
        file_hash = self._get_file_hash(file_path)
        metadata = _read_metadata(self.metadata_file)
        hash_key = _raw_hash_key(file_path, self.raw_dir)
        metadata.setdefault("file_hashes", {})[hash_key] = file_hash
        _write_metadata(self.metadata_file, metadata)

    def update_metadata(self, added_count: int) -> None:
        """Update metadata after incremental add."""
        metadata: dict = {}
        if self.metadata_file.exists():
            try:
                with open(self.metadata_file, "r", encoding="utf-8") as f:
                    metadata = json.load(f)
            except Exception:
                metadata = {}

        metadata["rag_provider"] = self.rag_provider
        metadata["needs_reindex"] = False
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        metadata["last_updated"] = timestamp
        if added_count > 0:
            metadata["last_indexed_at"] = timestamp
            metadata["last_indexed_count"] = added_count
            metadata["last_indexed_action"] = "upload"

        history = metadata.get("update_history", [])
        history.append(
            {
                "timestamp": metadata["last_updated"],
                "action": "incremental_add",
                "count": added_count,
                "provider": self.rag_provider,
            }
        )
        metadata["update_history"] = history

        _write_metadata(self.metadata_file, metadata)


async def _bootstrap_index_from_files(
    kb_name: str,
    source_files: list[str],
    base_dir: str,
    manager: "KnowledgeBaseManager",
) -> int:
    """Create a fresh index for an empty KB from the given source files.

    Called when :class:`DocumentAdder` rejects an add because the KB has no
    existing index (it was created empty, e.g. via the no-files fast path or
    a web/GitHub source sync before any documents were indexed). Uses
    :meth:`RAGService.initialize` to build the index in one batch, then
    records file hashes so subsequent incremental adds can detect duplicates.
    """
    rag_service = RAGService(kb_base_dir=base_dir)
    kb_dir = Path(base_dir) / kb_name
    raw_dir = kb_dir / "raw"
    metadata_file = kb_dir / "metadata.json"

    requested = {str(Path(path).resolve()): str(Path(path)) for path in source_files}
    confirmed: set[str] = set()

    def on_indexed_file(paths: list[str]) -> None:
        for path in paths:
            canonical = str(Path(path).resolve())
            if canonical in requested:
                confirmed.add(canonical)

    success = await rag_service.initialize(
        kb_name=kb_name,
        file_paths=source_files,
        indexed_file_callback=on_indexed_file,
    )
    if not success:
        raise RuntimeError(
            f"Failed to initialize index for KB '{kb_name}' from {len(source_files)} file(s)"
        )

    indexed_files = [requested[path] for path in sorted(confirmed)]
    indexed = len(indexed_files)
    provider = rag_service._resolve_provider(kb_name)
    try:
        # Record hashes so future syncs detect unchanged files.
        metadata = _read_metadata(metadata_file)
        metadata.setdefault("file_hashes", {}).update(
            hashes_for_indexed_files(indexed_files, raw_dir)
        )
        metadata["rag_provider"] = provider
        metadata["needs_reindex"] = False
        ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        metadata["last_updated"] = ts
        metadata["last_indexed_at"] = ts
        metadata["last_indexed_count"] = indexed
        metadata["last_indexed_action"] = "create"
        _write_metadata(metadata_file, metadata)

        manager.update_kb_status(
            name=kb_name,
            status="ready",
            progress={
                "stage": "completed",
                "message": f"Initialized index with {indexed} file(s).",
                "percent": 100,
                "current": indexed,
                "total": max(indexed, 1),
                "file_name": "",
                "error": None,
                "timestamp": datetime.now().isoformat(),
                "indexed_count": indexed,
                "index_changed": True,
                "index_action": "create",
            },
        )
    except Exception:
        if provider != LIGHTRAG_PROVIDER:
            raise
        logger.warning(
            "LightRAG bootstrap for '%s' published before bookkeeping failed",
            kb_name,
            exc_info=True,
        )
    logger.info("Bootstrapped index for empty KB '%s' with %d file(s)", kb_name, indexed)
    return indexed


async def add_documents(
    kb_name: str,
    source_files: list[str],
    base_dir: str = DEFAULT_BASE_DIR,
    api_key: Optional[str] = None,
    base_url: Optional[str] = None,
    allow_duplicates: bool = False,
    accepted_indexing_snapshot=None,
) -> int:
    """Convenience function used by CLI wrappers."""
    from deeptutor.knowledge.manager import KnowledgeBaseManager

    manager = KnowledgeBaseManager(base_dir=base_dir)
    published_count = 0
    try:
        manager.update_kb_status(
            name=kb_name,
            status="processing",
            progress={
                "stage": "processing_documents",
                "message": "Processing uploaded documents...",
                "percent": 0,
                "current": 0,
                "total": max(len(source_files), 1),
                "file_name": "",
                "error": None,
                "timestamp": datetime.now().isoformat(),
            },
        )

        try:
            adder = DocumentAdder(
                kb_name=kb_name,
                base_dir=base_dir,
                api_key=api_key,
                base_url=base_url,
                accepted_indexing_snapshot=accepted_indexing_snapshot,
            )
        except ValueError as exc:
            if "not initialized" not in str(exc).lower() or not source_files:
                raise
            # Empty KB (created without initial documents): bootstrap the
            # index from these files instead of erroring.
            return await _bootstrap_index_from_files(
                kb_name=kb_name,
                source_files=source_files,
                base_dir=base_dir,
                manager=manager,
            )

        new_files = adder.add_documents(source_files, allow_duplicates=allow_duplicates)
        if not new_files:
            manager.update_kb_status(
                name=kb_name,
                status="ready",
                progress={
                    "stage": "completed",
                    "message": "No new unique documents to process.",
                    "percent": 100,
                    "current": 1,
                    "total": 1,
                    "file_name": "",
                    "error": None,
                    "timestamp": datetime.now().isoformat(),
                },
            )
            return 0
        result = await adder.process_new_documents(new_files)
        if result.has_failures:
            raise RuntimeError(
                f"Failed to index {result.failed_count}/{len(new_files)} file(s): "
                f"{result.failure_summary()}"
            )
        if adder.rag_provider == LIGHTRAG_PROVIDER:
            published_count = result.processed_count
        adder.update_metadata(result.processed_count)

        manager.update_kb_status(
            name=kb_name,
            status="ready",
            progress={
                "stage": "completed",
                "message": f"Successfully processed {result.processed_count} files!",
                "percent": 100,
                "current": result.processed_count,
                "total": max(len(new_files), 1),
                "file_name": "",
                "error": None,
                "timestamp": datetime.now().isoformat(),
                "indexed_count": result.processed_count,
                "index_changed": result.processed_count > 0,
                "index_action": "upload",
            },
        )
        return result.processed_count
    except Exception as exc:
        if published_count:
            logger.warning(
                "LightRAG append for '%s' published before bookkeeping failed",
                kb_name,
                exc_info=True,
            )
            return published_count
        manager.update_kb_status(
            name=kb_name,
            status="error",
            progress={
                "stage": "error",
                "message": "Document upload failed",
                "percent": 0,
                "current": 0,
                "total": max(len(source_files), 1),
                "file_name": "",
                "error": str(exc),
                "timestamp": datetime.now().isoformat(),
            },
        )
        raise


async def main() -> None:
    try:
        llm_config = resolve_llm_runtime_config()
        default_api_key = llm_config.api_key
        default_base_url = llm_config.effective_url
    except Exception:
        default_api_key = ""
        default_base_url = ""

    parser = argparse.ArgumentParser(description="Incrementally add documents to a KB")
    parser.add_argument("kb_name", help="KB Name")
    parser.add_argument("--docs", nargs="+", help="Files")
    parser.add_argument("--docs-dir", help="Directory")
    parser.add_argument("--base-dir", default=DEFAULT_BASE_DIR)
    parser.add_argument("--api-key", default=default_api_key)
    parser.add_argument("--base-url", default=default_base_url)
    parser.add_argument("--allow-duplicates", action="store_true")

    args = parser.parse_args()
    doc_files: list[str] = []
    if args.docs:
        doc_files.extend(args.docs)
    if args.docs_dir:
        p = Path(args.docs_dir)
        doc_files.extend(str(f) for f in FileTypeRouter.collect_supported_files(p))

    if not doc_files:
        logger.error("No documents provided.")
        return

    processed_count = await add_documents(
        kb_name=args.kb_name,
        source_files=doc_files,
        base_dir=args.base_dir,
        api_key=args.api_key,
        base_url=args.base_url,
        allow_duplicates=args.allow_duplicates,
    )

    if processed_count:
        logger.info(f"Done! Successfully added {processed_count} documents.")
    else:
        logger.info("No new unique documents to add.")


if __name__ == "__main__":
    asyncio.run(main())