Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): resume embedding checkpoints

XZHf657d343eea3
diff --git a/deeptutor/knowledge/indexing_run.py b/deeptutor/knowledge/indexing_run.py
index 8a3a73774..c6624e300 100644
--- a/deeptutor/knowledge/indexing_run.py
+++ b/deeptutor/knowledge/indexing_run.py
@@ -30,8 +30,8 @@ from deeptutor.services.file_io import atomic_write_json, read_text
_CURRENT: ContextVar[IndexingRun | None] = ContextVar("indexing_run", default=None)
_TERMINAL = {"completed", "partial", "failed", "cancelled", "interrupted"}
_RECOVERY_DESCRIPTION = (
- "Retry reuses validated parser outputs and completed cloud-parsing slices. "
- "Uncheckpointed chunking and embedding stages restart."
+ "Retry reuses validated parser outputs and resumable LlamaIndex document "
+ "checkpoints. Uncheckpointed chunking and embedding work restarts."
)
@@ -190,6 +190,7 @@ class IndexingRun:
self._lock = None
self._token = None
self._source_paths: dict[str, Path] = {}
+ self.previous_run: dict[str, Any] | None = None
def __enter__(self):
return self._start()
@@ -254,6 +255,7 @@ class IndexingRun:
"recovery": _RECOVERY_DESCRIPTION,
"cancel_requested": False,
"usable_version": None,
+ "candidate_storage": None,
}
from deeptutor.services.rag.file_routing import FileTypeRouter
@@ -284,6 +286,7 @@ class IndexingRun:
"parse_reused": False,
}
self.active = True
+ self.previous_run = previous
if bind_context:
self._token = _CURRENT.set(self)
self._save()
@@ -329,6 +332,11 @@ class IndexingRun:
fcntl.flock(self._lock.fileno(), fcntl.LOCK_UN)
self._lock.close()
+ def set_candidate_storage(self, storage_dir: Path) -> None:
+ """Record the unpublished output that a retry may resume."""
+ self.data["candidate_storage"] = str(storage_dir)
+ self._save()
+
def source_key(self, source: Path) -> str:
try:
return source.resolve().relative_to((self.kb_dir / "raw").resolve()).as_posix()
@@ -337,6 +345,14 @@ class IndexingRun:
f"external-{sha256(str(source.resolve()).encode()).hexdigest()[:12]}/{source.name}"
)
+ def source_receipts(self) -> dict[str, tuple[Path, str]]:
+ """Return the current run's source path and hash by durable key."""
+ return {
+ key: (self._source_paths[key], str(doc.get("source_hash", "")))
+ for key, doc in self.data["documents"].items()
+ if key in self._source_paths and doc.get("source_hash")
+ }
+
@staticmethod
def _hash(source: Path) -> str:
digest = sha256()
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index e7b0628a6..e8f35f571 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -197,7 +197,16 @@ class LlamaIndexPipeline:
f"Could not clean up failed version dir for {storage_dir}: {cleanup_exc}"
)
+ @staticmethod
+ def _embedding_source_records(run: Any) -> dict[str, storage.EmbeddingSourceRecord]:
+ return {
+ key: storage.EmbeddingSourceRecord(key=key, path=path, source_hash=source_hash)
+ for key, (path, source_hash) in run.source_receipts().items()
+ }
+
async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+ from deeptutor.knowledge.indexing_run import current_run
+
progress_callback = kwargs.get("progress_callback")
image_progress_callback = kwargs.get("image_progress_callback")
kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)
@@ -209,7 +218,19 @@ class LlamaIndexPipeline:
)
signature = self._current_signature()
- storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True)
+ run = current_run()
+ source_records = self._embedding_source_records(run) if run is not None else {}
+ storage_dir = storage.resolve_resumable_rebuild_dir(
+ kb_dir,
+ signature,
+ previous_run=run.previous_run if run is not None else None,
+ source_records=source_records,
+ )
+ resumed = storage_dir is not None
+ if storage_dir is None:
+ storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True)
+ if run is not None:
+ run.set_candidate_storage(storage_dir)
try:
await self._verify_embedding_connectivity()
@@ -233,15 +254,32 @@ class LlamaIndexPipeline:
# reference them. Failed indexing can leave unused immutable assets;
# failed asset publication must not leave a partially accepted index.
VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)
+
+ def checkpoint_document(path: Path, embedding_reused: bool) -> None:
+ if run is not None:
+ run.document(
+ path,
+ "completed",
+ embedding_checkpoint=True,
+ embedding_reused=embedding_reused,
+ )
+
await _run_with_stall_guard(
- lambda: storage.create_index(
- documents, storage_dir, show_progress=should_show_progress()
+ lambda: storage.create_resumable_index(
+ documents,
+ storage_dir,
+ source_records=source_records,
+ signature=signature,
+ show_progress=should_show_progress(),
+ checkpoint_callback=checkpoint_document if run is not None else None,
),
progress_callback=progress_callback,
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Index persisted to {storage_dir}")
+ if resumed:
+ self.logger.info(f"Resumed interrupted LlamaIndex candidate at {storage_dir}")
await self._publish_verified(kb_dir, storage_dir, signature, documents)
indexed_file_callback = kwargs.get("indexed_file_callback")
diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.py
index 36d34fd12..71e211507 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/storage.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py
@@ -6,13 +6,19 @@ from collections import OrderedDict
from dataclasses import asdict, dataclass
import hashlib
import json
+import os
from pathlib import Path
import shutil
+import tempfile
import threading
import time
-from typing import Any
+from typing import Any, Callable
+from uuid import uuid4
+
+from llama_index.core import VectorStoreIndex
from deeptutor.services.embedding.validation import validate_embedding_batch
+from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.rag.index_versioning import (
EmbeddingSignature,
resolve_storage_dir_for_read,
@@ -20,6 +26,7 @@ from deeptutor.services.rag.index_versioning import (
)
from . import ingestion, retrievers, vector_store
+from .config import should_show_progress, vector_index_config_from_settings
@dataclass(frozen=True)
@@ -28,6 +35,19 @@ class AddStoragePlan:
storage_dir: Path
+@dataclass(frozen=True)
+class EmbeddingSourceRecord:
+ key: str
+ path: Path
+ source_hash: str
+
+
+@dataclass(frozen=True)
+class ResumeCandidate:
+ storage_dir: Path
+ completed_keys: frozenset[str]
+
+
def _storage_path_from_version_entry(entry: dict[str, Any]) -> Path | None:
storage_path = entry.get("storage_path")
if storage_path:
@@ -56,6 +76,83 @@ def cleanup_failed_version_dir(storage_dir: Path) -> bool:
return False
+def _read_embedding_checkpoint(storage_dir: Path) -> dict[str, Any] | None:
+ path = storage_dir / ".embedding-checkpoint.json"
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return None
+ if (
+ not isinstance(payload, dict)
+ or payload.get("schema") != 1
+ or payload.get("provider") != "llamaindex"
+ or payload.get("action") != "rebuild"
+ or not isinstance(payload.get("sources"), dict)
+ ):
+ return None
+ return payload
+
+
+def _is_unpublished_building_dir(storage_dir: Path) -> bool:
+ if storage_dir.name.startswith(".") or not storage_dir.name.startswith("version-"):
+ return False
+ if (storage_dir / "meta.json").exists():
+ return False
+ try:
+ building = json.loads((storage_dir / ".building.json").read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return False
+ return isinstance(building, dict) and building.get("state") == "building"
+
+
+def resolve_resumable_rebuild_dir(
+ kb_dir: Path,
+ signature: EmbeddingSignature | None,
+ *,
+ previous_run: dict[str, Any] | None,
+ source_records: dict[str, EmbeddingSourceRecord],
+) -> Path | None:
+ """Return an interrupted candidate only when every source still matches."""
+ if not previous_run or previous_run.get("provider") != "llamaindex":
+ return None
+ if previous_run.get("action") != "rebuild" or previous_run.get("state") not in {
+ "interrupted",
+ "failed",
+ "cancelled",
+ }:
+ return None
+ if previous_run.get("usable_version"):
+ return None
+
+ candidate_value = previous_run.get("candidate_storage")
+ if not isinstance(candidate_value, str) or not candidate_value:
+ return None
+ candidate = Path(candidate_value)
+ try:
+ candidate.resolve().relative_to(kb_dir.resolve())
+ except (OSError, ValueError):
+ return None
+ if candidate.parent.resolve() != kb_dir.resolve() or not _is_unpublished_building_dir(
+ candidate
+ ):
+ return None
+
+ payload = _read_embedding_checkpoint(candidate)
+ if payload is None:
+ return None
+ expected_signature = signature.hash() if signature is not None else None
+ if payload.get("signature") != expected_signature:
+ return None
+ sources = payload["sources"]
+ if set(sources) != set(source_records):
+ return None
+ for key, record in source_records.items():
+ row = sources.get(key)
+ if not isinstance(row, dict) or row.get("source_hash") != record.source_hash:
+ return None
+ return candidate
+
+
def resolve_add_storage_plan(kb_dir: Path, signature: EmbeddingSignature | None) -> AddStoragePlan:
"""Choose existing/new storage dirs for incremental adds."""
existing = resolve_storage_dir_for_read(kb_dir, signature)
@@ -65,6 +162,222 @@ def resolve_add_storage_plan(kb_dir: Path, signature: EmbeddingSignature | None)
)
+def _group_documents_by_source(
+ documents: list[Any], source_records: dict[str, EmbeddingSourceRecord]
+) -> dict[str, list[Any]] | None:
+ path_to_key = {str(record.path.resolve()): key for key, record in source_records.items()}
+ grouped: dict[str, list[Any]] = {}
+ for document in documents:
+ metadata = getattr(document, "metadata", None)
+ source_value = metadata.get("file_path") if isinstance(metadata, dict) else None
+ if not isinstance(source_value, str):
+ return None
+ try:
+ source_path = str(Path(source_value).resolve())
+ except OSError:
+ return None
+ key = path_to_key.get(source_path)
+ if key is None:
+ return None
+ grouped.setdefault(key, []).append(document)
+ return grouped
+
+
+def _resume_candidate(
+ storage_dir: Path, source_records: dict[str, EmbeddingSourceRecord]
+) -> ResumeCandidate | None:
+ if not _is_unpublished_building_dir(storage_dir):
+ return None
+ payload = _read_embedding_checkpoint(storage_dir)
+ if payload is None:
+ return None
+ if set(payload["sources"]) != set(source_records):
+ return None
+ completed_keys: set[str] = set()
+ for key, row in payload["sources"].items():
+ if not isinstance(row, dict) or row.get("source_hash") != source_records[key].source_hash:
+ return None
+ if row.get("completed") is True:
+ completed_keys.add(key)
+ if not completed_keys:
+ return None
+
+ try:
+ index = vector_store.load_index(storage_dir)
+ _validate_persisted_embeddings(index, storage_dir)
+ except Exception:
+ return None
+ indexed_paths = {
+ str(Path(getattr(node, "metadata", {}).get("file_path", "")).resolve())
+ for node in getattr(index, "docstore", {}).docs.values()
+ if isinstance(getattr(node, "metadata", None), dict) and node.metadata.get("file_path")
+ }
+ for key in completed_keys:
+ if str(source_records[key].path.resolve()) not in indexed_paths:
+ return None
+ return ResumeCandidate(storage_dir=storage_dir, completed_keys=frozenset(completed_keys))
+
+
+def _fsync_directory(path: Path) -> None:
+ try:
+ directory_fd = os.open(path, os.O_RDONLY)
+ try:
+ os.fsync(directory_fd)
+ finally:
+ os.close(directory_fd)
+ except OSError:
+ # Directory replacement still publishes a complete tree; fsync is a
+ # durability hint on filesystems that reject it for arbitrary files.
+ pass
+
+
+def _sync_checkpoint_tree(root: Path) -> None:
+ try:
+ for path in root.rglob("*"):
+ if path.is_file():
+ with path.open("rb") as handle:
+ os.fsync(handle.fileno())
+ _fsync_directory(root)
+ except OSError:
+ # Directory replacement still publishes a complete tree; fsync is a
+ # durability hint on filesystems that reject it for arbitrary files.
+ pass
+
+
+def _recover_candidate_swap(candidate: Path) -> None:
+ backup_prefix = f".{candidate.name}.previous-"
+ backups = sorted(
+ (path for path in candidate.parent.glob(backup_prefix + "*") if path.is_dir()),
+ key=lambda path: path.stat().st_mtime_ns,
+ )
+ if candidate.exists():
+ for backup in backups:
+ shutil.rmtree(backup, ignore_errors=True)
+ return
+ if backups:
+ backups[-1].replace(candidate)
+
+
+def _swap_candidate_snapshot(candidate: Path, snapshot: Path) -> None:
+ _recover_candidate_swap(candidate)
+ backup = candidate.parent / f".{candidate.name}.previous-{uuid4().hex}"
+ if candidate.exists():
+ candidate.replace(backup)
+ _fsync_directory(candidate.parent)
+ try:
+ snapshot.replace(candidate)
+ _fsync_directory(candidate.parent)
+ except BaseException:
+ if not candidate.exists() and backup.exists():
+ backup.replace(candidate)
+ raise
+ else:
+ if backup.exists():
+ shutil.rmtree(backup, ignore_errors=True)
+ for stale in candidate.parent.glob(f".{candidate.name}.checkpoint-*"):
+ shutil.rmtree(stale, ignore_errors=True)
+
+
+def _write_embedding_snapshot(
+ index: Any,
+ candidate: Path,
+ *,
+ source_records: dict[str, EmbeddingSourceRecord],
+ completed_keys: set[str],
+ document_group: list[Any],
+ signature: EmbeddingSignature | None,
+) -> None:
+ snapshot = Path(tempfile.mkdtemp(prefix=f".{candidate.name}.checkpoint-", dir=candidate.parent))
+ try:
+ index.storage_context.persist(persist_dir=str(snapshot))
+ retrievers.persist_bm25_retriever(index, snapshot, top_k=20)
+ atomic_write_json(snapshot / ".building.json", {"state": "building"})
+ existing_catalog = candidate / "source_pages.json"
+ if existing_catalog.is_file():
+ shutil.copy2(existing_catalog, snapshot / existing_catalog.name)
+ from deeptutor.services.rag.source_pages import persist_page_catalog
+
+ persist_page_catalog(document_group, snapshot)
+ atomic_write_json(
+ snapshot / ".embedding-checkpoint.json",
+ {
+ "schema": 1,
+ "provider": "llamaindex",
+ "action": "rebuild",
+ "signature": signature.hash() if signature is not None else None,
+ "sources": {
+ key: {
+ "path": str(record.path),
+ "source_hash": record.source_hash,
+ "completed": key in completed_keys,
+ }
+ for key, record in source_records.items()
+ },
+ },
+ )
+ _sync_checkpoint_tree(snapshot)
+ _swap_candidate_snapshot(candidate, snapshot)
+ finally:
+ if snapshot.exists():
+ shutil.rmtree(snapshot, ignore_errors=True)
+
+
+def create_resumable_index(
+ documents: list[Any],
+ storage_dir: Path,
+ *,
+ source_records: dict[str, EmbeddingSourceRecord],
+ signature: EmbeddingSignature | None,
+ show_progress: bool | None = None,
+ checkpoint_callback: Callable[[Path, bool], None] | None = None,
+) -> int:
+ """Build a rebuild candidate one source at a time and snapshot each source."""
+ if show_progress is None:
+ show_progress = should_show_progress()
+ if (storage_dir / "meta.json").exists():
+ raise ValueError(f"Refusing to modify published index: {storage_dir}")
+ grouped = _group_documents_by_source(documents, source_records)
+ if grouped is None:
+ return create_index(documents, storage_dir, show_progress=show_progress)
+
+ _recover_candidate_swap(storage_dir)
+ resume = _resume_candidate(storage_dir, source_records)
+ index: Any = None
+ completed_keys = set(resume.completed_keys if resume else set())
+ if resume is not None and checkpoint_callback is not None:
+ for key in sorted(completed_keys):
+ checkpoint_callback(source_records[key].path, True)
+
+ for key, document_group in grouped.items():
+ if key in completed_keys:
+ continue
+ nodes = ingestion.documents_to_nodes(document_group, show_progress=show_progress)
+ if index is None:
+ storage_context = vector_store.storage_context_for_nodes(
+ nodes, vector_index_config_from_settings()
+ )
+ index = VectorStoreIndex(
+ nodes=nodes,
+ storage_context=storage_context,
+ show_progress=show_progress,
+ embed_model=ingestion.current_embedding(),
+ )
+ else:
+ index.insert_nodes(nodes)
+ completed_keys.add(key)
+ _write_embedding_snapshot(
+ index,
+ storage_dir,
+ source_records=source_records,
+ completed_keys=completed_keys,
+ document_group=document_group,
+ signature=signature,
+ )
+ if checkpoint_callback is not None:
+ checkpoint_callback(source_records[key].path, False)
+ return len(documents)
+
+
def create_index(
documents: list[Any], storage_dir: Path, *, show_progress: bool | None = None
) -> int:
diff --git a/tests/services/rag/test_llamaindex_embedding_resume.py b/tests/services/rag/test_llamaindex_embedding_resume.py
new file mode 100644
index 000000000..9297695b1
--- /dev/null
+++ b/tests/services/rag/test_llamaindex_embedding_resume.py
@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from deeptutor.services.rag.index_versioning import EmbeddingSignature
+
+
+def _signature() -> EmbeddingSignature:
+ return EmbeddingSignature(
+ binding="test",
+ model="embedding-resume",
+ dimension=4,
+ base_url="https://embedding.test",
+ api_version="",
+ )
+
+
+def _records(kb_dir: Path, names: list[str]) -> dict[str, object]:
+ from deeptutor.services.rag.pipelines.llamaindex import storage
+
+ records = {}
+ for name in names:
+ source = kb_dir / "raw" / name
+ records[name] = storage.EmbeddingSourceRecord(
+ key=name,
+ path=source,
+ source_hash=hashlib.sha256(source.read_bytes()).hexdigest(),
+ )
+ return records
+
+
+async def test_rebuild_resumes_completed_documents_after_failure(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from llama_index.core import Settings
+ from llama_index.core.embeddings import MockEmbedding
+
+ from deeptutor.knowledge.indexing_run import IndexingRun, load_run
+ from deeptutor.services.rag.pipelines.llamaindex import storage
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+
+ kb_dir = tmp_path / "kb"
+ raw = kb_dir / "raw"
+ raw.mkdir(parents=True)
+ first, second = raw / "first.txt", raw / "second.txt"
+ first.write_text("The first source explains Fourier transforms.", encoding="utf-8")
+ second.write_text("The second source explains sampling.", encoding="utf-8")
+
+ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))
+ monkeypatch.setattr(
+ storage.retrievers, "persist_bm25_retriever", lambda *_args, **_kwargs: False
+ )
+ original_snapshot = storage._write_embedding_snapshot
+ snapshots = 0
+
+ def fail_second_snapshot(*args, **kwargs):
+ nonlocal snapshots
+ snapshots += 1
+ if snapshots == 2:
+ raise OSError("simulated power loss boundary")
+ return original_snapshot(*args, **kwargs)
+
+ def make_pipeline() -> LlamaIndexPipeline:
+ pipeline = LlamaIndexPipeline(str(tmp_path), signature_provider=_signature)
+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)
+
+ async def verify_embedding() -> None:
+ return None
+
+ monkeypatch.setattr(
+ pipeline,
+ "_verify_embedding_connectivity",
+ verify_embedding,
+ )
+ return pipeline
+
+ monkeypatch.setattr(storage, "_write_embedding_snapshot", fail_second_snapshot)
+ with pytest.raises(OSError, match="simulated power loss"):
+ async with IndexingRun(kb_dir, [str(first), str(second)], provider="llamaindex"):
+ await make_pipeline().initialize("kb", [str(first), str(second)])
+
+ failed_run = load_run(kb_dir)
+ candidate = Path(failed_run["candidate_storage"])
+ checkpoint = json.loads((candidate / ".embedding-checkpoint.json").read_text(encoding="utf-8"))
+ assert checkpoint["sources"]["first.txt"]["completed"] is True
+ assert checkpoint["sources"]["second.txt"]["completed"] is False
+ assert failed_run["documents"]["first.txt"]["status"] == "completed"
+ assert failed_run["documents"]["first.txt"]["embedding_reused"] is False
+
+ monkeypatch.setattr(storage, "_write_embedding_snapshot", original_snapshot)
+ embedded_sources: list[str] = []
+ original_documents_to_nodes = storage.ingestion.documents_to_nodes
+
+ def track_documents(documents, **kwargs):
+ embedded_sources.extend(Path(document.metadata["file_path"]).name for document in documents)
+ return original_documents_to_nodes(documents, **kwargs)
+
+ monkeypatch.setattr(storage.ingestion, "documents_to_nodes", track_documents)
+ async with IndexingRun(kb_dir, [str(first), str(second)], provider="llamaindex") as run:
+ assert await make_pipeline().initialize("kb", [str(first), str(second)])
+ assert run.data["candidate_storage"] == str(candidate)
+ run.finish("completed", version=candidate.name)
+
+ resumed_run = load_run(kb_dir)
+ assert resumed_run["state"] == "completed"
+ assert resumed_run["documents"]["first.txt"]["embedding_reused"] is True
+ assert resumed_run["documents"]["second.txt"]["embedding_reused"] is False
+ assert embedded_sources == [second.name]
+ storage.verify_persisted_index(candidate)
+ assert checkpoint["signature"] == _signature().hash()
+
+
+def test_resume_requires_exact_source_hashes_and_checkpoint(
+ tmp_path: Path,
+) -> None:
+ from deeptutor.services.rag.pipelines.llamaindex import storage
+
+ kb_dir = tmp_path / "kb"
+ raw = kb_dir / "raw"
+ raw.mkdir(parents=True)
+ source = raw / "book.txt"
+ source.write_text("stable source", encoding="utf-8")
+ candidate = kb_dir / "version-1"
+ candidate.mkdir()
+ (candidate / ".building.json").write_text('{"state":"building"}', encoding="utf-8")
+
+ records = _records(kb_dir, ["book.txt"])
+ previous = {
+ "state": "interrupted",
+ "action": "rebuild",
+ "provider": "llamaindex",
+ "usable_version": None,
+ "candidate_storage": str(candidate),
+ }
+ assert (
+ storage.resolve_resumable_rebuild_dir(
+ kb_dir, _signature(), previous_run=previous, source_records=records
+ )
+ is None
+ )
+
+ manifest = {
+ "schema": 1,
+ "provider": "llamaindex",
+ "action": "rebuild",
+ "signature": _signature().hash(),
+ "sources": {
+ key: {
+ "path": str(record.path),
+ "source_hash": record.source_hash,
+ "completed": True,
+ }
+ for key, record in records.items()
+ },
+ }
+ (candidate / ".embedding-checkpoint.json").write_text(json.dumps(manifest), encoding="utf-8")
+ resolved = storage.resolve_resumable_rebuild_dir(
+ kb_dir,
+ _signature(),
+ previous_run=previous,
+ source_records=records,
+ )
+ assert resolved == candidate
+ interrupted = {**previous, "state": "interrupted"}
+ assert (
+ storage.resolve_resumable_rebuild_dir(
+ kb_dir,
+ _signature(),
+ previous_run=interrupted,
+ source_records=records,
+ )
+ == candidate
+ )
+ (candidate / "meta.json").write_text('{"state":"published"}', encoding="utf-8")
+ assert (
+ storage.resolve_resumable_rebuild_dir(
+ kb_dir,
+ _signature(),
+ previous_run=interrupted,
+ source_records=records,
+ )
+ is None
+ )
+ (candidate / "meta.json").unlink()
+
+ source.write_text("changed source", encoding="utf-8")
+ changed_records = _records(kb_dir, ["book.txt"])
+ assert (
+ storage.resolve_resumable_rebuild_dir(
+ kb_dir,
+ _signature(),
+ previous_run=previous,
+ source_records=changed_records,
+ )
+ is None
+ )
+
+
+async def test_empty_rebuild_preserves_existing_published_index(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from deeptutor.services.rag.index_versioning import (
+ list_kb_versions,
+ resolve_storage_dir_for_read,
+ write_version_meta,
+ )
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+
+ kb_dir = tmp_path / "kb"
+ kb_dir.mkdir()
+ published = kb_dir / "version-1"
+ published.mkdir()
+ (published / "docstore.json").write_text("{}", encoding="utf-8")
+ write_version_meta(tmp_path / "kb", _signature(), published)
+
+ class Loader:
+ async def load(self, *_args, **_kwargs):
+ return []
+
+ pipeline = LlamaIndexPipeline(
+ str(tmp_path),
+ signature_provider=_signature,
+ document_loader=Loader(), # type: ignore[arg-type]
+ )
+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)
+
+ async def verify_embedding() -> None:
+ return None
+
+ monkeypatch.setattr(pipeline, "_verify_embedding_connectivity", verify_embedding)
+ assert await pipeline.initialize("kb", [str(kb_dir / "raw" / "empty.txt")]) is False
+ assert resolve_storage_dir_for_read(kb_dir, _signature()) == published
+ assert (
+ next(row for row in list_kb_versions(kb_dir) if row["version"] == "version-1")["ready"]
+ is True
+ )