Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: retain checkpoint index and recover interrupted snapshot swaps

Bingxi Zhao (Frank)0b6ff7abae6b
diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.py
index 71e211507..dfef44ce8 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/storage.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py
@@ -46,6 +46,7 @@ class EmbeddingSourceRecord:
class ResumeCandidate:
storage_dir: Path
completed_keys: frozenset[str]
+ index: Any
def _storage_path_from_version_entry(entry: dict[str, Any]) -> Path | None:
@@ -132,10 +133,17 @@ def resolve_resumable_rebuild_dir(
candidate.resolve().relative_to(kb_dir.resolve())
except (OSError, ValueError):
return None
- if candidate.parent.resolve() != kb_dir.resolve() or not _is_unpublished_building_dir(
- candidate
+ if (
+ candidate.parent.resolve() != kb_dir.resolve()
+ or not candidate.name.startswith("version-")
+ or (candidate / "meta.json").exists()
):
return None
+ # A crash can occur after moving the candidate aside but before publishing
+ # its replacement. Restore that checkpoint before testing its build marker.
+ _recover_candidate_swap(candidate)
+ if not _is_unpublished_building_dir(candidate):
+ return None
payload = _read_embedding_checkpoint(candidate)
if payload is None:
@@ -215,7 +223,9 @@ def _resume_candidate(
for key in completed_keys:
if str(source_records[key].path.resolve()) not in indexed_paths:
return None
- return ResumeCandidate(storage_dir=storage_dir, completed_keys=frozenset(completed_keys))
+ return ResumeCandidate(
+ storage_dir=storage_dir, completed_keys=frozenset(completed_keys), index=index
+ )
def _fsync_directory(path: Path) -> None:
@@ -342,7 +352,7 @@ def create_resumable_index(
_recover_candidate_swap(storage_dir)
resume = _resume_candidate(storage_dir, source_records)
- index: Any = None
+ index: Any = resume.index if resume else None
completed_keys = set(resume.completed_keys if resume else set())
if resume is not None and checkpoint_callback is not None:
for key in sorted(completed_keys):
diff --git a/tests/services/rag/test_llamaindex_embedding_resume.py b/tests/services/rag/test_llamaindex_embedding_resume.py
index 9297695b1..0bbb41e02 100644
--- a/tests/services/rag/test_llamaindex_embedding_resume.py
+++ b/tests/services/rag/test_llamaindex_embedding_resume.py
@@ -34,8 +34,9 @@ def _records(kb_dir: Path, names: list[str]) -> dict[str, object]:
@pytest.mark.asyncio
[email protected]("interrupted_swap", [False, True])
async def test_rebuild_resumes_completed_documents_after_failure(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interrupted_swap: bool
) -> None:
from llama_index.core import Settings
from llama_index.core.embeddings import MockEmbedding
@@ -92,6 +93,9 @@ async def test_rebuild_resumes_completed_documents_after_failure(
assert failed_run["documents"]["first.txt"]["status"] == "completed"
assert failed_run["documents"]["first.txt"]["embedding_reused"] is False
+ if interrupted_swap:
+ candidate.rename(candidate.with_name(f".{candidate.name}.previous-crash"))
+
monkeypatch.setattr(storage, "_write_embedding_snapshot", original_snapshot)
embedded_sources: list[str] = []
original_documents_to_nodes = storage.ingestion.documents_to_nodes
@@ -112,6 +116,18 @@ async def test_rebuild_resumes_completed_documents_after_failure(
assert resumed_run["documents"]["second.txt"]["embedding_reused"] is False
assert embedded_sources == [second.name]
storage.verify_persisted_index(candidate)
+ # Shape validation alone does not detect silently replacing the old index
+ # with one containing only the remaining sources.
+ restored = storage.vector_store.load_index(candidate)
+ retained = {
+ Path(node.metadata["file_path"]).name
+ for node in restored.docstore.docs.values()
+ if node.metadata.get("file_path")
+ }
+ assert retained == {first.name, second.name}
+ retained_text = "\n".join(node.get_content() for node in restored.docstore.docs.values())
+ assert "Fourier transforms" in retained_text
+ assert "sampling" in retained_text
assert checkpoint["signature"] == _signature().hash()