DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: retain checkpoint index and recover interrupted snapshot swaps
Bingxi Zhao (Frank)0b6ff7abae6b
diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.pyindex 71e211507..dfef44ce8 100644--- a/deeptutor/services/rag/pipelines/llamaindex/storage.py+++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py@@ -46,6 +46,7 @@ class EmbeddingSourceRecord:class ResumeCandidate:storage_dir: Pathcompleted_keys: frozenset[str]+ index: Anydef _storage_path_from_version_entry(entry: dict[str, Any]) -> Path | None:@@ -132,10 +133,17 @@ def resolve_resumable_rebuild_dir(candidate.resolve().relative_to(kb_dir.resolve())except (OSError, ValueError):return None- if candidate.parent.resolve() != kb_dir.resolve() or not _is_unpublished_building_dir(- candidate+ if (+ candidate.parent.resolve() != kb_dir.resolve()+ or not candidate.name.startswith("version-")+ or (candidate / "meta.json").exists()):return None+ # A crash can occur after moving the candidate aside but before publishing+ # its replacement. Restore that checkpoint before testing its build marker.+ _recover_candidate_swap(candidate)+ if not _is_unpublished_building_dir(candidate):+ return Nonepayload = _read_embedding_checkpoint(candidate)if payload is None:@@ -215,7 +223,9 @@ def _resume_candidate(for key in completed_keys:if str(source_records[key].path.resolve()) not in indexed_paths:return None- return ResumeCandidate(storage_dir=storage_dir, completed_keys=frozenset(completed_keys))+ return ResumeCandidate(+ storage_dir=storage_dir, completed_keys=frozenset(completed_keys), index=index+ )def _fsync_directory(path: Path) -> None:@@ -342,7 +352,7 @@ def create_resumable_index(_recover_candidate_swap(storage_dir)resume = _resume_candidate(storage_dir, source_records)- index: Any = None+ index: Any = resume.index if resume else Nonecompleted_keys = set(resume.completed_keys if resume else set())if resume is not None and checkpoint_callback is not None:for key in sorted(completed_keys):diff --git a/tests/services/rag/test_llamaindex_embedding_resume.py b/tests/services/rag/test_llamaindex_embedding_resume.pyindex 9297695b1..0bbb41e02 100644--- a/tests/services/rag/test_llamaindex_embedding_resume.py+++ b/tests/services/rag/test_llamaindex_embedding_resume.py@@ -34,8 +34,9 @@ def _records(kb_dir: Path, names: list[str]) -> dict[str, object]:@pytest.mark.asyncio[email protected]("interrupted_swap", [False, True])async def test_rebuild_resumes_completed_documents_after_failure(- tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, interrupted_swap: bool) -> None:from llama_index.core import Settingsfrom llama_index.core.embeddings import MockEmbedding@@ -92,6 +93,9 @@ async def test_rebuild_resumes_completed_documents_after_failure(assert failed_run["documents"]["first.txt"]["status"] == "completed"assert failed_run["documents"]["first.txt"]["embedding_reused"] is False+ if interrupted_swap:+ candidate.rename(candidate.with_name(f".{candidate.name}.previous-crash"))+monkeypatch.setattr(storage, "_write_embedding_snapshot", original_snapshot)embedded_sources: list[str] = []original_documents_to_nodes = storage.ingestion.documents_to_nodes@@ -112,6 +116,18 @@ async def test_rebuild_resumes_completed_documents_after_failure(assert resumed_run["documents"]["second.txt"]["embedding_reused"] is Falseassert embedded_sources == [second.name]storage.verify_persisted_index(candidate)+ # Shape validation alone does not detect silently replacing the old index+ # with one containing only the remaining sources.+ restored = storage.vector_store.load_index(candidate)+ retained = {+ Path(node.metadata["file_path"]).name+ for node in restored.docstore.docs.values()+ if node.metadata.get("file_path")+ }+ assert retained == {first.name, second.name}+ retained_text = "\n".join(node.get_content() for node in restored.docstore.docs.values())+ assert "Fourier transforms" in retained_text+ assert "sampling" in retained_textassert checkpoint["signature"] == _signature().hash()