DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(lightrag): merge compatible indexing caches across versions (#1577)
Bingxi Zhao (Frank)9073cd32f0ad
diff --git a/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.pyindex 2a86fd867..f276d2366 100644--- a/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py+++ b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py@@ -81,16 +81,17 @@ def _index_entries(path: Path) -> dict[str, dict[str, Any]]:def inherit_index_cache(kb_dir: Path, target_root: Path, policy: dict[str, Any]) -> bool:"""Carry only verified indexing responses into a fresh version workspace.- The newest interrupted version is considered even without ``meta.json``;- when older published versions are considered, matching nested role- fingerprints are preferred. Hash keys still guard against model drift.+ Merge matching published versions with the immediately preceding+ interrupted version, which may have no ``meta.json``. A partial interrupted+ cache must not hide the rest of an older published cache. Newer entries win+ on duplicate content keys; query answers never enter the merge."""target = target_root / workspace_for(target_root) / _CACHE_FILEif target.exists():return Falsetarget_identity = _policy_identity(policy)target_version = int(target_root.name.removeprefix("version-"))- candidates: list[tuple[int, int, Path]] = []+ candidates: list[tuple[int, int, int, Path]] = []for root in kb_dir.glob("version-*"):if not root.is_dir() or root == target_root:continue@@ -98,29 +99,39 @@ def inherit_index_cache(kb_dir: Path, target_root: Path, policy: dict[str, Any])version = int(root.name.removeprefix("version-"))except ValueError:continue+ if version >= target_version:+ continuedonor_identity = _read_policy_identity(root)- # The immediately preceding interrupted candidate may contain progress- # that was never published. Otherwise prefer matching role policies.- identity_rank = 1+ # An unpublished predecessor may already have useful content-addressed+ # extraction calls. Published versions must match the current roles.if donor_identity is None and version == target_version - 1:identity_rank = 3- elif donor_identity and target_identity:- identity_rank = 2 if donor_identity == target_identity else 0+ elif donor_identity is not None and (+ target_identity is None or donor_identity == target_identity+ ):+ identity_rank = 2+ else:+ continuepaths = [root / _CACHE_FILE, *root.glob(f"deeptutor_*/{_CACHE_FILE}")]for path in paths:if path.is_file() and not path.is_symlink():- candidates.append((version, identity_rank, path))+ candidates.append((identity_rank, version, int(path.parent != root), path))if not candidates:return False- for _, _, donor in sorted(candidates, key=lambda item: (item[1], item[0]), reverse=True):+ merged: dict[str, dict[str, Any]] = {}+ used = 0+ for _, _, _, donor in sorted(candidates):entries = _index_entries(donor)if not entries:continue- atomic_write_json(target, entries)- logger.info("Inherited %d LightRAG indexing cache entries from %s", len(entries), donor)- return True- return False+ merged.update(entries)+ used += 1+ if not merged:+ return False+ atomic_write_json(target, merged)+ logger.info("Inherited %d LightRAG indexing cache entries from %d donors", len(merged), used)+ return True__all__ = ["inherit_index_cache"]diff --git a/tests/services/rag/test_lightrag_cache_reuse.py b/tests/services/rag/test_lightrag_cache_reuse.pyindex 86d17a0a1..85d276971 100644--- a/tests/services/rag/test_lightrag_cache_reuse.py+++ b/tests/services/rag/test_lightrag_cache_reuse.py@@ -78,7 +78,39 @@ def test_interrupted_latest_version_without_meta_keeps_its_cache(tmp_path: Path)target.mkdir()assert inherit_index_cache(tmp_path, target, _policy("same")) is True- assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == "new"+ copied = json.loads(_cache_path(target).read_text())+ assert {item["return"] for item in copied.values()} == {"old", "new"}+++def test_partial_interrupted_cache_merges_with_published_extractions(tmp_path: Path) -> None:+ published = tmp_path / "version-1"+ published.mkdir()+ (published / "meta.json").write_text(+ json.dumps({"indexing_policy": _policy("same")}), encoding="utf-8"+ )+ old_entries = {+ f"default:extract:{index:032x}": _entry("extract", f"old-{index}") for index in range(900)+ }+ old_entries[f"global:query:{'f' * 32}"] = _entry("query", "stale answer")+ _cache(published, old_entries)++ interrupted = tmp_path / "version-2"+ recent_entries = {+ f"default:extract:{index:032x}": _entry("extract", f"new-{index}")+ for index in range(895, 905)+ }+ recent_entries[f"global:keywords:{'e' * 32}"] = _entry("keywords", "stale keywords")+ _cache(interrupted, recent_entries)+ target = tmp_path / "version-3"+ target.mkdir()++ assert inherit_index_cache(tmp_path, target, _policy("same")) is True+ copied = json.loads(_cache_path(target).read_text())+ assert len(copied) == 905+ assert copied[f"default:extract:{0:032x}"]["return"] == "old-0"+ assert copied[f"default:extract:{895:032x}"]["return"] == "new-895"+ assert copied[f"default:extract:{904:032x}"]["return"] == "new-904"+ assert all(value["cache_type"] == "extract" for value in copied.values())def test_matching_nested_role_policy_beats_newer_mismatch(tmp_path: Path) -> None:@@ -98,6 +130,6 @@ def test_matching_nested_role_policy_beats_newer_mismatch(tmp_path: Path) -> Nontarget.mkdir()assert inherit_index_cache(tmp_path, target, _policy("same")) is True- assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == (+ assert {value["return"] for value in json.loads(_cache_path(target).read_text()).values()} == {"matching"- )+ }