DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(lightrag): reuse only content-bound indexing cache (#1577)
Bingxi Zhao (Frank)0f4e4ae8a65d
diff --git a/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.pynew file mode 100644index 000000000..2a86fd867--- /dev/null+++ b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py@@ -0,0 +1,126 @@+"""Seed a rebuild with content-bound LightRAG indexing responses (#1577)."""++from __future__ import annotations++import json+import logging+from pathlib import Path+import re+from typing import Any++from deeptutor.services.file_io import atomic_write_json++from .engine import workspace_for++logger = logging.getLogger(__name__)++_CACHE_FILE = "kv_store_llm_response_cache.json"+# Query answers and keyword extraction depend on the current graph and must+# never cross a full rebuild. These types are produced from source content by+# the pinned LightRAG SDK and include the model identity in their hash.+_INDEX_TYPES = frozenset({"extract", "summary", "analysis", "smartheading"})+_CACHE_KEY = re.compile(r"^[^:]+:([a-z]+):[0-9a-f]{32,64}$")+++def _policy_identity(policy: Any) -> tuple[str, str, str] | None:+ """Read both current nested role fingerprints and older flat policies."""+ if not isinstance(policy, dict):+ return None+ extract = policy.get("extract")+ if isinstance(extract, dict):+ vlm = policy.get("vlm") if isinstance(policy.get("vlm"), dict) else {}+ vlm_snapshot = vlm.get("snapshot") if isinstance(vlm.get("snapshot"), dict) else {}+ fingerprint = extract.get("fingerprint")+ if isinstance(fingerprint, str) and fingerprint:+ return (+ fingerprint,+ str(vlm.get("mode") or "disabled"),+ str(vlm_snapshot.get("fingerprint") or ""),+ )+ fingerprint = policy.get("fingerprint")+ if isinstance(fingerprint, str) and fingerprint:+ return (fingerprint, "", "")+ return None+++def _read_policy_identity(root: Path) -> tuple[str, str, str] | None:+ try:+ meta = json.loads((root / "meta.json").read_text(encoding="utf-8"))+ except (OSError, ValueError):+ # Interrupted versions have no publication marker but may already+ # have a durable cache worth preserving.+ return None+ return _policy_identity(meta.get("indexing_policy")) if isinstance(meta, dict) else None+++def _index_entries(path: Path) -> dict[str, dict[str, Any]]:+ if path.is_symlink() or not path.is_file():+ return {}+ try:+ payload = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, ValueError) as exc:+ logger.warning("Ignoring unreadable LightRAG cache %s: %s", path, type(exc).__name__)+ return {}+ if not isinstance(payload, dict):+ return {}+ entries: dict[str, dict[str, Any]] = {}+ for key, value in payload.items():+ match = _CACHE_KEY.fullmatch(key) if isinstance(key, str) else None+ if (+ match+ and match.group(1) in _INDEX_TYPES+ and isinstance(value, dict)+ and value.get("cache_type") == match.group(1)+ and isinstance(value.get("return"), str)+ and value["return"]+ ):+ entries[key] = value+ return entries+++def inherit_index_cache(kb_dir: Path, target_root: Path, policy: dict[str, Any]) -> bool:+ """Carry only verified indexing responses into a fresh version workspace.++ The newest interrupted version is considered even without ``meta.json``;+ when older published versions are considered, matching nested role+ fingerprints are preferred. Hash keys still guard against model drift.+ """+ target = target_root / workspace_for(target_root) / _CACHE_FILE+ if target.exists():+ return False+ target_identity = _policy_identity(policy)+ target_version = int(target_root.name.removeprefix("version-"))+ candidates: list[tuple[int, int, Path]] = []+ for root in kb_dir.glob("version-*"):+ if not root.is_dir() or root == target_root:+ continue+ try:+ version = int(root.name.removeprefix("version-"))+ except ValueError:+ continue+ donor_identity = _read_policy_identity(root)+ # The immediately preceding interrupted candidate may contain progress+ # that was never published. Otherwise prefer matching role policies.+ identity_rank = 1+ if donor_identity is None and version == target_version - 1:+ identity_rank = 3+ elif donor_identity and target_identity:+ identity_rank = 2 if donor_identity == target_identity else 0+ paths = [root / _CACHE_FILE, *root.glob(f"deeptutor_*/{_CACHE_FILE}")]+ for path in paths:+ if path.is_file() and not path.is_symlink():+ candidates.append((version, identity_rank, path))+ if not candidates:+ return False++ for _, _, donor in sorted(candidates, key=lambda item: (item[1], item[0]), reverse=True):+ entries = _index_entries(donor)+ if not entries:+ continue+ atomic_write_json(target, entries)+ logger.info("Inherited %d LightRAG indexing cache entries from %s", len(entries), donor)+ return True+ return False+++__all__ = ["inherit_index_cache"]diff --git a/deeptutor/services/rag/pipelines/lightrag/pipeline.py b/deeptutor/services/rag/pipelines/lightrag/pipeline.pyindex a9b88049b..68baaa93b 100644--- a/deeptutor/services/rag/pipelines/lightrag/pipeline.py+++ b/deeptutor/services/rag/pipelines/lightrag/pipeline.py@@ -21,7 +21,7 @@ from deeptutor.services.rag.index_versioning import ()from deeptutor.services.rag.kb_paths import resolve_kb_dir-from . import block_policy, engine, indexing_policy, ingress, storage+from . import block_policy, cache_reuse, engine, indexing_policy, ingress, storagefrom . import config as lr_configfrom .indexing_policy import (IndexingPolicyError,@@ -432,6 +432,14 @@ class LightRagPipeline:snapshot = indexing_policy.with_image_analysis(snapshot, kwargs["image_analysis"])root_dir = resolve_storage_dir_for_rebuild(kb_dir, None)try:+ persisted_policy = getattr(snapshot, "persisted_policy", None)+ if callable(persisted_policy):+ await asyncio.to_thread(+ cache_reuse.inherit_index_cache,+ kb_dir,+ root_dir,+ persisted_policy(),+ )outcome = await self._run_indexing(root_dir,file_paths,diff --git a/tests/services/rag/test_lightrag_cache_reuse.py b/tests/services/rag/test_lightrag_cache_reuse.pynew file mode 100644index 000000000..86d17a0a1--- /dev/null+++ b/tests/services/rag/test_lightrag_cache_reuse.py@@ -0,0 +1,103 @@+"""Full rebuilds may reuse indexing calls without inheriting stale answers."""++from __future__ import annotations++import json+from pathlib import Path++from deeptutor.services.rag.pipelines.lightrag.cache_reuse import inherit_index_cache+from deeptutor.services.rag.pipelines.lightrag.engine import workspace_for+++def _policy(fingerprint: str) -> dict:+ return {+ "schema_version": 2,+ "policy": "pinned",+ "extract": {"fingerprint": fingerprint},+ "vlm": {"mode": "disabled"},+ }+++def _cache(root: Path, entries: dict, *, nested: bool = True) -> Path:+ target = root / workspace_for(root) if nested else root+ target.mkdir(parents=True, exist_ok=True)+ path = target / "kv_store_llm_response_cache.json"+ path.write_text(json.dumps(entries), encoding="utf-8")+ return path+++def _entry(kind: str, value: str) -> dict:+ return {"cache_type": kind, "return": value, "original_prompt": "content bound"}+++def test_rebuild_inherits_only_content_bound_index_responses(tmp_path: Path) -> None:+ old = tmp_path / "version-1"+ old.mkdir()+ (old / "meta.json").write_text(json.dumps({"indexing_policy": _policy("same")}), "utf-8")+ entries = {+ f"default:extract:{'a' * 32}": _entry("extract", "entities"),+ f"default:analysis:{'b' * 32}": _entry("analysis", "image description"),+ f"default:summary:{'c' * 32}": _entry("summary", "entity summary"),+ f"default:smartheading:{'d' * 32}": _entry("smartheading", "heading"),+ f"global:query:{'e' * 32}": _entry("query", "stale answer"),+ f"global:keywords:{'f' * 32}": _entry("keywords", "stale keywords"),+ "bad": _entry("extract", "bad key"),+ f"default:extract:{'1' * 32}": _entry("query", "mismatched type"),+ }+ _cache(old, entries)+ target = tmp_path / "version-2"+ target.mkdir()++ assert inherit_index_cache(tmp_path, target, _policy("same")) is True++ copied = json.loads(_cache_path(target).read_text(encoding="utf-8"))+ assert len(copied) == 4+ assert {value["cache_type"] for value in copied.values()} == {+ "extract",+ "analysis",+ "summary",+ "smartheading",+ }+ assert inherit_index_cache(tmp_path, target, _policy("same")) is False+++def _cache_path(root: Path) -> Path:+ return root / workspace_for(root) / "kv_store_llm_response_cache.json"+++def test_interrupted_latest_version_without_meta_keeps_its_cache(tmp_path: Path) -> None:+ published = tmp_path / "version-1"+ published.mkdir()+ (published / "meta.json").write_text(+ json.dumps({"indexing_policy": _policy("same")}), encoding="utf-8"+ )+ _cache(published, {f"default:extract:{'a' * 32}": _entry("extract", "old")})+ interrupted = tmp_path / "version-2"+ _cache(interrupted, {f"default:extract:{'b' * 32}": _entry("extract", "new")})+ target = tmp_path / "version-3"+ target.mkdir()++ assert inherit_index_cache(tmp_path, target, _policy("same")) is True+ assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == "new"+++def test_matching_nested_role_policy_beats_newer_mismatch(tmp_path: Path) -> None:+ matching = tmp_path / "version-1"+ matching.mkdir()+ (matching / "meta.json").write_text(+ json.dumps({"indexing_policy": _policy("same")}), encoding="utf-8"+ )+ _cache(matching, {f"default:extract:{'a' * 32}": _entry("extract", "matching")}, nested=False)+ different = tmp_path / "version-2"+ different.mkdir()+ (different / "meta.json").write_text(+ json.dumps({"indexing_policy": _policy("different")}), encoding="utf-8"+ )+ _cache(different, {f"default:extract:{'b' * 32}": _entry("extract", "different")})+ target = tmp_path / "version-3"+ target.mkdir()++ assert inherit_index_cache(tmp_path, target, _policy("same")) is True+ assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == (+ "matching"+ )