Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(lightrag): reuse only content-bound indexing cache (#1577)

Bingxi Zhao (Frank)0f4e4ae8a65d
diff --git a/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py
new file mode 100644
index 000000000..2a86fd867
--- /dev/null
+++ b/deeptutor/services/rag/pipelines/lightrag/cache_reuse.py
@@ -0,0 +1,126 @@
+"""Seed a rebuild with content-bound LightRAG indexing responses (#1577)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+import re
+from typing import Any
+
+from deeptutor.services.file_io import atomic_write_json
+
+from .engine import workspace_for
+
+logger = logging.getLogger(__name__)
+
+_CACHE_FILE = "kv_store_llm_response_cache.json"
+# Query answers and keyword extraction depend on the current graph and must
+# never cross a full rebuild. These types are produced from source content by
+# the pinned LightRAG SDK and include the model identity in their hash.
+_INDEX_TYPES = frozenset({"extract", "summary", "analysis", "smartheading"})
+_CACHE_KEY = re.compile(r"^[^:]+:([a-z]+):[0-9a-f]{32,64}$")
+
+
+def _policy_identity(policy: Any) -> tuple[str, str, str] | None:
+ """Read both current nested role fingerprints and older flat policies."""
+ if not isinstance(policy, dict):
+ return None
+ extract = policy.get("extract")
+ if isinstance(extract, dict):
+ vlm = policy.get("vlm") if isinstance(policy.get("vlm"), dict) else {}
+ vlm_snapshot = vlm.get("snapshot") if isinstance(vlm.get("snapshot"), dict) else {}
+ fingerprint = extract.get("fingerprint")
+ if isinstance(fingerprint, str) and fingerprint:
+ return (
+ fingerprint,
+ str(vlm.get("mode") or "disabled"),
+ str(vlm_snapshot.get("fingerprint") or ""),
+ )
+ fingerprint = policy.get("fingerprint")
+ if isinstance(fingerprint, str) and fingerprint:
+ return (fingerprint, "", "")
+ return None
+
+
+def _read_policy_identity(root: Path) -> tuple[str, str, str] | None:
+ try:
+ meta = json.loads((root / "meta.json").read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ # Interrupted versions have no publication marker but may already
+ # have a durable cache worth preserving.
+ return None
+ return _policy_identity(meta.get("indexing_policy")) if isinstance(meta, dict) else None
+
+
+def _index_entries(path: Path) -> dict[str, dict[str, Any]]:
+ if path.is_symlink() or not path.is_file():
+ return {}
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError) as exc:
+ logger.warning("Ignoring unreadable LightRAG cache %s: %s", path, type(exc).__name__)
+ return {}
+ if not isinstance(payload, dict):
+ return {}
+ entries: dict[str, dict[str, Any]] = {}
+ for key, value in payload.items():
+ match = _CACHE_KEY.fullmatch(key) if isinstance(key, str) else None
+ if (
+ match
+ and match.group(1) in _INDEX_TYPES
+ and isinstance(value, dict)
+ and value.get("cache_type") == match.group(1)
+ and isinstance(value.get("return"), str)
+ and value["return"]
+ ):
+ entries[key] = value
+ return entries
+
+
+def inherit_index_cache(kb_dir: Path, target_root: Path, policy: dict[str, Any]) -> bool:
+ """Carry only verified indexing responses into a fresh version workspace.
+
+ The newest interrupted version is considered even without ``meta.json``;
+ when older published versions are considered, matching nested role
+ fingerprints are preferred. Hash keys still guard against model drift.
+ """
+ target = target_root / workspace_for(target_root) / _CACHE_FILE
+ if target.exists():
+ return False
+ target_identity = _policy_identity(policy)
+ target_version = int(target_root.name.removeprefix("version-"))
+ candidates: list[tuple[int, int, Path]] = []
+ for root in kb_dir.glob("version-*"):
+ if not root.is_dir() or root == target_root:
+ continue
+ try:
+ version = int(root.name.removeprefix("version-"))
+ except ValueError:
+ continue
+ donor_identity = _read_policy_identity(root)
+ # The immediately preceding interrupted candidate may contain progress
+ # that was never published. Otherwise prefer matching role policies.
+ identity_rank = 1
+ if donor_identity is None and version == target_version - 1:
+ identity_rank = 3
+ elif donor_identity and target_identity:
+ identity_rank = 2 if donor_identity == target_identity else 0
+ paths = [root / _CACHE_FILE, *root.glob(f"deeptutor_*/{_CACHE_FILE}")]
+ for path in paths:
+ if path.is_file() and not path.is_symlink():
+ candidates.append((version, identity_rank, path))
+ if not candidates:
+ return False
+
+ for _, _, donor in sorted(candidates, key=lambda item: (item[1], item[0]), reverse=True):
+ entries = _index_entries(donor)
+ if not entries:
+ continue
+ atomic_write_json(target, entries)
+ logger.info("Inherited %d LightRAG indexing cache entries from %s", len(entries), donor)
+ return True
+ return False
+
+
+__all__ = ["inherit_index_cache"]
diff --git a/deeptutor/services/rag/pipelines/lightrag/pipeline.py b/deeptutor/services/rag/pipelines/lightrag/pipeline.py
index a9b88049b..68baaa93b 100644
--- a/deeptutor/services/rag/pipelines/lightrag/pipeline.py
+++ b/deeptutor/services/rag/pipelines/lightrag/pipeline.py
@@ -21,7 +21,7 @@ from deeptutor.services.rag.index_versioning import (
)
from deeptutor.services.rag.kb_paths import resolve_kb_dir
-from . import block_policy, engine, indexing_policy, ingress, storage
+from . import block_policy, cache_reuse, engine, indexing_policy, ingress, storage
from . import config as lr_config
from .indexing_policy import (
IndexingPolicyError,
@@ -432,6 +432,14 @@ class LightRagPipeline:
snapshot = indexing_policy.with_image_analysis(snapshot, kwargs["image_analysis"])
root_dir = resolve_storage_dir_for_rebuild(kb_dir, None)
try:
+ persisted_policy = getattr(snapshot, "persisted_policy", None)
+ if callable(persisted_policy):
+ await asyncio.to_thread(
+ cache_reuse.inherit_index_cache,
+ kb_dir,
+ root_dir,
+ persisted_policy(),
+ )
outcome = await self._run_indexing(
root_dir,
file_paths,
diff --git a/tests/services/rag/test_lightrag_cache_reuse.py b/tests/services/rag/test_lightrag_cache_reuse.py
new file mode 100644
index 000000000..86d17a0a1
--- /dev/null
+++ b/tests/services/rag/test_lightrag_cache_reuse.py
@@ -0,0 +1,103 @@
+"""Full rebuilds may reuse indexing calls without inheriting stale answers."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from deeptutor.services.rag.pipelines.lightrag.cache_reuse import inherit_index_cache
+from deeptutor.services.rag.pipelines.lightrag.engine import workspace_for
+
+
+def _policy(fingerprint: str) -> dict:
+ return {
+ "schema_version": 2,
+ "policy": "pinned",
+ "extract": {"fingerprint": fingerprint},
+ "vlm": {"mode": "disabled"},
+ }
+
+
+def _cache(root: Path, entries: dict, *, nested: bool = True) -> Path:
+ target = root / workspace_for(root) if nested else root
+ target.mkdir(parents=True, exist_ok=True)
+ path = target / "kv_store_llm_response_cache.json"
+ path.write_text(json.dumps(entries), encoding="utf-8")
+ return path
+
+
+def _entry(kind: str, value: str) -> dict:
+ return {"cache_type": kind, "return": value, "original_prompt": "content bound"}
+
+
+def test_rebuild_inherits_only_content_bound_index_responses(tmp_path: Path) -> None:
+ old = tmp_path / "version-1"
+ old.mkdir()
+ (old / "meta.json").write_text(json.dumps({"indexing_policy": _policy("same")}), "utf-8")
+ entries = {
+ f"default:extract:{'a' * 32}": _entry("extract", "entities"),
+ f"default:analysis:{'b' * 32}": _entry("analysis", "image description"),
+ f"default:summary:{'c' * 32}": _entry("summary", "entity summary"),
+ f"default:smartheading:{'d' * 32}": _entry("smartheading", "heading"),
+ f"global:query:{'e' * 32}": _entry("query", "stale answer"),
+ f"global:keywords:{'f' * 32}": _entry("keywords", "stale keywords"),
+ "bad": _entry("extract", "bad key"),
+ f"default:extract:{'1' * 32}": _entry("query", "mismatched type"),
+ }
+ _cache(old, entries)
+ target = tmp_path / "version-2"
+ target.mkdir()
+
+ assert inherit_index_cache(tmp_path, target, _policy("same")) is True
+
+ copied = json.loads(_cache_path(target).read_text(encoding="utf-8"))
+ assert len(copied) == 4
+ assert {value["cache_type"] for value in copied.values()} == {
+ "extract",
+ "analysis",
+ "summary",
+ "smartheading",
+ }
+ assert inherit_index_cache(tmp_path, target, _policy("same")) is False
+
+
+def _cache_path(root: Path) -> Path:
+ return root / workspace_for(root) / "kv_store_llm_response_cache.json"
+
+
+def test_interrupted_latest_version_without_meta_keeps_its_cache(tmp_path: Path) -> None:
+ published = tmp_path / "version-1"
+ published.mkdir()
+ (published / "meta.json").write_text(
+ json.dumps({"indexing_policy": _policy("same")}), encoding="utf-8"
+ )
+ _cache(published, {f"default:extract:{'a' * 32}": _entry("extract", "old")})
+ interrupted = tmp_path / "version-2"
+ _cache(interrupted, {f"default:extract:{'b' * 32}": _entry("extract", "new")})
+ target = tmp_path / "version-3"
+ target.mkdir()
+
+ assert inherit_index_cache(tmp_path, target, _policy("same")) is True
+ assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == "new"
+
+
+def test_matching_nested_role_policy_beats_newer_mismatch(tmp_path: Path) -> None:
+ matching = tmp_path / "version-1"
+ matching.mkdir()
+ (matching / "meta.json").write_text(
+ json.dumps({"indexing_policy": _policy("same")}), encoding="utf-8"
+ )
+ _cache(matching, {f"default:extract:{'a' * 32}": _entry("extract", "matching")}, nested=False)
+ different = tmp_path / "version-2"
+ different.mkdir()
+ (different / "meta.json").write_text(
+ json.dumps({"indexing_policy": _policy("different")}), encoding="utf-8"
+ )
+ _cache(different, {f"default:extract:{'b' * 32}": _entry("extract", "different")})
+ target = tmp_path / "version-3"
+ target.mkdir()
+
+ assert inherit_index_cache(tmp_path, target, _policy("same")) is True
+ assert next(iter(json.loads(_cache_path(target).read_text()).values()))["return"] == (
+ "matching"
+ )