Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(wiki): treat MAP-extracted chunks as cache hits instead of missing (#19208)

Zhu Lei0c3902afdbea
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index 4d01f04c1e..619f8d6a6b 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -692,6 +692,7 @@ def _wiki_build_resume_doc(
doc_id: str,
per_chunk_extract: dict,
chunk_hash: str = "",
+ kb_id: str = "",
) -> dict:
"""Build the non-searchable ES doc that records a per-chunk MAP extract.
@@ -702,6 +703,10 @@ def _wiki_build_resume_doc(
``chunk_hash`` fingerprints the chunk's content as of extraction time.
The incremental MAP re-run reads it back and compares against the
current chunk's hash to decide whether to re-extract.
+
+ ``kb_id`` is stamped on the row because every doc-store search in this
+ pipeline injects a ``kb_id`` filter. A MAP row without it is invisible
+ to cache resolution even after a successful bulk write.
"""
content_with_weight = json.dumps(per_chunk_extract, ensure_ascii=False)
doc_id_str = str(doc_id)
@@ -710,6 +715,7 @@ def _wiki_build_resume_doc(
# identity lets A -> B -> A reuse the first extraction instead of
# replacing it when B is compiled.
"id": _stable_row_id(WIKI_MAP_COMPILE_KWD, doc_id_str, chunk_id, chunk_hash),
+ "kb_id": str(kb_id),
"doc_id": doc_id_str,
"compile_kwd": WIKI_MAP_COMPILE_KWD,
"source_chunk_ids": [chunk_id],
@@ -813,6 +819,7 @@ async def _wiki_persist_extracts(
doc_id,
extract,
chunk_hash=hashes.get(chunk_id, ""),
+ kb_id=kb_id,
)
for chunk_id, extract in per_chunk.items()
if chunk_id
diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_map_state.py b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_map_state.py
index 7961e819df..e3afd78ae2 100644
--- a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_map_state.py
+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_map_state.py
@@ -73,6 +73,50 @@ class _StateDocStore:
del self.rows[row_id]
+class _KbScopedDocStore(_StateDocStore):
+ """Mirrors ES/OpenSearch search: every query is pinned to ``kb_id``."""
+
+ def search(self, fields, highlights, condition, matches, order, offset, limit, index, dataset_ids):
+ scoped = dict(condition or {})
+ if dataset_ids:
+ scoped["kb_id"] = list(dataset_ids)
+ rows = []
+ for row in self.rows.values():
+ if scoped.get("kb_id") and row.get("kb_id") not in scoped["kb_id"]:
+ continue
+ if any(scoped.get(key) and row.get(key) not in scoped[key] for key in ("id", "compile_kwd", "type_kwd")):
+ continue
+ if scoped.get("doc_id") and row.get("doc_id") not in scoped["doc_id"]:
+ continue
+ if scoped.get("source_chunk_ids") and not set(row.get("source_chunk_ids") or []) & set(scoped["source_chunk_ids"]):
+ continue
+ if scoped.get("chunk_hash_kwd") and row.get("chunk_hash_kwd") not in scoped["chunk_hash_kwd"]:
+ continue
+ rows.append(row)
+ return rows[offset : offset + limit]
+
+
+# Reporter chunk ids from infiniflow/ragflow#19092. MAP logged extracted=9 then
+# immediately listed these same ids as missing_chunks.
+_REPORTER_CHUNK_IDS = [
+ "0b9beb4afcaf7717",
+ "2382483997f1183d",
+ "31c63a0feb6b8207",
+ "39feff69089a3d59",
+ "5467040776c88761",
+ "60ffd6504434bae2",
+ "87b76b7a1ebb3dc5",
+ "d96c875495acd52c",
+ "e96857ea65980dec",
+]
+
+
+def _missing_after_resolve(wiki, tenant_id, kb_id, state, target_chunk_ids):
+ resolved = asyncio.run(wiki._wiki_load_map_extracts_for_state(tenant_id, kb_id, state, target_chunk_ids))
+ resolved_ids = {str((extract.get("_map_version") or {}).get("chunk_id") or "") for extract in resolved}
+ return set(target_chunk_ids) - resolved_ids
+
+
def test_active_map_state_switches_marker_after_new_generation(monkeypatch):
wiki = _load_wiki_module(monkeypatch)
marker_id = wiki._stable_row_id(wiki.WIKI_MAP_STATE_META_COMPILE_KWD, "kb-1")
@@ -165,3 +209,163 @@ def test_failed_state_write_keeps_previous_generation_active(monkeypatch):
asyncio.run(wiki._wiki_commit_active_map_state("tenant-1", "kb-1", {"chunk-new": {"doc_id": "doc-new", "hash": "hash-new"}}))
assert asyncio.run(wiki._wiki_load_active_map_state("tenant-1", "kb-1")) == {"chunk-old": {"doc_id": "doc-old", "hash": "hash-old"}}
+
+
+def test_persisted_map_extracts_are_not_missing_from_completeness_check(monkeypatch):
+ """Successful MAP persist must satisfy the incremental completeness check.
+
+ Issue 19092: wiki_map logged requested=9 cache_hits=0 extracted=9, then the
+ follow-up cache resolution reported the same 9 source-chunk ids as missing.
+ The rows were written, but searches always filter by kb_id and the resume
+ doc omitted that field, so every extracted chunk looked absent.
+ """
+ wiki = _load_wiki_module(monkeypatch)
+ store = _KbScopedDocStore()
+ monkeypatch.setattr(sys.modules["common.settings"], "docStoreConn", store)
+
+ doc_id = "6dc70e7aa5a411f19c0109b856bc47ac"
+ kb_id = "ff675c34a5a111f19c0109b856bc47ac"
+ per_chunk = {chunk_id: wiki._wiki_empty_extract() for chunk_id in _REPORTER_CHUNK_IDS}
+ hashes = {chunk_id: f"hash-{chunk_id}" for chunk_id in _REPORTER_CHUNK_IDS}
+ state = {chunk_id: {"doc_id": doc_id, "hash": hashes[chunk_id]} for chunk_id in _REPORTER_CHUNK_IDS}
+ target = set(_REPORTER_CHUNK_IDS)
+
+ asyncio.run(wiki._wiki_persist_extracts(per_chunk, doc_id, "tenant-1", kb_id, chunk_hashes=hashes))
+
+ written = [row for batch in store.insert_calls for row in batch]
+ assert {row["kb_id"] for row in written} == {kb_id}
+ assert {row["source_chunk_ids"][0] for row in written} == target
+ assert _missing_after_resolve(wiki, "tenant-1", kb_id, state, target) == set()
+
+
+def test_completeness_check_still_reports_chunks_that_were_never_persisted(monkeypatch):
+ wiki = _load_wiki_module(monkeypatch)
+ store = _KbScopedDocStore()
+ monkeypatch.setattr(sys.modules["common.settings"], "docStoreConn", store)
+
+ doc_id = "doc-1"
+ kb_id = "kb-1"
+ persisted = _REPORTER_CHUNK_IDS[:-1]
+ missing_id = _REPORTER_CHUNK_IDS[-1]
+ hashes = {chunk_id: f"hash-{chunk_id}" for chunk_id in _REPORTER_CHUNK_IDS}
+ state = {chunk_id: {"doc_id": doc_id, "hash": hashes[chunk_id]} for chunk_id in _REPORTER_CHUNK_IDS}
+ per_chunk = {chunk_id: wiki._wiki_empty_extract() for chunk_id in persisted}
+
+ asyncio.run(wiki._wiki_persist_extracts(per_chunk, doc_id, "tenant-1", kb_id, chunk_hashes=hashes))
+
+ assert _missing_after_resolve(wiki, "tenant-1", kb_id, state, set(_REPORTER_CHUNK_IDS)) == {missing_id}
+
+
+def test_map_extract_without_kb_id_is_invisible_to_completeness_check(monkeypatch):
+ """Rows that omit kb_id must not count as resolved under a kb-scoped search."""
+ wiki = _load_wiki_module(monkeypatch)
+ store = _KbScopedDocStore()
+ monkeypatch.setattr(sys.modules["common.settings"], "docStoreConn", store)
+
+ chunk_id = _REPORTER_CHUNK_IDS[0]
+ doc_id = "doc-1"
+ kb_id = "kb-1"
+ chunk_hash = "hash-1"
+ store.rows["orphan"] = {
+ "id": "orphan",
+ "doc_id": doc_id,
+ "compile_kwd": wiki.WIKI_MAP_COMPILE_KWD,
+ "source_chunk_ids": [chunk_id],
+ "chunk_hash_kwd": chunk_hash,
+ "content_with_weight": json.dumps({"entities": []}),
+ }
+ state = {chunk_id: {"doc_id": doc_id, "hash": chunk_hash}}
+
+ assert _missing_after_resolve(wiki, "tenant-1", kb_id, state, {chunk_id}) == {chunk_id}
+
+
+def test_wiki_map_from_chunks_successful_extract_is_not_reported_missing(monkeypatch):
+ """Successful MAP through wiki_map_from_chunks must not look missing afterward.
+
+ Issue 19092: wiki_map logged extracted=9 then the generator completeness
+ check listed the same source-chunk ids as missing_chunks. The isolated
+ persist + lookup tests cover the row shape; this drives the public MAP
+ entry so extract → persist → kb-scoped completeness run as one flow.
+ Extraction is stubbed; persist and completeness use the real helpers
+ against a kb-scoped fake doc store.
+ """
+ wiki = _load_wiki_module(monkeypatch)
+ store = _KbScopedDocStore()
+ monkeypatch.setattr(sys.modules["common.settings"], "docStoreConn", store)
+
+ def _pack_batches(chunks, _chat_mdl, **kwargs):
+ resume = kwargs.get("resume_chunk_ids") or set()
+ picker = kwargs.get("chunk_text_picker") or (lambda chunk: chunk.get("text") or "")
+ packed = []
+ for idx, chunk in enumerate(chunks):
+ chunk_id = chunk.get("id") or chunk.get("chunk_id")
+ if not isinstance(chunk_id, str) or not chunk_id or chunk_id in resume:
+ continue
+ packed.append({"label": f"C{idx + 1}", "chunk_id": chunk_id, "text": picker(chunk) or ""})
+ return ([packed] if packed else [], {"n_batches": int(bool(packed))})
+
+ async def _run_pipeline(batches, *, process_batch, aggregate=None, **_kwargs):
+ if not batches:
+ return aggregate([]) if aggregate else []
+ results = [await process_batch(batch, idx, len(batches)) for idx, batch in enumerate(batches)]
+ return aggregate(results) if aggregate else results
+
+ extracted_from_llm = []
+
+ async def _extract_ok(packed, *_args, **_kwargs):
+ extracted_from_llm.extend(entry["chunk_id"] for entry in packed)
+ return wiki._wiki_empty_extract()
+
+ monkeypatch.setattr(wiki, "_build_chunk_batches", _pack_batches)
+ monkeypatch.setattr(wiki, "_run_chunked_pipeline", _run_pipeline)
+ monkeypatch.setattr(wiki, "_wiki_extract_one_batch", _extract_ok)
+
+ doc_id = "6dc70e7aa5a411f19c0109b856bc47ac"
+ kb_id = "ff675c34a5a111f19c0109b856bc47ac"
+ tenant_id = "tenant-1"
+ never_extracted_id = "never-extracted-chunk"
+ chunks = [{"id": chunk_id, "text": f"body of {chunk_id}"} for chunk_id in _REPORTER_CHUNK_IDS]
+ chat_mdl = SimpleNamespace(max_length=8192, llm_name="mock-chat")
+
+ first = asyncio.run(
+ wiki.wiki_map_from_chunks(
+ chunks=chunks,
+ chat_mdl=chat_mdl,
+ embd_mdl=None,
+ doc_id=doc_id,
+ tenant_id=tenant_id,
+ kb_id=kb_id,
+ )
+ )
+
+ assert first["_meta"]["requested"] == len(_REPORTER_CHUNK_IDS)
+ assert first["_meta"]["extracted"] == len(_REPORTER_CHUNK_IDS)
+ assert first["_meta"]["cache_hits"] == 0
+ assert set(extracted_from_llm) == set(_REPORTER_CHUNK_IDS)
+
+ written = [row for batch in store.insert_calls for row in batch]
+ assert {row["kb_id"] for row in written} == {kb_id}
+ assert {row["source_chunk_ids"][0] for row in written} == set(_REPORTER_CHUNK_IDS)
+
+ hashes = {chunk_id: wiki._chunk_hash(f"body of {chunk_id}") for chunk_id in _REPORTER_CHUNK_IDS}
+ state = {chunk_id: {"doc_id": doc_id, "hash": hashes[chunk_id]} for chunk_id in _REPORTER_CHUNK_IDS}
+ state[never_extracted_id] = {"doc_id": doc_id, "hash": "hash-never"}
+ target = set(_REPORTER_CHUNK_IDS) | {never_extracted_id}
+
+ # Same completeness path as run_wiki_incremental after MAP: load extracts
+ # for the current chunk state and treat unresolved ids as missing_chunks.
+ assert _missing_after_resolve(wiki, tenant_id, kb_id, state, target) == {never_extracted_id}
+
+ second = asyncio.run(
+ wiki.wiki_map_from_chunks(
+ chunks=chunks,
+ chat_mdl=chat_mdl,
+ embd_mdl=None,
+ doc_id=doc_id,
+ tenant_id=tenant_id,
+ kb_id=kb_id,
+ )
+ )
+ assert second["_meta"]["cache_hits"] == len(_REPORTER_CHUNK_IDS)
+ assert second["_meta"]["extracted"] == 0
+ assert extracted_from_llm == list(_REPORTER_CHUNK_IDS)