RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(wiki): stamp retrieval fields so compiled pages are searchable (#18787)
Sourav Rajviff667331ac20
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.pyindex c7535aa0e5..cd1c1fa01a 100644--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py@@ -503,7 +503,10 @@ def _build_canonical_entity_doc(source_chunk_ids: list[str] | None = None,) -> dict:"""Build a canonical entity row for insert or update."""+ from rag.nlp import rag_tokenizer+dim = len(embedding) if embedding else 768+ content_ltks = rag_tokenizer.tokenize(entity_name) if entity_name else ""doc = {"id": _stable_row_id(WIKI_CANONICAL_ENTITY_COMPILE_KWD, kb_id, entity_name),"entity_kwd": entity_name,@@ -514,6 +517,11 @@ def _build_canonical_entity_doc("mention_count_int": claim_count,"compile_kwd": WIKI_CANONICAL_ENTITY_COMPILE_KWD,"kb_id": kb_id,+ "doc_id": kb_id, # KB-scoped sentinel; real provenance is source_doc_ids+ "available_int": 1,+ "content_with_weight": entity_name,+ "content_ltks": content_ltks,+ "content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks) if content_ltks else "",}if embedding is not None:vec_col = f"q_{dim}_vec"@@ -2175,9 +2183,12 @@ async def _wiki_refine_page(page = {"id": _stable_row_id(WIKI_PAGE_COMPILE_KWD, kb_id, page_id),+ "kb_id": kb_id,+ "doc_id": kb_id, # KB-scoped sentinel; real provenance is source_doc_ids"slug_kwd": page_id,"title_kwd": title,"md_with_weight": content,+ "content_with_weight": content,"summary_with_weight": summary or title,"entity_names_kwd": sorted(set(entity_names or [page_title])),"source_chunk_ids": sorted(source_chunk_ids),@@ -2192,6 +2203,7 @@ async def _wiki_refine_page("title_tks": rag_tokenizer.tokenize(title),"content_ltks": content_ltks,"content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),+ "available_int": 1,}# Insert vector (adds q_{dim}_vec field)vec_col = f"q_{vec_dim}_vec"@@ -3361,6 +3373,13 @@ async def _wiki_finalize(outlinks = outlink_map.get(pid) or []update["outlinks_kwd"] = list(outlinks)update["outlinks_int"] = len(outlinks)+ # Backfill retrieval fields on every FINALIZE so pages written before+ # available_int/doc_id were stamped become visible to RAG on re-run.+ update["available_int"] = 1+ update["doc_id"] = kb_id+ body = rendered_content or original or page.get("md_with_weight") or ""+ if body:+ update["content_with_weight"] = bodyawait thread_pool_exec(settings.docStoreConn.update,diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex ad02320827..1d2246f612 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -47,6 +47,34 @@ def index_name(uid):return f"ragflow_{uid}"+def _chunk_scalar(value) -> str:+ """Normalize a doc-store scalar that Infinity may return as a one-item list."""+ if isinstance(value, (list, tuple)):+ value = value[0] if value else ""+ return str(value or "").strip()+++def is_kb_scoped_chunk(chunk: dict | None) -> bool:+ """True for compilation/graph rows keyed to the KB, not a source document.++ Wiki persist stamps ``doc_id = kb_id``. Those rows must not be treated as+ leftover chunks from a deleted ``document`` row. A missing ``doc_id`` is+ not enough on its own: only an explicit compilation marker (``compile_kwd``)+ or the KB-id sentinel bypasses deleted-document pruning. Otherwise stale+ ordinary chunks without ``doc_id`` would stay retrievable after their+ source document is deleted.+ """+ if not isinstance(chunk, dict):+ return False+ doc_id = _chunk_scalar(chunk.get("doc_id"))+ kb_id = _chunk_scalar(chunk.get("kb_id"))+ if kb_id and doc_id == kb_id:+ return True+ if doc_id:+ return False+ return bool(_chunk_scalar(chunk.get("compile_kwd")))++class Dealer:# Short-lived cache of "doc_id exists in MySQL" used by _prune_deleted_chunks.# Every retrieval would otherwise hit MySQL per query (fan-out searches and the@@ -124,12 +152,39 @@ class Dealer:# is removed but the vector record is not fully cleaned up. We filter those# chunks here so chat/retrieval does not surface content from deleted docs.# Keep this as a fallback, not as the primary delete mechanism.- chunk_doc_ids = [chunk.get("doc_id") for chunk in sres.field.values() if chunk and chunk.get("doc_id")]- if not chunk_doc_ids:+ fields = sres.field or {}+ aligned_ids = [chunk_id for chunk_id in sres.ids if fields.get(chunk_id)]+ if len(aligned_ids) != len(sres.ids):+ fields = {chunk_id: fields[chunk_id] for chunk_id in aligned_ids}+ highlight = sres.highlight+ if highlight:+ highlight = {chunk_id: highlight[chunk_id] for chunk_id in aligned_ids if chunk_id in highlight}+ sres = self.SearchResult(+ total=len(aligned_ids),+ ids=aligned_ids,+ query_vector=sres.query_vector,+ field=fields,+ highlight=highlight,+ aggregation=sres.aggregation,+ keywords=sres.keywords,+ group_docs=sres.group_docs,+ )++ chunk_doc_ids = []+ unscoped_missing_doc_id = False+ for chunk in fields.values():+ if not chunk or is_kb_scoped_chunk(chunk):+ continue+ doc_id = _chunk_scalar(chunk.get("doc_id"))+ if doc_id:+ chunk_doc_ids.append(doc_id)+ else:+ unscoped_missing_doc_id = True+ if not chunk_doc_ids and not unscoped_missing_doc_id:return sres- existing_doc_ids = await self._existing_doc_ids(chunk_doc_ids)- if len(existing_doc_ids) == len(set(chunk_doc_ids)):+ existing_doc_ids = await self._existing_doc_ids(chunk_doc_ids) if chunk_doc_ids else set()+ if chunk_doc_ids and not unscoped_missing_doc_id and len(existing_doc_ids) == len(set(chunk_doc_ids)):return sresfiltered_ids = []@@ -138,8 +193,17 @@ class Dealer:removed = 0for chunk_id in sres.ids:- chunk = sres.field.get(chunk_id)- if not chunk or chunk.get("doc_id") not in existing_doc_ids:+ chunk = fields.get(chunk_id)+ if not chunk:+ removed += 1+ continue+ if is_kb_scoped_chunk(chunk):+ filtered_ids.append(chunk_id)+ filtered_field[chunk_id] = chunk+ if sres.highlight and chunk_id in sres.highlight:+ filtered_highlight[chunk_id] = sres.highlight[chunk_id]+ continue+ if _chunk_scalar(chunk.get("doc_id")) not in existing_doc_ids:removed += 1continuediff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.pyindex 8d81d0b89f..7dd7422f06 100644--- a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py@@ -1117,6 +1117,9 @@ async def test_finalize_writes_outlinks():# the old-mode writer) which Infinity stores/reads back correctly.assert upd["outlinks_kwd"] == ["concept/B"], f"Got {upd['outlinks_kwd']}"assert upd["outlinks_int"] == 1+ assert upd["available_int"] == 1+ assert upd["doc_id"] == "kb1"+ assert upd["content_with_weight"]breakelse:pytest.fail("No update for concept/A found")@@ -1545,6 +1548,11 @@ def test_refine_page_persists_source_doc_ids_as_array():inserted_page = doc_store.insert.call_args.args[0][0]assert inserted_page["source_doc_ids"] == ["doc_1", "doc_2"]assert inserted_page["topic_kwd"] == "Technology companies"+ assert inserted_page["available_int"] == 1+ assert inserted_page["doc_id"] == "kb1"+ assert inserted_page["kb_id"] == "kb1"+ assert inserted_page["content_with_weight"] == inserted_page["md_with_weight"]+ assert inserted_page["content_with_weight"]def test_topics_for_docs_only_returns_source_scoped_candidates():@@ -2075,6 +2083,31 @@ async def test_reduce_entity_deletes_page_when_last_chunk_is_removed():assert [claim["statement"] for claim in delta["retractions"]] == ["only claim"]+def test_canonical_entity_doc_stamps_retrieval_fields():+ tokenizer = MagicMock()+ tokenizer.tokenize.side_effect = lambda text: text+ tokenizer.fine_grained_tokenize.side_effect = lambda text: text+ with patch("rag.nlp.rag_tokenizer", tokenizer, create=True):+ doc = _wiki._build_canonical_entity_doc(+ tenant_id="t1",+ kb_id="kb1",+ entity_name="Apple",+ entity_type="org",+ aliases=["Apple Inc."],+ source_doc_ids=["doc_1"],+ claim_count=2,+ embedding=[0.1, 0.2],+ )++ assert doc["available_int"] == 1+ assert doc["doc_id"] == "kb1"+ assert doc["kb_id"] == "kb1"+ assert doc["content_with_weight"] == "Apple"+ assert doc["content_ltks"] == "Apple"+ assert doc["content_sm_ltks"] == "Apple"+ assert doc["q_2_vec"] == [0.1, 0.2]++def test_as_int_accepts_string_doc_store_values():assert _wiki._as_int("7") == 7assert _wiki._as_int(None, 3) == 3diff --git a/test/unit_test/rag/test_prune_deleted_chunks.py b/test/unit_test/rag/test_prune_deleted_chunks.pynew file mode 100644index 0000000000..e61ea08d62--- /dev/null+++ b/test/unit_test/rag/test_prune_deleted_chunks.py@@ -0,0 +1,242 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""KB-scoped wiki/compilation rows must survive deleted-document pruning."""++import importlib.util+import sys+import types+from pathlib import Path++import pytest++_ROOT = Path(__file__).parents[3]+++def _load_search():+ fake_rag_nlp = types.ModuleType("rag.nlp")+ fake_rag_nlp.__path__ = []++ fake_tokenizer = types.ModuleType("rag.nlp.rag_tokenizer")+ fake_rag_nlp.rag_tokenizer = fake_tokenizer++ fake_query = types.ModuleType("rag.nlp.query")++ class _DummyFulltextQueryer:+ pass++ fake_query.FulltextQueryer = _DummyFulltextQueryer+ fake_rag_nlp.query = fake_query+ fake_settings = types.ModuleType("common.settings")++ stub_names = ("rag.nlp", "rag.nlp.rag_tokenizer", "rag.nlp.query", "common.settings")+ saved = {name: sys.modules.get(name) for name in stub_names}+ sys.modules["rag.nlp"] = fake_rag_nlp+ sys.modules["rag.nlp.rag_tokenizer"] = fake_tokenizer+ sys.modules["rag.nlp.query"] = fake_query+ sys.modules["common.settings"] = fake_settings++ import common++ saved_common_settings = getattr(common, "settings", None)+ common.settings = fake_settings++ spec = importlib.util.spec_from_file_location("rag.nlp.search_prune_test", _ROOT / "rag" / "nlp" / "search.py")+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ sys.modules["rag.nlp.search_prune_test"] = module+ try:+ spec.loader.exec_module(module)+ finally:+ if saved_common_settings is None:+ if getattr(common, "settings", None) is fake_settings:+ delattr(common, "settings")+ else:+ common.settings = saved_common_settings+ for name, previous in saved.items():+ if previous is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = previous+ return module+++_search = _load_search()+Dealer = _search.Dealer+is_kb_scoped_chunk = _search.is_kb_scoped_chunk+++def test_is_kb_scoped_chunk_missing_or_empty_doc_id():+ assert is_kb_scoped_chunk({"kb_id": "kb1"}) is False+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": ""}) is False+++def test_is_kb_scoped_chunk_compilation_row_without_doc_id():+ assert is_kb_scoped_chunk({"kb_id": "kb1", "compile_kwd": "wiki_page"}) is True+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "", "compile_kwd": ["wiki_canonical_entity"]}) is True+++def test_is_kb_scoped_chunk_sentinel_matches_kb():+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "kb1"}) is True+ assert is_kb_scoped_chunk({"kb_id": ["kb1"], "doc_id": ["kb1"]}) is True+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "kb1", "compile_kwd": "wiki_page"}) is True+++def test_is_kb_scoped_chunk_real_document():+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "doc1"}) is False+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "doc1", "compile_kwd": "wiki_doc_page_source"}) is False+ assert is_kb_scoped_chunk(None) is False+++def _result(ids, field):+ return Dealer.SearchResult(total=len(ids), ids=list(ids), field=field)+++async def test_prune_keeps_wiki_pages_when_mixed_with_source_chunks():+ dealer = Dealer(dataStore=None)++ async def existing(doc_ids):+ return {d for d in doc_ids if d == "doc-live"}++ dealer._existing_doc_ids = existing+ sres = _result(+ ["src-live", "src-dead", "wiki-page"],+ {+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},+ "src-dead": {"doc_id": "doc-gone", "kb_id": "kb1"},+ "wiki-page": {"doc_id": "kb1", "kb_id": "kb1", "compile_kwd": "wiki_page"},+ },+ )+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["src-live", "wiki-page"]+ assert "src-dead" not in out.field+ assert "wiki-page" in out.field+++async def test_prune_keeps_wiki_pages_that_omit_doc_id():+ dealer = Dealer(dataStore=None)++ async def existing(doc_ids):+ return {d for d in doc_ids if d == "doc-live"}++ dealer._existing_doc_ids = existing+ sres = _result(+ ["src-live", "wiki-page"],+ {+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},+ },+ )+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["src-live", "wiki-page"]+++async def test_prune_drops_ordinary_chunks_that_omit_doc_id():+ dealer = Dealer(dataStore=None)+ called = []++ async def existing(doc_ids):+ called.append(list(doc_ids))+ return {d for d in doc_ids if d == "doc-live"}++ dealer._existing_doc_ids = existing+ sres = _result(+ ["src-live", "src-orphan", "wiki-page"],+ {+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},+ "src-orphan": {"kb_id": "kb1"},+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},+ },+ )+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["src-live", "wiki-page"]+ assert "src-orphan" not in out.field+ assert called == [["doc-live"]]+++async def test_prune_drops_orphan_chunks_when_every_hit_omits_doc_id():+ dealer = Dealer(dataStore=None)+ called = []++ async def existing(doc_ids):+ called.append(list(doc_ids))+ return set()++ dealer._existing_doc_ids = existing+ sres = _result(+ ["src-orphan", "wiki-page"],+ {+ "src-orphan": {"kb_id": "kb1", "doc_id": ""},+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},+ },+ )+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["wiki-page"]+ assert "src-orphan" not in out.field+ assert called == []+++async def test_prune_drops_all_ordinary_chunks_without_doc_id():+ dealer = Dealer(dataStore=None)+ called = []++ async def existing(doc_ids):+ called.append(list(doc_ids))+ return set()++ dealer._existing_doc_ids = existing+ sres = _result(["src-orphan"], {"src-orphan": {"kb_id": "kb1"}})+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == []+ assert out.field == {}+ assert called == []+++async def test_prune_skips_lookup_when_all_hits_are_kb_scoped():+ dealer = Dealer(dataStore=None)+ called = []++ async def existing(doc_ids):+ called.append(list(doc_ids))+ return set()++ dealer._existing_doc_ids = existing+ sres = _result(["wiki"], {"wiki": {"doc_id": "kb1", "kb_id": "kb1"}})+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["wiki"]+ assert called == []+++async def test_prune_drops_ids_missing_from_fields_when_all_hits_are_kb_scoped():+ dealer = Dealer(dataStore=None)+ called = []++ async def existing(doc_ids):+ called.append(list(doc_ids))+ return set()++ dealer._existing_doc_ids = existing+ sres = _result(["wiki", "ghost"], {"wiki": {"doc_id": "kb1", "kb_id": "kb1"}})+ out = await dealer._prune_deleted_chunks(sres)+ assert out.ids == ["wiki"]+ assert "ghost" not in out.field+ assert called == []