Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(wiki): stamp retrieval fields so compiled pages are searchable (#18787)

Sourav Rajviff667331ac20
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
index c7535aa0e5..cd1c1fa01a 100644
--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py
+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
@@ -503,7 +503,10 @@ def _build_canonical_entity_doc(
source_chunk_ids: list[str] | None = None,
) -> dict:
"""Build a canonical entity row for insert or update."""
+ from rag.nlp import rag_tokenizer
+
dim = len(embedding) if embedding else 768
+ content_ltks = rag_tokenizer.tokenize(entity_name) if entity_name else ""
doc = {
"id": _stable_row_id(WIKI_CANONICAL_ENTITY_COMPILE_KWD, kb_id, entity_name),
"entity_kwd": entity_name,
@@ -514,6 +517,11 @@ def _build_canonical_entity_doc(
"mention_count_int": claim_count,
"compile_kwd": WIKI_CANONICAL_ENTITY_COMPILE_KWD,
"kb_id": kb_id,
+ "doc_id": kb_id, # KB-scoped sentinel; real provenance is source_doc_ids
+ "available_int": 1,
+ "content_with_weight": entity_name,
+ "content_ltks": content_ltks,
+ "content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks) if content_ltks else "",
}
if embedding is not None:
vec_col = f"q_{dim}_vec"
@@ -2175,9 +2183,12 @@ async def _wiki_refine_page(
page = {
"id": _stable_row_id(WIKI_PAGE_COMPILE_KWD, kb_id, page_id),
+ "kb_id": kb_id,
+ "doc_id": kb_id, # KB-scoped sentinel; real provenance is source_doc_ids
"slug_kwd": page_id,
"title_kwd": title,
"md_with_weight": content,
+ "content_with_weight": content,
"summary_with_weight": summary or title,
"entity_names_kwd": sorted(set(entity_names or [page_title])),
"source_chunk_ids": sorted(source_chunk_ids),
@@ -2192,6 +2203,7 @@ async def _wiki_refine_page(
"title_tks": rag_tokenizer.tokenize(title),
"content_ltks": content_ltks,
"content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),
+ "available_int": 1,
}
# Insert vector (adds q_{dim}_vec field)
vec_col = f"q_{vec_dim}_vec"
@@ -3361,6 +3373,13 @@ async def _wiki_finalize(
outlinks = outlink_map.get(pid) or []
update["outlinks_kwd"] = list(outlinks)
update["outlinks_int"] = len(outlinks)
+ # Backfill retrieval fields on every FINALIZE so pages written before
+ # available_int/doc_id were stamped become visible to RAG on re-run.
+ update["available_int"] = 1
+ update["doc_id"] = kb_id
+ body = rendered_content or original or page.get("md_with_weight") or ""
+ if body:
+ update["content_with_weight"] = body
await thread_pool_exec(
settings.docStoreConn.update,
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index ad02320827..1d2246f612 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -47,6 +47,34 @@ def index_name(uid):
return f"ragflow_{uid}"
+def _chunk_scalar(value) -> str:
+ """Normalize a doc-store scalar that Infinity may return as a one-item list."""
+ if isinstance(value, (list, tuple)):
+ value = value[0] if value else ""
+ return str(value or "").strip()
+
+
+def is_kb_scoped_chunk(chunk: dict | None) -> bool:
+ """True for compilation/graph rows keyed to the KB, not a source document.
+
+ Wiki persist stamps ``doc_id = kb_id``. Those rows must not be treated as
+ leftover chunks from a deleted ``document`` row. A missing ``doc_id`` is
+ not enough on its own: only an explicit compilation marker (``compile_kwd``)
+ or the KB-id sentinel bypasses deleted-document pruning. Otherwise stale
+ ordinary chunks without ``doc_id`` would stay retrievable after their
+ source document is deleted.
+ """
+ if not isinstance(chunk, dict):
+ return False
+ doc_id = _chunk_scalar(chunk.get("doc_id"))
+ kb_id = _chunk_scalar(chunk.get("kb_id"))
+ if kb_id and doc_id == kb_id:
+ return True
+ if doc_id:
+ return False
+ return bool(_chunk_scalar(chunk.get("compile_kwd")))
+
+
class Dealer:
# Short-lived cache of "doc_id exists in MySQL" used by _prune_deleted_chunks.
# Every retrieval would otherwise hit MySQL per query (fan-out searches and the
@@ -124,12 +152,39 @@ class Dealer:
# is removed but the vector record is not fully cleaned up. We filter those
# chunks here so chat/retrieval does not surface content from deleted docs.
# Keep this as a fallback, not as the primary delete mechanism.
- chunk_doc_ids = [chunk.get("doc_id") for chunk in sres.field.values() if chunk and chunk.get("doc_id")]
- if not chunk_doc_ids:
+ fields = sres.field or {}
+ aligned_ids = [chunk_id for chunk_id in sres.ids if fields.get(chunk_id)]
+ if len(aligned_ids) != len(sres.ids):
+ fields = {chunk_id: fields[chunk_id] for chunk_id in aligned_ids}
+ highlight = sres.highlight
+ if highlight:
+ highlight = {chunk_id: highlight[chunk_id] for chunk_id in aligned_ids if chunk_id in highlight}
+ sres = self.SearchResult(
+ total=len(aligned_ids),
+ ids=aligned_ids,
+ query_vector=sres.query_vector,
+ field=fields,
+ highlight=highlight,
+ aggregation=sres.aggregation,
+ keywords=sres.keywords,
+ group_docs=sres.group_docs,
+ )
+
+ chunk_doc_ids = []
+ unscoped_missing_doc_id = False
+ for chunk in fields.values():
+ if not chunk or is_kb_scoped_chunk(chunk):
+ continue
+ doc_id = _chunk_scalar(chunk.get("doc_id"))
+ if doc_id:
+ chunk_doc_ids.append(doc_id)
+ else:
+ unscoped_missing_doc_id = True
+ if not chunk_doc_ids and not unscoped_missing_doc_id:
return sres
- existing_doc_ids = await self._existing_doc_ids(chunk_doc_ids)
- if len(existing_doc_ids) == len(set(chunk_doc_ids)):
+ existing_doc_ids = await self._existing_doc_ids(chunk_doc_ids) if chunk_doc_ids else set()
+ if chunk_doc_ids and not unscoped_missing_doc_id and len(existing_doc_ids) == len(set(chunk_doc_ids)):
return sres
filtered_ids = []
@@ -138,8 +193,17 @@ class Dealer:
removed = 0
for chunk_id in sres.ids:
- chunk = sres.field.get(chunk_id)
- if not chunk or chunk.get("doc_id") not in existing_doc_ids:
+ chunk = fields.get(chunk_id)
+ if not chunk:
+ removed += 1
+ continue
+ if is_kb_scoped_chunk(chunk):
+ filtered_ids.append(chunk_id)
+ filtered_field[chunk_id] = chunk
+ if sres.highlight and chunk_id in sres.highlight:
+ filtered_highlight[chunk_id] = sres.highlight[chunk_id]
+ continue
+ if _chunk_scalar(chunk.get("doc_id")) not in existing_doc_ids:
removed += 1
continue
diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
index 8d81d0b89f..7dd7422f06 100644
--- a/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/test_wiki_incremental.py
@@ -1117,6 +1117,9 @@ async def test_finalize_writes_outlinks():
# the old-mode writer) which Infinity stores/reads back correctly.
assert upd["outlinks_kwd"] == ["concept/B"], f"Got {upd['outlinks_kwd']}"
assert upd["outlinks_int"] == 1
+ assert upd["available_int"] == 1
+ assert upd["doc_id"] == "kb1"
+ assert upd["content_with_weight"]
break
else:
pytest.fail("No update for concept/A found")
@@ -1545,6 +1548,11 @@ def test_refine_page_persists_source_doc_ids_as_array():
inserted_page = doc_store.insert.call_args.args[0][0]
assert inserted_page["source_doc_ids"] == ["doc_1", "doc_2"]
assert inserted_page["topic_kwd"] == "Technology companies"
+ assert inserted_page["available_int"] == 1
+ assert inserted_page["doc_id"] == "kb1"
+ assert inserted_page["kb_id"] == "kb1"
+ assert inserted_page["content_with_weight"] == inserted_page["md_with_weight"]
+ assert inserted_page["content_with_weight"]
def test_topics_for_docs_only_returns_source_scoped_candidates():
@@ -2075,6 +2083,31 @@ async def test_reduce_entity_deletes_page_when_last_chunk_is_removed():
assert [claim["statement"] for claim in delta["retractions"]] == ["only claim"]
+def test_canonical_entity_doc_stamps_retrieval_fields():
+ tokenizer = MagicMock()
+ tokenizer.tokenize.side_effect = lambda text: text
+ tokenizer.fine_grained_tokenize.side_effect = lambda text: text
+ with patch("rag.nlp.rag_tokenizer", tokenizer, create=True):
+ doc = _wiki._build_canonical_entity_doc(
+ tenant_id="t1",
+ kb_id="kb1",
+ entity_name="Apple",
+ entity_type="org",
+ aliases=["Apple Inc."],
+ source_doc_ids=["doc_1"],
+ claim_count=2,
+ embedding=[0.1, 0.2],
+ )
+
+ assert doc["available_int"] == 1
+ assert doc["doc_id"] == "kb1"
+ assert doc["kb_id"] == "kb1"
+ assert doc["content_with_weight"] == "Apple"
+ assert doc["content_ltks"] == "Apple"
+ assert doc["content_sm_ltks"] == "Apple"
+ assert doc["q_2_vec"] == [0.1, 0.2]
+
+
def test_as_int_accepts_string_doc_store_values():
assert _wiki._as_int("7") == 7
assert _wiki._as_int(None, 3) == 3
diff --git a/test/unit_test/rag/test_prune_deleted_chunks.py b/test/unit_test/rag/test_prune_deleted_chunks.py
new file mode 100644
index 0000000000..e61ea08d62
--- /dev/null
+++ b/test/unit_test/rag/test_prune_deleted_chunks.py
@@ -0,0 +1,242 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""KB-scoped wiki/compilation rows must survive deleted-document pruning."""
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_ROOT = Path(__file__).parents[3]
+
+
+def _load_search():
+ fake_rag_nlp = types.ModuleType("rag.nlp")
+ fake_rag_nlp.__path__ = []
+
+ fake_tokenizer = types.ModuleType("rag.nlp.rag_tokenizer")
+ fake_rag_nlp.rag_tokenizer = fake_tokenizer
+
+ fake_query = types.ModuleType("rag.nlp.query")
+
+ class _DummyFulltextQueryer:
+ pass
+
+ fake_query.FulltextQueryer = _DummyFulltextQueryer
+ fake_rag_nlp.query = fake_query
+ fake_settings = types.ModuleType("common.settings")
+
+ stub_names = ("rag.nlp", "rag.nlp.rag_tokenizer", "rag.nlp.query", "common.settings")
+ saved = {name: sys.modules.get(name) for name in stub_names}
+ sys.modules["rag.nlp"] = fake_rag_nlp
+ sys.modules["rag.nlp.rag_tokenizer"] = fake_tokenizer
+ sys.modules["rag.nlp.query"] = fake_query
+ sys.modules["common.settings"] = fake_settings
+
+ import common
+
+ saved_common_settings = getattr(common, "settings", None)
+ common.settings = fake_settings
+
+ spec = importlib.util.spec_from_file_location("rag.nlp.search_prune_test", _ROOT / "rag" / "nlp" / "search.py")
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ sys.modules["rag.nlp.search_prune_test"] = module
+ try:
+ spec.loader.exec_module(module)
+ finally:
+ if saved_common_settings is None:
+ if getattr(common, "settings", None) is fake_settings:
+ delattr(common, "settings")
+ else:
+ common.settings = saved_common_settings
+ for name, previous in saved.items():
+ if previous is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = previous
+ return module
+
+
+_search = _load_search()
+Dealer = _search.Dealer
+is_kb_scoped_chunk = _search.is_kb_scoped_chunk
+
+
+def test_is_kb_scoped_chunk_missing_or_empty_doc_id():
+ assert is_kb_scoped_chunk({"kb_id": "kb1"}) is False
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": ""}) is False
+
+
+def test_is_kb_scoped_chunk_compilation_row_without_doc_id():
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "compile_kwd": "wiki_page"}) is True
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "", "compile_kwd": ["wiki_canonical_entity"]}) is True
+
+
+def test_is_kb_scoped_chunk_sentinel_matches_kb():
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "kb1"}) is True
+ assert is_kb_scoped_chunk({"kb_id": ["kb1"], "doc_id": ["kb1"]}) is True
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "kb1", "compile_kwd": "wiki_page"}) is True
+
+
+def test_is_kb_scoped_chunk_real_document():
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "doc1"}) is False
+ assert is_kb_scoped_chunk({"kb_id": "kb1", "doc_id": "doc1", "compile_kwd": "wiki_doc_page_source"}) is False
+ assert is_kb_scoped_chunk(None) is False
+
+
+def _result(ids, field):
+ return Dealer.SearchResult(total=len(ids), ids=list(ids), field=field)
+
+
+async def test_prune_keeps_wiki_pages_when_mixed_with_source_chunks():
+ dealer = Dealer(dataStore=None)
+
+ async def existing(doc_ids):
+ return {d for d in doc_ids if d == "doc-live"}
+
+ dealer._existing_doc_ids = existing
+ sres = _result(
+ ["src-live", "src-dead", "wiki-page"],
+ {
+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},
+ "src-dead": {"doc_id": "doc-gone", "kb_id": "kb1"},
+ "wiki-page": {"doc_id": "kb1", "kb_id": "kb1", "compile_kwd": "wiki_page"},
+ },
+ )
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["src-live", "wiki-page"]
+ assert "src-dead" not in out.field
+ assert "wiki-page" in out.field
+
+
+async def test_prune_keeps_wiki_pages_that_omit_doc_id():
+ dealer = Dealer(dataStore=None)
+
+ async def existing(doc_ids):
+ return {d for d in doc_ids if d == "doc-live"}
+
+ dealer._existing_doc_ids = existing
+ sres = _result(
+ ["src-live", "wiki-page"],
+ {
+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},
+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},
+ },
+ )
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["src-live", "wiki-page"]
+
+
+async def test_prune_drops_ordinary_chunks_that_omit_doc_id():
+ dealer = Dealer(dataStore=None)
+ called = []
+
+ async def existing(doc_ids):
+ called.append(list(doc_ids))
+ return {d for d in doc_ids if d == "doc-live"}
+
+ dealer._existing_doc_ids = existing
+ sres = _result(
+ ["src-live", "src-orphan", "wiki-page"],
+ {
+ "src-live": {"doc_id": "doc-live", "kb_id": "kb1"},
+ "src-orphan": {"kb_id": "kb1"},
+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},
+ },
+ )
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["src-live", "wiki-page"]
+ assert "src-orphan" not in out.field
+ assert called == [["doc-live"]]
+
+
+async def test_prune_drops_orphan_chunks_when_every_hit_omits_doc_id():
+ dealer = Dealer(dataStore=None)
+ called = []
+
+ async def existing(doc_ids):
+ called.append(list(doc_ids))
+ return set()
+
+ dealer._existing_doc_ids = existing
+ sres = _result(
+ ["src-orphan", "wiki-page"],
+ {
+ "src-orphan": {"kb_id": "kb1", "doc_id": ""},
+ "wiki-page": {"kb_id": "kb1", "compile_kwd": "wiki_page"},
+ },
+ )
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["wiki-page"]
+ assert "src-orphan" not in out.field
+ assert called == []
+
+
+async def test_prune_drops_all_ordinary_chunks_without_doc_id():
+ dealer = Dealer(dataStore=None)
+ called = []
+
+ async def existing(doc_ids):
+ called.append(list(doc_ids))
+ return set()
+
+ dealer._existing_doc_ids = existing
+ sres = _result(["src-orphan"], {"src-orphan": {"kb_id": "kb1"}})
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == []
+ assert out.field == {}
+ assert called == []
+
+
+async def test_prune_skips_lookup_when_all_hits_are_kb_scoped():
+ dealer = Dealer(dataStore=None)
+ called = []
+
+ async def existing(doc_ids):
+ called.append(list(doc_ids))
+ return set()
+
+ dealer._existing_doc_ids = existing
+ sres = _result(["wiki"], {"wiki": {"doc_id": "kb1", "kb_id": "kb1"}})
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["wiki"]
+ assert called == []
+
+
+async def test_prune_drops_ids_missing_from_fields_when_all_hits_are_kb_scoped():
+ dealer = Dealer(dataStore=None)
+ called = []
+
+ async def existing(doc_ids):
+ called.append(list(doc_ids))
+ return set()
+
+ dealer._existing_doc_ids = existing
+ sres = _result(["wiki", "ghost"], {"wiki": {"doc_id": "kb1", "kb_id": "kb1"}})
+ out = await dealer._prune_deleted_chunks(sres)
+ assert out.ids == ["wiki"]
+ assert "ghost" not in out.field
+ assert called == []