Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: align wiki alteration provenance (#19140)

buua436cf441d4c7299
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 223dda10cd..9f2ba094bd 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -2320,6 +2320,8 @@ async def _wiki_chunk_alteration(
dataset_id: str,
eligible_doc_ids: set[str],
involved_doc_ids: set[str],
+ current_chunk_state: dict | None = None,
+ previous_map_state: dict | None = None,
) -> dict:
"""Compare active source chunks with the hashes used by Wiki MAP.
@@ -2338,15 +2340,15 @@ async def _wiki_chunk_alteration(
if not eligible_doc_ids and not involved_doc_ids:
return empty
- from rag.advanced_rag.knowlege_compile.wiki import (
- _wiki_compare_chunk_states,
- _wiki_load_active_map_state,
- _wiki_scan_current_chunk_state,
- )
+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_compare_chunk_states, _wiki_load_active_map_state, _wiki_scan_current_chunk_state
try:
- current = await _wiki_scan_current_chunk_state(tenant_id, dataset_id, eligible_doc_ids)
- previous = await _wiki_load_active_map_state(tenant_id, dataset_id)
+ current = current_chunk_state
+ if current is None:
+ current = await _wiki_scan_current_chunk_state(tenant_id, dataset_id, eligible_doc_ids)
+ previous = previous_map_state
+ if previous is None:
+ previous = await _wiki_load_active_map_state(tenant_id, dataset_id)
except Exception as exc:
logging.exception("alteration: failed to compare Wiki chunk state for kb=%s", dataset_id)
raise RuntimeError(f"Failed to compare Wiki chunk state for alteration (kb={dataset_id})") from exc
@@ -2456,10 +2458,20 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fi
return involved
-async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> set:
+async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str, tenant_id: str, wiki_map_state: dict | None = None) -> set:
"""Gather the doc ids baked into the compiled product for ``kind``."""
if kind == "wiki":
- return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": [WIKI_PAGE_COMPILE_KWD]}, "source_doc_ids", from_list=True)
+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state
+
+ # Wiki MAP state is committed only after the compilation run has
+ # successfully completed. Use that active snapshot as the
+ # compilation provenance, rather than wiki_page.source_doc_ids:
+ # a document can participate in MAP/REDUCE without producing a page
+ # (for example when the extractor finds no page-worthy entities).
+ state = wiki_map_state
+ if state is None:
+ state = await _wiki_load_active_map_state(tenant_id, dataset_id)
+ return {str(item.get("doc_id")) for item in state.values() if item.get("doc_id")}
if kind in _ALTERATION_KIND_TO_MERGED_ROW_KIND:
condition = {
"knowledge_graph_kwd": ["entity", "relation"],
@@ -2514,14 +2526,61 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
if pack is not None:
index_nm, _ = pack
- involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)
+ if kind == "wiki":
+ # A document may match the Wiki template while still having no
+ # available source chunks (for example, parsing has not produced
+ # chunks yet or all chunks are disabled). Such a document is not
+ # an actionable Wiki input and must not be reported as newly
+ # uploaded.
+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_scan_current_chunk_state
+
+ eligible_before_chunk_filter = len(eligible_doc_ids)
+ current_chunk_state = await _wiki_scan_current_chunk_state(
+ kb.tenant_id,
+ dataset_id,
+ eligible_doc_ids,
+ )
+ chunk_doc_ids = {str(item.get("doc_id")) for item in current_chunk_state.values() if item.get("doc_id")}
+ eligible_doc_ids &= chunk_doc_ids
+ logging.debug(
+ "alteration: Wiki chunk eligibility kb=%s tenant=%s before=%d after=%d chunks=%d",
+ dataset_id,
+ kb.tenant_id,
+ eligible_before_chunk_filter,
+ len(eligible_doc_ids),
+ len(current_chunk_state),
+ )
+ wiki_map_state = None
+ if kind == "wiki":
+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state
+
+ wiki_map_state = await _wiki_load_active_map_state(kb.tenant_id, dataset_id)
+ logging.debug(
+ "alteration: Wiki MAP provenance kb=%s tenant=%s involved=%d eligible=%d",
+ dataset_id,
+ kb.tenant_id,
+ len({str(item.get("doc_id")) for item in wiki_map_state.values() if item.get("doc_id")}),
+ len(eligible_doc_ids),
+ )
+ involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind, kb.tenant_id, wiki_map_state)
if kind == "wiki":
chunk_changes = await _wiki_chunk_alteration(
kb.tenant_id,
dataset_id,
eligible_doc_ids,
involved_doc_ids,
+ current_chunk_state,
+ wiki_map_state,
)
+ elif kind == "wiki":
+ # Without the compiled source index there are no current chunks that
+ # can be considered Wiki inputs.
+ eligible_doc_ids = set()
+ logging.debug(
+ "alteration: Wiki compiled index missing kb=%s tenant=%s eligible=0",
+ dataset_id,
+ kb.tenant_id,
+ )
# Wiki membership follows compilation eligibility. Disabling a document or
# removing its Wiki template is therefore a removal; enabling it again or
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
index 5c665f6ef7..2417a2e970 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
@@ -450,8 +450,34 @@ def test_wiki_alteration_treats_wiki_template_as_eligible(monkeypatch):
assert module._eligible_doc_ids_for_kind(docs, "tenant-1", "wiki") == {"doc-wiki"}
+async def test_wiki_involved_ids_use_active_map_state_when_pages_are_missing(monkeypatch):
+ """Use MAP provenance even when a participating document has no page row."""
+ module, _, _ = _load_list_datasets_module(
+ monkeypatch,
+ kbs=[],
+ parsing_status_by_kb={},
+ )
+
+ async def _load_active_map_state(tenant_id, dataset_id):
+ assert tenant_id == "tenant-1"
+ assert dataset_id == "kb-1"
+ return {
+ "chunk-a": {"doc_id": "doc-with-page"},
+ "chunk-b": {"doc_id": "doc-without-page"},
+ }
+
+ wiki_stub = sys.modules["rag.advanced_rag.knowlege_compile.wiki"]
+ wiki_stub._wiki_load_active_map_state = _load_active_map_state
+
+ involved = await module._involved_doc_ids_for_kind("index", "kb-1", "wiki", "tenant-1")
+
+ assert involved == {"doc-with-page", "doc-without-page"}
+
+
@pytest.mark.asyncio
async def test_wiki_chunk_alteration_uses_full_successful_state(monkeypatch):
+ """Reuse supplied current and previous states without scanning the store again."""
module, _, _ = _load_list_datasets_module(
monkeypatch,
kbs=[],
@@ -501,6 +527,8 @@ async def test_wiki_chunk_alteration_uses_full_successful_state(monkeypatch):
"kb-1",
{"doc-existing", "doc-new"},
{"doc-existing", "doc-template-off", "doc-removed"},
+ current_chunk_state=current,
+ previous_map_state=previous,
)
assert result == {