RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: align wiki alteration provenance (#19140)
buua436cf441d4c7299
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 223dda10cd..9f2ba094bd 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -2320,6 +2320,8 @@ async def _wiki_chunk_alteration(dataset_id: str,eligible_doc_ids: set[str],involved_doc_ids: set[str],+ current_chunk_state: dict | None = None,+ previous_map_state: dict | None = None,) -> dict:"""Compare active source chunks with the hashes used by Wiki MAP.@@ -2338,15 +2340,15 @@ async def _wiki_chunk_alteration(if not eligible_doc_ids and not involved_doc_ids:return empty- from rag.advanced_rag.knowlege_compile.wiki import (- _wiki_compare_chunk_states,- _wiki_load_active_map_state,- _wiki_scan_current_chunk_state,- )+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_compare_chunk_states, _wiki_load_active_map_state, _wiki_scan_current_chunk_statetry:- current = await _wiki_scan_current_chunk_state(tenant_id, dataset_id, eligible_doc_ids)- previous = await _wiki_load_active_map_state(tenant_id, dataset_id)+ current = current_chunk_state+ if current is None:+ current = await _wiki_scan_current_chunk_state(tenant_id, dataset_id, eligible_doc_ids)+ previous = previous_map_state+ if previous is None:+ previous = await _wiki_load_active_map_state(tenant_id, dataset_id)except Exception as exc:logging.exception("alteration: failed to compare Wiki chunk state for kb=%s", dataset_id)raise RuntimeError(f"Failed to compare Wiki chunk state for alteration (kb={dataset_id})") from exc@@ -2456,10 +2458,20 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fireturn involved-async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> set:+async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str, tenant_id: str, wiki_map_state: dict | None = None) -> set:"""Gather the doc ids baked into the compiled product for ``kind``."""if kind == "wiki":- return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": [WIKI_PAGE_COMPILE_KWD]}, "source_doc_ids", from_list=True)+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state++ # Wiki MAP state is committed only after the compilation run has+ # successfully completed. Use that active snapshot as the+ # compilation provenance, rather than wiki_page.source_doc_ids:+ # a document can participate in MAP/REDUCE without producing a page+ # (for example when the extractor finds no page-worthy entities).+ state = wiki_map_state+ if state is None:+ state = await _wiki_load_active_map_state(tenant_id, dataset_id)+ return {str(item.get("doc_id")) for item in state.values() if item.get("doc_id")}if kind in _ALTERATION_KIND_TO_MERGED_ROW_KIND:condition = {"knowledge_graph_kwd": ["entity", "relation"],@@ -2514,14 +2526,61 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):pack = _compiled_index_or_none(kb.tenant_id, dataset_id)if pack is not None:index_nm, _ = pack- involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)+ if kind == "wiki":+ # A document may match the Wiki template while still having no+ # available source chunks (for example, parsing has not produced+ # chunks yet or all chunks are disabled). Such a document is not+ # an actionable Wiki input and must not be reported as newly+ # uploaded.+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_scan_current_chunk_state++ eligible_before_chunk_filter = len(eligible_doc_ids)+ current_chunk_state = await _wiki_scan_current_chunk_state(+ kb.tenant_id,+ dataset_id,+ eligible_doc_ids,+ )+ chunk_doc_ids = {str(item.get("doc_id")) for item in current_chunk_state.values() if item.get("doc_id")}+ eligible_doc_ids &= chunk_doc_ids+ logging.debug(+ "alteration: Wiki chunk eligibility kb=%s tenant=%s before=%d after=%d chunks=%d",+ dataset_id,+ kb.tenant_id,+ eligible_before_chunk_filter,+ len(eligible_doc_ids),+ len(current_chunk_state),+ )+ wiki_map_state = None+ if kind == "wiki":+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state++ wiki_map_state = await _wiki_load_active_map_state(kb.tenant_id, dataset_id)+ logging.debug(+ "alteration: Wiki MAP provenance kb=%s tenant=%s involved=%d eligible=%d",+ dataset_id,+ kb.tenant_id,+ len({str(item.get("doc_id")) for item in wiki_map_state.values() if item.get("doc_id")}),+ len(eligible_doc_ids),+ )+ involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind, kb.tenant_id, wiki_map_state)if kind == "wiki":chunk_changes = await _wiki_chunk_alteration(kb.tenant_id,dataset_id,eligible_doc_ids,involved_doc_ids,+ current_chunk_state,+ wiki_map_state,)+ elif kind == "wiki":+ # Without the compiled source index there are no current chunks that+ # can be considered Wiki inputs.+ eligible_doc_ids = set()+ logging.debug(+ "alteration: Wiki compiled index missing kb=%s tenant=%s eligible=0",+ dataset_id,+ kb.tenant_id,+ )# Wiki membership follows compilation eligibility. Disabling a document or# removing its Wiki template is therefore a removal; enabling it again ordiff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex 5c665f6ef7..2417a2e970 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -450,8 +450,34 @@ def test_wiki_alteration_treats_wiki_template_as_eligible(monkeypatch):assert module._eligible_doc_ids_for_kind(docs, "tenant-1", "wiki") == {"doc-wiki"}+async def test_wiki_involved_ids_use_active_map_state_when_pages_are_missing(monkeypatch):+ """Use MAP provenance even when a participating document has no page row."""+ module, _, _ = _load_list_datasets_module(+ monkeypatch,+ kbs=[],+ parsing_status_by_kb={},+ )++ async def _load_active_map_state(tenant_id, dataset_id):+ assert tenant_id == "tenant-1"+ assert dataset_id == "kb-1"+ return {+ "chunk-a": {"doc_id": "doc-with-page"},+ "chunk-b": {"doc_id": "doc-without-page"},+ }++ wiki_stub = sys.modules["rag.advanced_rag.knowlege_compile.wiki"]+ wiki_stub._wiki_load_active_map_state = _load_active_map_state++ involved = await module._involved_doc_ids_for_kind("index", "kb-1", "wiki", "tenant-1")++ assert involved == {"doc-with-page", "doc-without-page"}++@pytest.mark.asyncioasync def test_wiki_chunk_alteration_uses_full_successful_state(monkeypatch):+ """Reuse supplied current and previous states without scanning the store again."""module, _, _ = _load_list_datasets_module(monkeypatch,kbs=[],@@ -501,6 +527,8 @@ async def test_wiki_chunk_alteration_uses_full_successful_state(monkeypatch):"kb-1",{"doc-existing", "doc-new"},{"doc-existing", "doc-template-off", "doc-removed"},+ current_chunk_state=current,+ previous_map_state=previous,)assert result == {