RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: exclude disabled documents from wiki compilation (#18261)
buua43615a63bc15cef
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex bc740316e3..cd0004b6a7 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -833,7 +833,9 @@ async def get_wiki_alteration(tenant_id, dataset_id):GET /api/v1/datasets/<dataset_id>/artifacts/alteration?kind=<kind>``kind`` (default ``wiki``) is one of: wiki | graph | mindmap | timeline |tree (tree covers both ``tree`` and ``page_index``).- Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}+ Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}.+ Wiki responses also include ``changed`` and ``changed_doc_ids`` based on+ source chunk hash drift."""kind = (request.args.get("kind") or "wiki").strip().lower()if kind not in {"wiki", "graph", "mindmap", "timeline", "tree"}:diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 5722bbc8e9..17ce38e782 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -32,7 +32,7 @@ from api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_from common import settingsfrom common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnumfrom common.misc_utils import thread_pool_exec, thread_pool_exec_long_time-from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD+from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD, _chunk_hash# KB-wide structure-graph merge index types. Each (re)builds the ``dataset_graph``# rows for one structure kind via ``rebuild_dataset_structure_graph_json``; the@@ -2227,6 +2227,125 @@ def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc}+async def _wiki_chunk_alteration(+ index_nm,+ dataset_id: str,+ active_doc_ids: set[str],+ disabled_doc_ids: set[str],+) -> dict:+ """Compare active source chunks with the hashes used by Wiki MAP.++ Document-level provenance cannot detect an edited, added, or removed+ chunk. The MAP resume rows already contain the exact chunk hash used by+ compilation, so they are the authoritative compiled-side snapshot.+ A document is changed when a chunk is added, removed, or has a different+ hash. A disabled document with retained MAP rows is also changed because+ those rows must no longer participate in the next Wiki build.+ """+ from common.doc_store.doc_store_base import OrderByExpr++ empty = {+ "changed": 0,+ "changed_doc_ids": [],+ }+ if not active_doc_ids and not disabled_doc_ids:+ return empty++ current: dict[str, dict] = {}+ for doc_id in active_doc_ids:+ offset = 0+ while True:+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["id", "doc_id", "content_with_weight"],+ [],+ {"doc_id": [doc_id], "available_int": 1, "must_not": {"exists": "compile_kwd"}},+ [],+ OrderByExpr(),+ offset,+ 1000,+ index_nm,+ [dataset_id],+ )+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight"]) or {}+ except Exception as exc:+ logging.exception("alteration: failed to load source chunks for doc=%s", doc_id)+ raise RuntimeError(f"Failed to load source chunks for Wiki alteration (kb={dataset_id}, doc={doc_id})") from exc+ if not rows:+ break+ for row in rows.values():+ chunk_id = str(row.get("id") or "")+ if not chunk_id:+ continue+ current[chunk_id] = {+ "id": chunk_id,+ "doc_id": doc_id,+ "hash": _chunk_hash(row.get("content_with_weight") or ""),+ }+ if len(rows) < 1000:+ break+ offset += 1000++ compiled: dict[str, dict] = {}+ offset = 0+ while True:+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"],+ [],+ {"compile_kwd": ["wiki_map_extract"]},+ [],+ OrderByExpr(),+ offset,+ 1000,+ index_nm,+ [dataset_id],+ )+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]) or {}+ except Exception as exc:+ logging.exception("alteration: failed to load Wiki MAP hashes for kb=%s", dataset_id)+ raise RuntimeError(f"Failed to load Wiki MAP hashes for alteration (kb={dataset_id})") from exc+ if not rows:+ break+ for row in rows.values():+ row_doc_ids = _flatten_provenance_doc_ids(row.get("doc_id"))+ matching_doc_ids = row_doc_ids & (active_doc_ids | disabled_doc_ids)+ if not matching_doc_ids:+ continue+ doc_id = next(iter(matching_doc_ids))+ chunk_ids = row.get("source_chunk_ids") or []+ if isinstance(chunk_ids, str):+ chunk_ids = [chunk_ids]+ saved_hash = row.get("chunk_hash_kwd")+ saved_hash = saved_hash if isinstance(saved_hash, str) else ""+ for chunk_id in chunk_ids:+ chunk_id = str(chunk_id or "")+ if chunk_id:+ compiled.setdefault(chunk_id, {"id": chunk_id, "doc_id": doc_id, "hash": saved_hash})+ if len(rows) < 1000:+ break+ offset += 1000++ changed_doc_ids: set[str] = set()+ for chunk_id, item in current.items():+ old = compiled.get(chunk_id)+ if old is None or old["hash"] != item["hash"]:+ changed_doc_ids.add(item["doc_id"])++ # A missing current chunk means it was removed from an active document.+ # For a disabled document, all retained MAP chunks are stale by definition.+ for chunk_id, item in compiled.items():+ if item["doc_id"] in disabled_doc_ids or (item["doc_id"] in active_doc_ids and chunk_id not in current):+ changed_doc_ids.add(item["doc_id"])++ return {+ "changed": len(changed_doc_ids),+ "changed_doc_ids": sorted(changed_doc_ids),+ }++def _flatten_provenance_doc_ids(value) -> set[str]:"""Normalize source_doc_ids stored as JSON strings, lists, or scalars."""if value is None:@@ -2368,15 +2487,27 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):return False, "Invalid Dataset ID"docs, current_doc_ids = await _current_dataset_docs(dataset_id)+ eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)involved_doc_ids: set[str] = set()+ chunk_changes = Nonepack = _compiled_index_or_none(kb.tenant_id, dataset_id)if pack is not None:index_nm, _ = packinvolved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)+ if kind == "wiki":+ disabled_doc_ids = await _disabled_dataset_doc_ids(dataset_id)+ chunk_changes = await _wiki_chunk_alteration(+ index_nm,+ dataset_id,+ eligible_doc_ids,+ disabled_doc_ids,+ )- eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)- return True, _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)+ result = _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)+ if chunk_changes is not None:+ result.update(chunk_changes)+ return True, resultasync def get_wiki_alteration(dataset_id: str, tenant_id: str):diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex 39dc307437..556c0c1f2a 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -90,6 +90,30 @@ DEFAULT_WIKI_MAP_WORKERS = 20DEFAULT_WIKI_MAP_TIMEOUT = 600+async def _wiki_disabled_doc_ids(kb_id: str) -> set[str]:+ """Return disabled documents so cached MAP rows cannot re-enter a build."""+ from api.db.services.document_service import DocumentService++ disabled = await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)+ return _wiki_doc_ids(disabled)+++def _wiki_doc_ids(value) -> set[str]:+ """Normalize a scalar or list-valued document-id field."""+ if value is None:+ return set()+ if isinstance(value, str):+ value = value.strip()+ return {value} if value else set()+ if isinstance(value, (list, tuple, set)):+ result: set[str] = set()+ for item in value:+ result.update(_wiki_doc_ids(item))+ return result+ value = str(value).strip()+ return {value} if value else set()++WIKI_MAP_SYSTEM = ("You are a knowledge extraction engine. Extract structured knowledge from the ""provided document section. Return ONLY valid JSON matching the schema exactly. "@@ -1182,8 +1206,9 @@ async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:from rag.nlp import search as _rag_searchindex = _rag_search.index_name(tenant_id)+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}- select_fields = ["id", "content_with_weight"]+ select_fields = ["id", "content_with_weight", "doc_id"]PAGE_SIZE = 1000offset = 0@@ -1213,6 +1238,8 @@ async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:breakfor row in field_map.values():+ if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:+ continuecontent = row.get("content_with_weight")if not isinstance(content, str) or not content:continue@@ -1254,6 +1281,7 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:from rag.nlp import search as _rag_searchindex = _rag_search.index_name(tenant_id)+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}select_fields = ["id", "doc_id"]@@ -1282,10 +1310,8 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:if not field_map:breakfor row in field_map.values():- raw = row.get("doc_id")- candidates = raw if isinstance(raw, list) else [raw]- for d in candidates:- if isinstance(d, str) and d and d not in seen:+ for d in _wiki_doc_ids(row.get("doc_id")):+ if d not in disabled_doc_ids and d not in seen:seen.add(d)doc_ids.append(d)if len(field_map) < PAGE_SIZE:@@ -1318,8 +1344,9 @@ async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:from rag.nlp import search as _rag_searchindex = _rag_search.index_name(tenant_id)+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}- select_fields = ["id", "source_chunk_ids", "chunk_hash_kwd"]+ select_fields = ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]PAGE_SIZE = 128offset = 0@@ -1352,6 +1379,8 @@ async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:if not field_map:breakfor row in field_map.values():+ if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:+ continuehh = row.get("chunk_hash_kwd")if not isinstance(hh, str):hh = ""diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.pyindex 91bc0e29ad..775e77b658 100644--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py@@ -3361,6 +3361,10 @@ async def wiki_compile_incremental(# concept / claim / relation / topic lists are NOT separate columns, so# they must be parsed out of that blob to rebuild the map_result shape# that _extract_raw_entities and REDUCE expect.+ from api.db.services.document_service import DocumentService+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_doc_ids++ disabled_doc_ids = _wiki_doc_ids(await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id))select_fields = ["content_with_weight", "doc_id"]offset = 0page_size = 1000@@ -3383,6 +3387,9 @@ async def wiki_compile_incremental(logging.exception("wiki: failed to load MAP results for kb=%s", kb_id)breakfor row in field_map.values():+ row_doc_ids = _wiki_doc_ids(row.get("doc_id"))+ if row_doc_ids & disabled_doc_ids:+ continueraw = row.get("content_with_weight")if isinstance(raw, str) and raw:try:@@ -3395,7 +3402,7 @@ async def wiki_compile_incremental(extract = Noneif not isinstance(extract, dict):continue- extract["doc_id"] = row.get("doc_id", "")+ extract["doc_id"] = next(iter(row_doc_ids), "")map_results.append(extract)if len(field_map) < page_size:breakdiff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.pyindex 54ae52f20f..5a47f3c2d9 100644--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py@@ -57,6 +57,7 @@ from common.misc_utils import thread_pool_execfrom rag.nlp import searchfrom rag.advanced_rag.knowlege_compile.structure import LLMCallPoolfrom rag.advanced_rag.knowlege_compile.wiki import (+ _wiki_doc_ids,wiki_map_from_chunks,wiki_plan_from_reduction,wiki_reduce_from_extracts,@@ -333,12 +334,14 @@ def _wiki_eligible_docs(all_docs, tenant_id: str, skip_doc_ids=None) -> list[tupasync def _wiki_existing_map_doc_ids(tenant_id: str, kb_id: str) -> set[str]:from common.doc_store.doc_store_base import OrderByExpr+ from api.db.services.document_service import DocumentServiceindex = search.index_name(tenant_id)if not settings.docStoreConn.index_exist(index, kb_id):return set()doc_ids: set[str] = set()+ disabled_doc_ids = _wiki_doc_ids(await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id))select_fields = ["id", "doc_id"]offset = 0page_size = 1000@@ -363,9 +366,7 @@ async def _wiki_existing_map_doc_ids(tenant_id: str, kb_id: str) -> set[str]:if not field_map:breakfor row in field_map.values():- doc_id = row.get("doc_id")- if isinstance(doc_id, str) and doc_id:- doc_ids.add(doc_id)+ doc_ids.update(_wiki_doc_ids(row.get("doc_id")) - disabled_doc_ids)if len(field_map) < page_size:breakoffset += page_sizediff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex 1430200945..5425885d6f 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -181,6 +181,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):monkeypatch,"rag.advanced_rag.knowlege_compile.wiki",WIKI_PAGE_COMPILE_KWD="wiki",+ _chunk_hash=lambda content: "stub-hash",)repo_root = Path(__file__).resolve().parents[5]diff --git a/test/unit_test/api/apps/services/test_delete_datasets.py b/test/unit_test/api/apps/services/test_delete_datasets.pyindex 873791779f..12862fcc5a 100644--- a/test/unit_test/api/apps/services/test_delete_datasets.py+++ b/test/unit_test/api/apps/services/test_delete_datasets.py@@ -164,6 +164,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):monkeypatch,"rag.advanced_rag.knowlege_compile.wiki",WIKI_PAGE_COMPILE_KWD="wiki",+ _chunk_hash=lambda content: "stub-hash",)_stub(monkeypatch,