Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: exclude disabled documents from wiki compilation (#18261)

buua43615a63bc15cef
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py
index bc740316e3..cd0004b6a7 100644
--- a/api/apps/restful_apis/dataset_api.py
+++ b/api/apps/restful_apis/dataset_api.py
@@ -833,7 +833,9 @@ async def get_wiki_alteration(tenant_id, dataset_id):
GET /api/v1/datasets/<dataset_id>/artifacts/alteration?kind=<kind>
``kind`` (default ``wiki``) is one of: wiki | graph | mindmap | timeline |
tree (tree covers both ``tree`` and ``page_index``).
- Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}
+ Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}.
+ Wiki responses also include ``changed`` and ``changed_doc_ids`` based on
+ source chunk hash drift.
"""
kind = (request.args.get("kind") or "wiki").strip().lower()
if kind not in {"wiki", "graph", "mindmap", "timeline", "tree"}:
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 5722bbc8e9..17ce38e782 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -32,7 +32,7 @@ from api.utils.api_utils import deep_merge, get_parser_config, remap_dictionary_
from common import settings
from common.constants import PAGERANK_FLD, FileSource, LLMType, RetCode, StatusEnum
from common.misc_utils import thread_pool_exec, thread_pool_exec_long_time
-from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD
+from rag.advanced_rag.knowlege_compile.wiki import WIKI_PAGE_COMPILE_KWD, _chunk_hash
# KB-wide structure-graph merge index types. Each (re)builds the ``dataset_graph``
# rows for one structure kind via ``rebuild_dataset_structure_graph_json``; the
@@ -2227,6 +2227,125 @@ def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc
}
+async def _wiki_chunk_alteration(
+ index_nm,
+ dataset_id: str,
+ active_doc_ids: set[str],
+ disabled_doc_ids: set[str],
+) -> dict:
+ """Compare active source chunks with the hashes used by Wiki MAP.
+
+ Document-level provenance cannot detect an edited, added, or removed
+ chunk. The MAP resume rows already contain the exact chunk hash used by
+ compilation, so they are the authoritative compiled-side snapshot.
+ A document is changed when a chunk is added, removed, or has a different
+ hash. A disabled document with retained MAP rows is also changed because
+ those rows must no longer participate in the next Wiki build.
+ """
+ from common.doc_store.doc_store_base import OrderByExpr
+
+ empty = {
+ "changed": 0,
+ "changed_doc_ids": [],
+ }
+ if not active_doc_ids and not disabled_doc_ids:
+ return empty
+
+ current: dict[str, dict] = {}
+ for doc_id in active_doc_ids:
+ offset = 0
+ while True:
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ ["id", "doc_id", "content_with_weight"],
+ [],
+ {"doc_id": [doc_id], "available_int": 1, "must_not": {"exists": "compile_kwd"}},
+ [],
+ OrderByExpr(),
+ offset,
+ 1000,
+ index_nm,
+ [dataset_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight"]) or {}
+ except Exception as exc:
+ logging.exception("alteration: failed to load source chunks for doc=%s", doc_id)
+ raise RuntimeError(f"Failed to load source chunks for Wiki alteration (kb={dataset_id}, doc={doc_id})") from exc
+ if not rows:
+ break
+ for row in rows.values():
+ chunk_id = str(row.get("id") or "")
+ if not chunk_id:
+ continue
+ current[chunk_id] = {
+ "id": chunk_id,
+ "doc_id": doc_id,
+ "hash": _chunk_hash(row.get("content_with_weight") or ""),
+ }
+ if len(rows) < 1000:
+ break
+ offset += 1000
+
+ compiled: dict[str, dict] = {}
+ offset = 0
+ while True:
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"],
+ [],
+ {"compile_kwd": ["wiki_map_extract"]},
+ [],
+ OrderByExpr(),
+ offset,
+ 1000,
+ index_nm,
+ [dataset_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]) or {}
+ except Exception as exc:
+ logging.exception("alteration: failed to load Wiki MAP hashes for kb=%s", dataset_id)
+ raise RuntimeError(f"Failed to load Wiki MAP hashes for alteration (kb={dataset_id})") from exc
+ if not rows:
+ break
+ for row in rows.values():
+ row_doc_ids = _flatten_provenance_doc_ids(row.get("doc_id"))
+ matching_doc_ids = row_doc_ids & (active_doc_ids | disabled_doc_ids)
+ if not matching_doc_ids:
+ continue
+ doc_id = next(iter(matching_doc_ids))
+ chunk_ids = row.get("source_chunk_ids") or []
+ if isinstance(chunk_ids, str):
+ chunk_ids = [chunk_ids]
+ saved_hash = row.get("chunk_hash_kwd")
+ saved_hash = saved_hash if isinstance(saved_hash, str) else ""
+ for chunk_id in chunk_ids:
+ chunk_id = str(chunk_id or "")
+ if chunk_id:
+ compiled.setdefault(chunk_id, {"id": chunk_id, "doc_id": doc_id, "hash": saved_hash})
+ if len(rows) < 1000:
+ break
+ offset += 1000
+
+ changed_doc_ids: set[str] = set()
+ for chunk_id, item in current.items():
+ old = compiled.get(chunk_id)
+ if old is None or old["hash"] != item["hash"]:
+ changed_doc_ids.add(item["doc_id"])
+
+ # A missing current chunk means it was removed from an active document.
+ # For a disabled document, all retained MAP chunks are stale by definition.
+ for chunk_id, item in compiled.items():
+ if item["doc_id"] in disabled_doc_ids or (item["doc_id"] in active_doc_ids and chunk_id not in current):
+ changed_doc_ids.add(item["doc_id"])
+
+ return {
+ "changed": len(changed_doc_ids),
+ "changed_doc_ids": sorted(changed_doc_ids),
+ }
+
+
def _flatten_provenance_doc_ids(value) -> set[str]:
"""Normalize source_doc_ids stored as JSON strings, lists, or scalars."""
if value is None:
@@ -2368,15 +2487,27 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
return False, "Invalid Dataset ID"
docs, current_doc_ids = await _current_dataset_docs(dataset_id)
+ eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)
involved_doc_ids: set[str] = set()
+ chunk_changes = None
pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
if pack is not None:
index_nm, _ = pack
involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)
+ if kind == "wiki":
+ disabled_doc_ids = await _disabled_dataset_doc_ids(dataset_id)
+ chunk_changes = await _wiki_chunk_alteration(
+ index_nm,
+ dataset_id,
+ eligible_doc_ids,
+ disabled_doc_ids,
+ )
- eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)
- return True, _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)
+ result = _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)
+ if chunk_changes is not None:
+ result.update(chunk_changes)
+ return True, result
async def get_wiki_alteration(dataset_id: str, tenant_id: str):
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index 39dc307437..556c0c1f2a 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -90,6 +90,30 @@ DEFAULT_WIKI_MAP_WORKERS = 20
DEFAULT_WIKI_MAP_TIMEOUT = 600
+async def _wiki_disabled_doc_ids(kb_id: str) -> set[str]:
+ """Return disabled documents so cached MAP rows cannot re-enter a build."""
+ from api.db.services.document_service import DocumentService
+
+ disabled = await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)
+ return _wiki_doc_ids(disabled)
+
+
+def _wiki_doc_ids(value) -> set[str]:
+ """Normalize a scalar or list-valued document-id field."""
+ if value is None:
+ return set()
+ if isinstance(value, str):
+ value = value.strip()
+ return {value} if value else set()
+ if isinstance(value, (list, tuple, set)):
+ result: set[str] = set()
+ for item in value:
+ result.update(_wiki_doc_ids(item))
+ return result
+ value = str(value).strip()
+ return {value} if value else set()
+
+
WIKI_MAP_SYSTEM = (
"You are a knowledge extraction engine. Extract structured knowledge from the "
"provided document section. Return ONLY valid JSON matching the schema exactly. "
@@ -1182,8 +1206,9 @@ async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:
from rag.nlp import search as _rag_search
index = _rag_search.index_name(tenant_id)
+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)
condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}
- select_fields = ["id", "content_with_weight"]
+ select_fields = ["id", "content_with_weight", "doc_id"]
PAGE_SIZE = 1000
offset = 0
@@ -1213,6 +1238,8 @@ async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:
break
for row in field_map.values():
+ if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:
+ continue
content = row.get("content_with_weight")
if not isinstance(content, str) or not content:
continue
@@ -1254,6 +1281,7 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:
from rag.nlp import search as _rag_search
index = _rag_search.index_name(tenant_id)
+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)
condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}
select_fields = ["id", "doc_id"]
@@ -1282,10 +1310,8 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:
if not field_map:
break
for row in field_map.values():
- raw = row.get("doc_id")
- candidates = raw if isinstance(raw, list) else [raw]
- for d in candidates:
- if isinstance(d, str) and d and d not in seen:
+ for d in _wiki_doc_ids(row.get("doc_id")):
+ if d not in disabled_doc_ids and d not in seen:
seen.add(d)
doc_ids.append(d)
if len(field_map) < PAGE_SIZE:
@@ -1318,8 +1344,9 @@ async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:
from rag.nlp import search as _rag_search
index = _rag_search.index_name(tenant_id)
+ disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)
condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}
- select_fields = ["id", "source_chunk_ids", "chunk_hash_kwd"]
+ select_fields = ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]
PAGE_SIZE = 128
offset = 0
@@ -1352,6 +1379,8 @@ async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:
if not field_map:
break
for row in field_map.values():
+ if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:
+ continue
hh = row.get("chunk_hash_kwd")
if not isinstance(hh, str):
hh = ""
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
index 91bc0e29ad..775e77b658 100644
--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py
+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
@@ -3361,6 +3361,10 @@ async def wiki_compile_incremental(
# concept / claim / relation / topic lists are NOT separate columns, so
# they must be parsed out of that blob to rebuild the map_result shape
# that _extract_raw_entities and REDUCE expect.
+ from api.db.services.document_service import DocumentService
+ from rag.advanced_rag.knowlege_compile.wiki import _wiki_doc_ids
+
+ disabled_doc_ids = _wiki_doc_ids(await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id))
select_fields = ["content_with_weight", "doc_id"]
offset = 0
page_size = 1000
@@ -3383,6 +3387,9 @@ async def wiki_compile_incremental(
logging.exception("wiki: failed to load MAP results for kb=%s", kb_id)
break
for row in field_map.values():
+ row_doc_ids = _wiki_doc_ids(row.get("doc_id"))
+ if row_doc_ids & disabled_doc_ids:
+ continue
raw = row.get("content_with_weight")
if isinstance(raw, str) and raw:
try:
@@ -3395,7 +3402,7 @@ async def wiki_compile_incremental(
extract = None
if not isinstance(extract, dict):
continue
- extract["doc_id"] = row.get("doc_id", "")
+ extract["doc_id"] = next(iter(row_doc_ids), "")
map_results.append(extract)
if len(field_map) < page_size:
break
diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
index 54ae52f20f..5a47f3c2d9 100644
--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py
+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
@@ -57,6 +57,7 @@ from common.misc_utils import thread_pool_exec
from rag.nlp import search
from rag.advanced_rag.knowlege_compile.structure import LLMCallPool
from rag.advanced_rag.knowlege_compile.wiki import (
+ _wiki_doc_ids,
wiki_map_from_chunks,
wiki_plan_from_reduction,
wiki_reduce_from_extracts,
@@ -333,12 +334,14 @@ def _wiki_eligible_docs(all_docs, tenant_id: str, skip_doc_ids=None) -> list[tup
async def _wiki_existing_map_doc_ids(tenant_id: str, kb_id: str) -> set[str]:
from common.doc_store.doc_store_base import OrderByExpr
+ from api.db.services.document_service import DocumentService
index = search.index_name(tenant_id)
if not settings.docStoreConn.index_exist(index, kb_id):
return set()
doc_ids: set[str] = set()
+ disabled_doc_ids = _wiki_doc_ids(await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id))
select_fields = ["id", "doc_id"]
offset = 0
page_size = 1000
@@ -363,9 +366,7 @@ async def _wiki_existing_map_doc_ids(tenant_id: str, kb_id: str) -> set[str]:
if not field_map:
break
for row in field_map.values():
- doc_id = row.get("doc_id")
- if isinstance(doc_id, str) and doc_id:
- doc_ids.add(doc_id)
+ doc_ids.update(_wiki_doc_ids(row.get("doc_id")) - disabled_doc_ids)
if len(field_map) < page_size:
break
offset += page_size
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
index 1430200945..5425885d6f 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
@@ -181,6 +181,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):
monkeypatch,
"rag.advanced_rag.knowlege_compile.wiki",
WIKI_PAGE_COMPILE_KWD="wiki",
+ _chunk_hash=lambda content: "stub-hash",
)
repo_root = Path(__file__).resolve().parents[5]
diff --git a/test/unit_test/api/apps/services/test_delete_datasets.py b/test/unit_test/api/apps/services/test_delete_datasets.py
index 873791779f..12862fcc5a 100644
--- a/test/unit_test/api/apps/services/test_delete_datasets.py
+++ b/test/unit_test/api/apps/services/test_delete_datasets.py
@@ -164,6 +164,7 @@ def _load_delete_datasets_module(monkeypatch, *, f2d_rows, file_filter_delete):
monkeypatch,
"rag.advanced_rag.knowlege_compile.wiki",
WIKI_PAGE_COMPILE_KWD="wiki",
+ _chunk_hash=lambda content: "stub-hash",
)
_stub(
monkeypatch,