RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refa: remove unused knowledge compilation code (#19437)
buua4362aedd79b8fb6
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 0b8c86966a..6de92596af 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -1596,8 +1596,8 @@ async def search_datasets(tenant_id: str, req: dict):# Artifact (knowledge compilation) page surface## These three helpers power the dataset-level "Artifact" tab. They query rows-# with ``compile_kwd="wiki_page"`` written by TaskHandler's-# ``persist_wiki_pages``. The schema fields they rely on are:+# with ``compile_kwd="wiki_page"`` written by incremental Wiki REFINE. The+# schema fields they rely on are:# slug_kwd, title_kwd, page_type_kwd, content_with_weight,# topic_kwd, entity_names_kwd, outlinks_kwd, related_kb_pages_kwd,# source_chunk_ids, source_doc_idsdiff --git a/docs/guides/knowledge_compilation/runtime_configuration.md b/docs/guides/knowledge_compilation/runtime_configuration.mdindex 1588fabfa1..1223f85008 100644--- a/docs/guides/knowledge_compilation/runtime_configuration.md+++ b/docs/guides/knowledge_compilation/runtime_configuration.md@@ -28,7 +28,6 @@ the valid range and logged.| `WIKI_REFINE_WORKERS` | `4` | Number of page-refinement workers. Actual LLM concurrency is still limited by the shared pool. || `WIKI_MAP_WORKERS` | `20` | Default worker count used by direct Wiki MAP calls. || `WIKI_MAP_TIMEOUT` | `600` seconds | Timeout for one Wiki MAP extraction call. |-| `WIKI_REDUCE_TIMEOUT` | `60` seconds | Timeout for one Wiki REDUCE disambiguation call. || `WIKI_PLAN_TIMEOUT` | `600` seconds | Timeout for Wiki PLAN calls, including page planning and MAYBE resolution. || `WIKI_REFINE_TIMEOUT` | `300` seconds | Timeout for one Wiki page-writing call. || `WIKI_MERGE_TIMEOUT` | `600` seconds | Timeout for merging an existing Wiki page with newly generated content. |diff --git a/rag/advanced_rag/knowlege_compile/__init__.py b/rag/advanced_rag/knowlege_compile/__init__.pyindex 2a848ac2f1..36fd2e1f2b 100644--- a/rag/advanced_rag/knowlege_compile/__init__.py+++ b/rag/advanced_rag/knowlege_compile/__init__.py@@ -23,7 +23,6 @@ from .wiki import (WIKI_REDUCE_COMPILE_KWD,wiki_map_from_chunks,wiki_plan_from_reduction,- wiki_reduce_from_extracts,wiki_refine_from_plan,)@@ -32,7 +31,6 @@ __all__ = ["compile_structure_from_text","merge_compiled_structures","wiki_map_from_chunks",- "wiki_reduce_from_extracts","wiki_plan_from_reduction","wiki_refine_from_plan","WIKI_MAP_COMPILE_KWD",diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.pyindex 9cd36157c4..f7243c5753 100644--- a/rag/advanced_rag/knowlege_compile/_common.py+++ b/rag/advanced_rag/knowlege_compile/_common.py@@ -197,30 +197,6 @@ def union_ordered(*lists: Optional[Iterable]) -> list[str]:return seen-# ----------------------------------------------------------------------------# Token-budget calculation for split_chunks-# ------------------------------------------------------------------------------def make_input_budget(- chat_mdl,- *prompts: str,- floor: int = 1024,- utilization: float = INPUT_UTILIZATION,-) -> int:- """``chat_mdl.max_length * utilization - num_tokens(sum of prompts)``,- floored at ``floor``.-- Mirrors the budget idiom used by ``compile_structure_from_text`` and- ``wiki_map_from_chunks``: caller passes the constant prompt scaffolding- (system prompt + user template) — ``split_chunks`` then sizes batches- to leave that much room.- """- overhead = num_tokens_from_string("".join(p or "" for p in prompts))- budget = int(chat_mdl.max_length * utilization) - overhead- return max(budget, floor)--# ---------------------------------------------------------------------------# Defensive LLMBundle validation# ---------------------------------------------------------------------------@@ -256,111 +232,6 @@ def ensure_llm_bundle(mdl, method: str, *, label: str = "model"):return None-# ----------------------------------------------------------------------------# ES I/O wrappers-# ------------------------------------------------------------------------------async def doc_storage_search(- select_fields: list[str],- condition: dict,- *,- tenant_id: str,- kb_ids: list[str],- match_expressions: list | None = None,- offset: int = 0,- limit: int = 1000,- label: str = "doc_storage_search",-) -> dict:- """Thin wrapper around ``docStoreConn.search`` + ``get_fields``.-- Returns ``{row_id: row_dict}``. Returns ``{}`` on failure (with a- logged exception). ``label`` is included in the failure log so each- call site is identifiable.- """- from common import settings- from common.doc_store.doc_store_base import OrderByExpr- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields,- [],- condition,- match_expressions or [],- OrderByExpr(),- offset,- limit,- index,- kb_ids,- )- return settings.docStoreConn.get_fields(res, select_fields) or {}- except Exception:- logging.exception("%s failed (condition=%r)", label, condition)- return {}---async def doc_storage_insert(- rows: list[dict],- tenant_id: str,- kb_id: str,- *,- label: str = "doc_storage_insert",-) -> None:- """Bulk insert wrapped in ``thread_pool_exec``. Logs on failure."""- if not rows:- return- from common import settings- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- try:- await thread_pool_exec(settings.docStoreConn.insert, rows, index, kb_id)- except Exception:- logging.exception("%s failed (%d row(s))", label, len(rows))---async def doc_storage_delete(- condition: dict,- tenant_id: str,- kb_id: str,- *,- label: str = "doc_storage_delete",-) -> None:- """Bulk delete wrapped in ``thread_pool_exec``. Best-effort; logs on- failure (some callers rely on id-based upsert as a fallback)."""- from common import settings- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- try:- await thread_pool_exec(settings.docStoreConn.delete, condition, index, kb_id)- except Exception:- logging.debug("%s failed (condition=%r); caller may rely on id-upsert", label, condition)---async def doc_storage_upsert_one(- filter_condition: dict,- row: dict,- tenant_id: str,- kb_id: str,- *,- label: str = "doc_storage_upsert_one",-) -> None:- """Delete-by-filter then insert. Used when an in-place update would- require knowing the existing row's id and we'd rather drop+re-create.-- Best-effort delete (failures are debug-logged) followed by the insert.- Set ``row["id"]`` to a stable value derived from the filter- (:func:`stable_row_id`) so id-based dedup at the connector catches any- race that bypasses the delete.- """- await doc_storage_delete(filter_condition, tenant_id, kb_id, label=f"{label}.delete")- await doc_storage_insert([row], tenant_id, kb_id, label=f"{label}.insert")--# ---------------------------------------------------------------------------# Doc-vector field discovery# ---------------------------------------------------------------------------@@ -1022,12 +893,7 @@ __all__ = ["encode","tokenize_for_search","union_ordered",- "make_input_budget","ensure_llm_bundle",- "doc_storage_search",- "doc_storage_insert",- "doc_storage_delete",- "doc_storage_upsert_one","find_vec_field",# New engines"normalize_key",diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex 638f99c699..7a447b02ce 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -1699,32 +1699,6 @@ Return ONLY JSON with this exact shape:}}"""-ES_GROUP_BATCH_MERGE_PROMPT = """You are judging multiple independent ES deduplication groups.--For every group, compare every incoming item with that group's existing item.-You must make a separate duplicated decision for every incoming item. Only-incoming items marked duplicated=true may contribute to that group's merged-payload. Incoming items marked duplicated=false must remain separate. Do not-merge items from different groups and do not invent data.--Return ONLY JSON with this exact shape:-{{- "groups": [- {{- "group_id": "<group id>",- "decisions": [- {{"incoming_index": 0, "duplicated": true}},- {{"incoming_index": 1, "duplicated": false}}- ],- "merged": <merged JSON object when any item is duplicated, otherwise null>- }}- ]-}}--Groups:-{groups}-"""-ES_GROUP_DECISION_BATCH_PROMPT = """You are judging multiple independent ES deduplication groups.For every incoming item, independently decide whether it is a duplicate of@@ -1798,61 +1772,6 @@ async def _struct_judge_doc_storage_group_batch(group_specs: list[dict], chat_mdreturn result-async def _struct_merge_doc_storage_group_batch(group_specs: list[dict], chat_mdl) -> dict[str, tuple[list[dict], dict | None]]:- """Judge multiple old_id groups in one LLM request."""- prompt_groups = []- for spec in group_specs:- old_doc = spec["old_doc"]- incoming_docs = spec["incoming_docs"]- try:- existing_payload = json.loads(old_doc.get("content_with_weight") or "{}")- incoming_payloads = [json.loads(d.get("content_with_weight") or "{}") for d in incoming_docs]- except Exception:- logging.exception("merge: failed to parse grouped content_with_weight")- continue- if not isinstance(existing_payload, dict) or not all(isinstance(p, dict) for p in incoming_payloads):- continue- prompt_groups.append(- {- "group_id": spec["old_id"],- "existing": existing_payload,- "incoming": [{"index": i, "item": payload} for i, payload in enumerate(incoming_payloads)],- }- )- if not prompt_groups:- return {spec["old_id"]: (list(spec["incoming_docs"]), None) for spec in group_specs}-- user_prompt = ES_GROUP_BATCH_MERGE_PROMPT.format(groups=json.dumps(prompt_groups, ensure_ascii=False))- system_prompt = MERGE_SYSTEM_PROMPT + "\n\n" + ES_GROUP_BATCH_MERGE_PROMPT.split("Groups:", 1)[0]- res = await gen_json(system_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.0}))- raw_groups = res.get("groups") if isinstance(res, dict) else None- if not isinstance(raw_groups, list):- return {spec["old_id"]: (list(spec["incoming_docs"]), None) for spec in group_specs}-- result = {}- by_id = {spec["old_id"]: spec for spec in group_specs}- for raw in raw_groups:- if not isinstance(raw, dict) or raw.get("group_id") not in by_id:- continue- spec = by_id[raw["group_id"]]- decisions = raw.get("decisions")- merged = raw.get("merged")- if not isinstance(decisions, list):- result[spec["old_id"]] = (list(spec["incoming_docs"]), None)- continue- duplicate_indices = {item.get("incoming_index") for item in decisions if isinstance(item, dict) and item.get("duplicated") is True and isinstance(item.get("incoming_index"), int)}- duplicate_indices = {i for i in duplicate_indices if 0 <= i < len(spec["incoming_docs"])}- if not duplicate_indices or not isinstance(merged, dict):- result[spec["old_id"]] = (list(spec["incoming_docs"]), None)- continue- separate = [d for i, d in enumerate(spec["incoming_docs"]) if i not in duplicate_indices]- result[spec["old_id"]] = (separate, merged)-- for spec in group_specs:- result.setdefault(spec["old_id"], (list(spec["incoming_docs"]), None))- return result--async def _struct_merge_doc_storage_group(old_doc: dict, incoming_docs: list[dict], chat_mdl) -> tuple[list[dict], dict | None]:"""Judge one ES candidate group with one LLM request.diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex b6b277555f..4d01f04c1e 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -49,7 +49,6 @@ import xxhash as _xxhashfrom ._common import (build_chunk_batches as _build_chunk_batches,- bulk_dedup_items as _bulk_dedup_items,ensure_llm_bundle as _ensure_llm_bundle,knowledge_compile_gen_conf as _knowledge_compile_gen_conf,run_chunked_pipeline as _run_chunked_pipeline,@@ -1369,99 +1368,9 @@ async def wiki_map_from_chunks(return merged-# ----------------------------------------------------------------------------# REDUCE phase (KB-scoped)-# ----------------------------------------------------------------------------#+# PLAN input cacheWIKI_REDUCE_COMPILE_KWD = "wiki_reduce_result"-DEFAULT_WIKI_REDUCE_MERGE_THRESHOLD = 0.95-DEFAULT_WIKI_REDUCE_AMBIGUOUS_LOW = 0.75-DEFAULT_WIKI_REDUCE_AMBIGUOUS_BATCH = 50-DEFAULT_WIKI_REDUCE_TIMEOUT = _env_int("WIKI_REDUCE_TIMEOUT", 60, minimum=1)---# System prompt for the LLM disambiguation batch. The shared engine-# (``_common.bulk_dedup_items``) defaults to the same wording via-# ``_common.DEFAULT_DISAMBIGUATE_SYSTEM``; we keep the local alias so the-# constant name stays usable by call sites and external imports.-WIKI_REDUCE_DISAMBIGUATE_SYSTEM = "You are a named-entity resolution assistant. Return only JSON."---# --- ES I/O -------------------------------------------------------------------async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:- """Aggregate every wiki_map_extract row in this KB into one merged dict.-- Pages through ES if the KB has more than the per-call cap. Returns a dict- in the same shape as wiki_map_from_chunks' return value.- """- from common import settings- from common.doc_store.doc_store_base import OrderByExpr- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)- condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}- select_fields = ["id", "content_with_weight", "doc_id"]-- PAGE_SIZE = 1000- offset = 0- merged = _wiki_empty_extract()- seen_topics: set[str] = set()-- while True:- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields,- [],- condition,- [],- OrderByExpr(),- offset,- PAGE_SIZE,- index,- [kb_id],- )- field_map = settings.docStoreConn.get_fields(res, select_fields)- except Exception:- logging.exception("wiki_reduce: failed to page wiki_map_extract rows")- break-- if not field_map:- break-- for row in field_map.values():- if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:- continue- content = row.get("content_with_weight")- if not isinstance(content, str) or not content:- continue- try:- payload = json.loads(content)- except Exception:- logging.debug("wiki_reduce: skipping unparseable extract row")- continue- if not isinstance(payload, dict):- continue- for key in _EXTRACT_LIST_KEYS:- items = payload.get(key)- if isinstance(items, list):- merged[key].extend(item for item in items if isinstance(item, dict))- topics = payload.get("topics")- if isinstance(topics, list):- for t in topics:- if isinstance(t, str) and t and t not in seen_topics:- seen_topics.add(t)- merged["topics"].append(t)-- if len(field_map) < PAGE_SIZE:- break- offset += PAGE_SIZE-- return mergedasync def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:@@ -1516,84 +1425,6 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:return doc_ids-async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:- """xxh64 fingerprint of the **current** ``wiki_map_extract`` rows for- this KB — used by REDUCE / PLAN to cache-bust when MAP changed.-- Built from ``sorted((chunk_id, chunk_hash))`` so:- * adding a new chunk → new pair appears → hash flips.- * editing a chunk → MAP row deleted + re-inserted with new hash → flips.- * deleting a chunk → MAP row gone → its pair drops → flips.- * everything stable → identical hash.-- Empty / missing ``chunk_hash_kwd`` (legacy rows) defaults to '' so a- legacy KB still produces a stable hash; once those rows are touched- by an incremental MAP run, the hash naturally upgrades.-- Pages through ES in windows of ``PAGE_SIZE`` rows — single-shot- "give me everything" reads hit doc-store limits on KBs with many- chunks. The accumulated ``pairs`` are sorted once at the end so the- fingerprint is independent of page order.- """- from common import settings- from common.doc_store.doc_store_base import OrderByExpr- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)- condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}- select_fields = ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]-- PAGE_SIZE = 128- offset = 0- pairs: list[tuple[str, str]] = []- while True:- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields,- [],- condition,- [],- OrderByExpr(),- offset,- PAGE_SIZE,- index,- [kb_id],- )- field_map = settings.docStoreConn.get_fields(res, select_fields)- except Exception:- logging.exception(- "wiki: failed to compute MAP input hash for kb=%s (offset=%d)",- kb_id,- offset,- )- # Partial scan → cannot trust the resulting hash; return ""- # so REDUCE / PLAN fall through to a full re-run rather than- # cache-hitting against an incomplete fingerprint.- return ""- if not field_map:- break- for row in field_map.values():- if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:- continue- hh = row.get("chunk_hash_kwd")- if not isinstance(hh, str):- hh = ""- src = row.get("source_chunk_ids") or []- if isinstance(src, list):- for cid in src:- if isinstance(cid, str) and cid:- pairs.append((cid, hh))- if len(field_map) < PAGE_SIZE:- break- offset += PAGE_SIZE-- pairs.sort()- body = "|".join(f"{cid}:{hh}" for cid, hh in pairs) + "|" + _WIKI_PIPELINE_REV- return _xxhash.xxh64(body.encode("utf-8", "surrogatepass")).hexdigest()--async def _wiki_load_reduce_resume(tenant_id: str,kb_id: str,@@ -1642,232 +1473,6 @@ async def _wiki_load_reduce_resume(return cached, stored_hash-async def _wiki_persist_reduce(- reduced: dict,- tenant_id: str,- kb_id: str,- input_hash: str = "",- source_doc_ids: Optional[list[str]] = None,-) -> None:- """Upsert the single non-searchable wiki_reduce_result row for this KB.-- ``input_hash`` records the MAP-state fingerprint this reduction was- computed from; the next call compares it before re-running.- ``source_doc_ids`` is the set of documents that fed this reduction, used- for delete-time reference counting.- """- from common import settings- from rag.nlp import search as _rag_search-- index = _rag_search.index_name(tenant_id)- kb_id_str = str(kb_id)- content_with_weight = json.dumps(reduced, ensure_ascii=False)- # Stable id per KB so a re-run upserts the same row.- row_id = _stable_row_id(WIKI_REDUCE_COMPILE_KWD, kb_id_str)- doc = {- "id": row_id,- "doc_id": kb_id_str, # sentinel — KB-scoped row, not a real document- "compile_kwd": WIKI_REDUCE_COMPILE_KWD,- "source_id": [kb_id_str],- "source_doc_ids": list(source_doc_ids or []),- "input_hash_kwd": input_hash,- "content_with_weight": content_with_weight,- "available_int": 0,- }- try:- # Best-effort delete then insert so re-runs replace cleanly.- try:- await thread_pool_exec(- settings.docStoreConn.delete,- {"compile_kwd": WIKI_REDUCE_COMPILE_KWD},- index,- kb_id,- )- except Exception:- logging.debug("wiki_reduce: prior result delete failed; will overwrite by id")- await thread_pool_exec(settings.docStoreConn.insert, [doc], index, kb_id)- except Exception:- logging.exception("wiki_reduce: failed to persist result row")---# --- public entry -------------------------------------------------------------async def wiki_reduce_from_extracts(- chat_mdl,- embd_mdl,- tenant_id: str,- kb_id: str,- merge_threshold: float = DEFAULT_WIKI_REDUCE_MERGE_THRESHOLD,- ambiguous_low: float = DEFAULT_WIKI_REDUCE_AMBIGUOUS_LOW,- ambiguous_batch_size: int = DEFAULT_WIKI_REDUCE_AMBIGUOUS_BATCH,- llm_timeout: int = DEFAULT_WIKI_REDUCE_TIMEOUT,- force_rerun: bool = False,- callback: Optional[Callable] = None,-) -> dict:- """Phase 2 (REDUCE/Dedup) — KB-scoped.-- Loads every ``wiki_map_extract`` row in this KB (across all documents) and- produces a single canonical dict of entities/concepts via:- 1. Exact dedup by ``(normalize(name), type)`` for entities and by- ``normalize(term)`` for concepts.- 2. Embedding dedup of entity names: vectorized pairwise cosine over- ``embd_mdl.encode(...)`` output. Pairs of the same type with- similarity ≥ ``merge_threshold`` auto-merge; pairs in- ``[ambiguous_low, merge_threshold)`` go to step 3.- 3. LLM disambiguation: batches of ambiguous pairs are sent to- ``chat_mdl`` via ``gen_json``; true verdicts collapse via union-find.- 4. Apply merges: sum ``mention_count``, union ``aliases`` and- ``chunk_ids`` per canonical entity.-- The result is persisted to ES as a single non-searchable- ``wiki_reduce_result`` row per KB. Subsequent calls with- ``force_rerun=False`` (default) return the cached row immediately; pass- ``force_rerun=True`` after new ``wiki_map_extract`` rows have been added.-- Args:- chat_mdl, embd_mdl: ragflow LLMBundle instances.- tenant_id, kb_id: address the doc-store index.- merge_threshold: cosine ≥ this auto-merges. Default 0.90.- ambiguous_low: cosine in [ambiguous_low, merge_threshold) goes to LLM.- ambiguous_batch_size: max pairs per LLM disambiguation call.- llm_timeout: seconds per LLM disambiguation batch.- force_rerun: bypass the cached wiki_reduce_result.- callback: optional ``(progress: float, msg: str)`` callback.-- Returns the canonical extract dict::-- {- "entities": [{"name","type","aliases","mention_count","chunk_ids"}, ...],- "concepts": [{"term","definition_excerpt","mention_count","chunk_ids"}, ...],- "claims": [...], # pass-through from MAP- "relations": [...], # pass-through from MAP- "topics": [...], # pass-through from MAP- }- """- # Incremental gate: the current MAP-state fingerprint is the union- # of every MAP row's (chunk_id, chunk_hash). If a cached REDUCE row- # exists AND its stored input_hash equals the current fingerprint,- # the upstream chunks haven't changed → cached output is still- # correct. ``force_rerun=True`` bypasses both checks for the- # legacy / admin "rebuild from scratch" path.- current_input_hash = await _wiki_compute_map_input_hash(tenant_id, kb_id)- reduce_source_doc_ids = await _wiki_all_map_doc_ids(tenant_id, kb_id)- if not force_rerun:- cached_pair = await _wiki_load_reduce_resume(tenant_id, kb_id)- if cached_pair is not None:- cached, stored_hash = cached_pair- if stored_hash and stored_hash == current_input_hash:- if callback:- try:- callback(1.0, "wiki REDUCE: cache hit (input unchanged)")- except Exception:- pass- return cached- # Cache present but stale (no hash, or hash mismatch). Fall- # through to a full re-reduce and write a fresh stamp.-- if callback:- try:- callback(0.05, "wiki REDUCE: loading MAP extracts")- except Exception:- pass-- raw = await _wiki_load_all_map_extracts(tenant_id, kb_id)- raw_entities = raw.get("entities") or []- raw_concepts = raw.get("concepts") or []- logging.info(- "wiki_reduce: kb=%s loaded raw entities=%d concepts=%d claims=%d relations=%d",- kb_id,- len(raw_entities),- len(raw_concepts),- len(raw.get("claims") or []),- len(raw.get("relations") or []),- )-- if not raw_entities and not raw_concepts:- # Nothing to reduce; persist an empty result so resume can short-circuit.- empty = _wiki_empty_extract()- await _wiki_persist_reduce(empty, tenant_id, kb_id, input_hash=current_input_hash, source_doc_ids=reduce_source_doc_ids)- return empty-- if callback:- try:- callback(0.25, "wiki REDUCE: dedup (exact + embedding + LLM)")- except Exception:- pass-- # Entities: full three-phase dedup keyed by (normalized name, type).- canonical_entities = await _bulk_dedup_items(- raw_entities,- name_key="name",- type_key="type",- chat_mdl=chat_mdl,- embd_mdl=embd_mdl,- merge_threshold=merge_threshold,- ambiguous_low=ambiguous_low,- ambiguous_batch_size=ambiguous_batch_size,- disambiguate_system_prompt=WIKI_REDUCE_DISAMBIGUATE_SYSTEM,- llm_timeout=llm_timeout,- )-- # Concepts: exact-dedup only (current behaviour); keep the longest- # definition_excerpt across the group via aggregate_extra.- def _concept_extras(group: list[dict]) -> dict:- best_def = max(- ((c.get("definition_excerpt") or "") for c in group if isinstance(c, dict)),- key=lambda s: len(s) if isinstance(s, str) else 0,- default="",- )- return {"definition_excerpt": best_def}-- canonical_concepts = await _bulk_dedup_items(- raw_concepts,- name_key="term",- type_key=None,- aggregate_extra=_concept_extras,- )-- logging.info(- "wiki_reduce: after dedup entities=%d concepts=%d",- len(canonical_entities),- len(canonical_concepts),- )-- reduced = {- "entities": canonical_entities,- "concepts": canonical_concepts,- "claims": list(raw.get("claims") or []),- "relations": list(raw.get("relations") or []),- "topics": list(raw.get("topics") or []),- }-- if callback:- try:- callback(0.9, "wiki REDUCE: persisting result")- except Exception:- pass- await _wiki_persist_reduce(reduced, tenant_id, kb_id, input_hash=current_input_hash, source_doc_ids=reduce_source_doc_ids)-- logging.info(- "wiki_reduce: kb=%s done — entities=%d concepts=%d claims=%d relations=%d topics=%d",- kb_id,- len(reduced["entities"]),- len(reduced["concepts"]),- len(reduced["claims"]),- len(reduced["relations"]),- len(reduced["topics"]),- )-- if callback:- try:- callback(1.0, "wiki REDUCE: done")- except Exception:- pass-- return reduced--# ---------------------------------------------------------------------------# PLAN phase (KB-scoped)# ---------------------------------------------------------------------------@@ -2844,9 +2449,7 @@ WIKI_TEMPLATE_EXAMPLE = (# placeholder is filled in at request time so each artifact compilation# template can override the page-structure section without touching the# rest of the writer's guidance. Use ``_build_refine_writer_system`` to-# materialize a concrete prompt; ``WIKI_REFINE_WRITER_SYSTEM`` is-# kept as the default-filled value for back-compat with any code that-# still imports it.+# materialize a concrete prompt.WIKI_REFINE_WRITER_SYSTEM_TEMPLATE = ("You are an enterprise knowledge compilation writer. Your job is to write a single, ""high-quality wiki page by reading the SOURCE TEXT provided and using the "@@ -2897,9 +2500,6 @@ def _build_refine_writer_system(instruction: str | None = None, example: str | N"""Return the writer system prompt with separate instruction and pageexample overrides. Empty values use the built-in defaults.- The default-filled form is also exposed as- ``WIKI_REFINE_WRITER_SYSTEM`` for callers that don't have an- override to apply."""instruction_body = (instruction or "").strip() or "Follow the page structure and writing requirements below."example_body = (example or "").strip() or WIKI_TEMPLATE_EXAMPLE@@ -2909,9 +2509,6 @@ def _build_refine_writer_system(instruction: str | None = None, example: str | N)-WIKI_REFINE_WRITER_SYSTEM = _build_refine_writer_system(None)--WIKI_REFINE_WRITER_USER_TEMPLATE = """\## Task{action} the following wiki page.@@ -4148,7 +3745,6 @@ __all__ = ["WIKI_PAGE_COMPILE_KWD","WIKI_DRAFT_COMPILE_KWD","wiki_map_from_chunks",- "wiki_reduce_from_extracts","wiki_plan_from_reduction","wiki_refine_from_plan",]diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.pyindex 02392f68b6..c7535aa0e5 100644--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py@@ -521,56 +521,6 @@ def _build_canonical_entity_doc(return doc-async def _save_canonical_entity(- tenant_id: str,- kb_id: str,- entity_name: str,- entity_type: str,- aliases: list[str],- source_doc_ids: list[str],- claim_count: int,- embedding: list[float] | None = None,- source_chunk_ids: list[str] | None = None,-) -> None:- """Insert or update a canonical entity row."""- index = search.index_name(tenant_id)- doc = _build_canonical_entity_doc(- tenant_id,- kb_id,- entity_name,- entity_type,- aliases,- source_doc_ids,- claim_count,- embedding,- source_chunk_ids,- )-- condition = {"compile_kwd": [WIKI_CANONICAL_ENTITY_COMPILE_KWD], "entity_kwd": [entity_name]}- existing = await thread_pool_exec(- settings.docStoreConn.search,- ["entity_kwd"],- [],- condition,- [],- OrderByExpr(),- 0,- 1,- index,- [kb_id],- )- if settings.docStoreConn.get_fields(existing, ["entity_kwd"]):- await thread_pool_exec(- settings.docStoreConn.update,- {"compile_kwd": [WIKI_CANONICAL_ENTITY_COMPILE_KWD], "entity_kwd": entity_name},- doc,- index,- kb_id,- )- else:- await thread_pool_exec(settings.docStoreConn.insert, [doc], index, kb_id)--async def _update_canonical_entity(tenant_id: str,kb_id: str,@@ -1935,24 +1885,6 @@ async def _wiki_load_doc_page_source(return None-async def _wiki_delete_doc_page_source(- tenant_id: str,- kb_id: str,- doc_id: str,-) -> None:- """Delete doc_page_source record when a document is removed."""- index = search.index_name(tenant_id)- await thread_pool_exec(- settings.docStoreConn.delete,- {- "compile_kwd": [WIKI_DOC_PAGE_SOURCE_COMPILE_KWD],- "doc_id": [doc_id],- },- index,- kb_id,- )--# ----- Mode A: no-plan REFINE -----------------------------------------------@@ -4805,191 +4737,12 @@ async def _wiki_load_plan_group_members(tenant_id: str, kb_id: str) -> dict[str,return result-async def wiki_handle_document_deleted(- tenant_id: str,- kb_id: str,- doc_id: str,- chat_mdl,- embd_mdl,- mode: str,-) -> dict:- """Clean up wiki pages + canonical entities when a document is deleted.-- Args:- mode: ``topic`` updates plan groups; ``entity`` does not.-- Returns: {pages_modified, pages_deleted, errors}- """- summary = {"pages_modified": 0, "pages_deleted": 0, "errors": []}-- # Step 1: Update canonical entity index (decrement claim_count)- dps = await _wiki_load_doc_page_source(tenant_id, kb_id, doc_id)- if not dps:- return summary-- entity_names = dps.get("entity_names", [])- if entity_names:- canonical_index = await _load_canonical_entities(tenant_id, kb_id)- for ename in entity_names:- centry = canonical_index.get(ename)- if centry:- src_ids = centry.get("source_doc_ids", [])- if isinstance(src_ids, str):- try:- src_ids = json.loads(src_ids) if src_ids else []- except (json.JSONDecodeError, TypeError):- src_ids = []- if doc_id in src_ids:- src_ids.remove(doc_id)-- if not src_ids:- await _delete_canonical_entity(tenant_id, kb_id, ename)- else:- # Keep existing mention_count_int; the next REFINE phase- # will recalculate claims precisely from wiki pages.- # Removing the doc_id from source_doc_ids prevents future- # incremental runs from re-tracking this deletion.- await _save_canonical_entity(- tenant_id,- kb_id,- ename,- centry.get("entity_type_kwd", "entity"),- centry.get("aliases", []),- src_ids,- centry.get("mention_count_int", len(src_ids)),- source_chunk_ids=centry.get("source_chunk_ids", []),- )-- affected_page_ids = dps.get("page_ids", [])- if not affected_page_ids:- return summary-- # Step 2: Update wiki pages- all_existing_pages = await _search_existing_pages(- tenant_id,- kb_id,- ["slug_kwd", "title_kwd", "md_with_weight", "claims", "source_doc_ids", "page_version_int", "entity_names_kwd", "page_type_kwd", "topic_kwd"],- )-- for page_id in affected_page_ids:- try:- existing = all_existing_pages.get(page_id)- if not existing:- continue-- source_doc_ids = existing.get("source_doc_ids", [])- if isinstance(source_doc_ids, str):- source_doc_ids = json.loads(source_doc_ids) if source_doc_ids else []-- if doc_id in source_doc_ids:- source_doc_ids.remove(doc_id)-- page_type = existing.get("page_type_kwd", "concept" if mode == "entity" else "entity")-- if not source_doc_ids:- await _wiki_refine_page(- mode="delete",- page_id=page_id,- page_title=existing.get("title_kwd", page_id),- existing_page=existing,- page_type_kwd=page_type,- additions=None,- retractions=None,- source_chunks=[],- claims=[],- available_pages=[],- contextual_hints="",- chat_mdl=chat_mdl,- embd_mdl=embd_mdl,- tenant_id=tenant_id,- kb_id=kb_id,- page_version=existing.get("page_version_int", 0),- source_doc_ids=source_doc_ids,- )- summary["pages_deleted"] += 1- else:- existing_claims = existing.get("claims", [])- if isinstance(existing_claims, str):- existing_claims = json.loads(existing_claims) if existing_claims else []-- retractions = [c for c in existing_claims if c.get("source_doc_id") == doc_id]- retained = [c for c in existing_claims if c.get("source_doc_id") != doc_id]-- await _wiki_refine_page(- mode="modify",- page_id=page_id,- page_title=existing.get("title_kwd", page_id),- existing_page=existing,- page_type_kwd=page_type,- additions=[],- retractions=retractions,- source_chunks=[],- claims=retained,- available_pages=list(all_existing_pages.keys()),- contextual_hints=_wiki_build_contextual_hints(page_id, existing, {}),- chat_mdl=chat_mdl,- embd_mdl=embd_mdl,- tenant_id=tenant_id,- kb_id=kb_id,- page_version=existing.get("page_version_int", 0),- )- summary["pages_modified"] += 1-- if mode == "topic":- plan_condition = {- "compile_kwd": [WIKI_PLAN_GROUP_COMPILE_KWD],- "page_id": [page_id],- }- index = search.index_name(tenant_id)- res_pg = await thread_pool_exec(- settings.docStoreConn.search,- ["entity_names", "source_doc_ids"],- [],- plan_condition,- [],- OrderByExpr(),- 0,- 1,- index,- [kb_id],- )- pg_map = settings.docStoreConn.get_fields(res_pg, ["entity_names", "source_doc_ids"])- for pg_row in pg_map.values():- pg_src_ids = pg_row.get("source_doc_ids", [])- if isinstance(pg_src_ids, str):- pg_src_ids = json.loads(pg_src_ids)- if doc_id in pg_src_ids:- pg_src_ids.remove(doc_id)- await _wiki_update_plan_group(- tenant_id,- kb_id,- page_id,- entity_names=json.loads(pg_row.get("entity_names", "[]")) if isinstance(pg_row.get("entity_names"), str) else pg_row.get("entity_names", []),- page_version=existing.get("page_version_int", 0),- )- break-- except Exception:- logging.exception("wiki: document deletion cleanup failed for page=%s doc=%s", page_id, doc_id)- summary["errors"].append(f"CLEANUP_FAILED:{page_id}")-- await _wiki_delete_doc_page_source(tenant_id, kb_id, doc_id)-- try:- await _wiki_finalize(tenant_id, kb_id, embd_mdl)- except Exception:- logging.exception("wiki: FINALIZE after deletion failed")-- return summary--__all__ = ["WIKI_PAGE_COMPILE_KWD","WIKI_PLAN_GROUP_COMPILE_KWD","WIKI_DOC_PAGE_SOURCE_COMPILE_KWD","WIKI_CANONICAL_ENTITY_COMPILE_KWD","wiki_compile_incremental",- "wiki_handle_document_deleted","_wiki_reduce_entity","_wiki_reduce_batch","_wiki_match_entities",@@ -4998,7 +4751,6 @@ __all__ = ["_wiki_refine_page","_wiki_update_doc_page_source","_load_canonical_entities",- "_save_canonical_entity","_delete_canonical_entity","_extract_raw_entities",]diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex 4480e7983f..492773406c 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -355,19 +355,6 @@ async def apply_tags(docs: list[dict], ctx: TaskContext) -> None:ctx.progress_cb(msg=f"Tagging {len(docs)} chunks completed in {timer() - st:.2f}s")-def count_with_key(docs: list[dict], key: str) -> int:- """Count docs that have a specific key.-- Args:- docs: List of chunk dictionaries.- key: The key to check for.-- Returns:- Count of docs that have the key.- """- return sum(1 for d in docs if d.get(key))--# =====================================================================# Document post-chunking pipeline# ---------------------------------------------------------------------diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.pyindex fbd827391d..312800c408 100644--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py@@ -14,19 +14,12 @@# limitations under the License.#-"""KB-wide wiki / artifact compilation.+"""KB-wide incremental wiki / artifact compilation.-Extracted from ``rag.svr.task_executor_refactor.task_handler`` where the-same pipeline previously lived as a set of ``_wiki_*`` / ``_persist_wiki_*``-methods and one ``_run_wiki`` orchestrator. The public entry point is-:func:`run_wiki`.--The pipeline runs MAP per (doc, template) — each MAP call resumes from-its own ``wiki_map_extract`` ES rows — then REDUCE / PLAN / REFINE-KB-wide via ``rag.advanced_rag.knowlege_compile.wiki``. Refined pages-land in ES as searchable ``wiki_page`` rows, one-``wiki_page_topic`` row per topic, and ``wiki_entity`` /-``wiki_relation`` rows for the dataset Artifact tab's canvas graph.+The public entry point is :func:`run_wiki_incremental`. MAP extraction and+its resume state live in ``rag.advanced_rag.knowlege_compile.wiki``; the+incremental REDUCE / REFINE / FINALIZE stages live in+``rag.advanced_rag.knowlege_compile.wiki_incremental``.Design notes:@@ -36,9 +29,6 @@ Design notes:``parser_config.compilation_template_group_id`` to a template listvia the shared parser-config helper and``CompilationTemplateGroupService.resolve_template_ids``.-* The persistence helpers (``persist_wiki_pages`` etc.) are- exposed at module level for testing but are only called from- :func:`run_wiki` in production."""from __future__ import annotations@@ -46,8 +36,7 @@ from __future__ import annotationsimport asyncioimport jsonimport logging-import re-from typing import AsyncIterator, Callable, Dict, List, Optional+from typing import AsyncIterator, Callable, Dict, Listimport xxhash@@ -66,9 +55,6 @@ from rag.advanced_rag.knowlege_compile.wiki import (_wiki_load_map_extracts_for_state,_wiki_scan_current_chunk_state,wiki_map_from_chunks,- wiki_plan_from_reduction,- wiki_reduce_from_extracts,- wiki_refine_from_plan,)from rag.svr.task_executor_refactor.task_context import TaskContext@@ -93,10 +79,6 @@ WIKI_MAP_MAX_PENDING = env_int("WIKI_MAP_MAX_PENDING", 25, minimum=WIKI_MAP_LLM_# configured worker and pending limits, this bounds in-memory MAP work.WIKI_MAP_QUEUE_SIZE = 5-# REFINE pages are independent. Keep enough page workers to feed the shared-# pool; the pool itself still caps actual LLM requests at the configured size.-WIKI_REFINE_WORKERS = env_int("WIKI_REFINE_WORKERS", 4, minimum=1)-def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:def _on_concurrency_change(old: int, new: int, reason: str) -> None:@@ -123,23 +105,16 @@ def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:# ``wiki_page`` row the UI deep-links into.WIKI_GRAPH_MAX_CHUNK_IDS_PER_NODE = 64-# Title + comments stamped on every ``wiki_commit`` row produced by-# :func:`run_wiki`'s regeneration path (as opposed to the dialog-edit-# path, which carries the user's own title/comments).-WIKI_REGEN_COMMIT_TITLE = "Regenerated by artifact compilation"-WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE = "Auto-update via run_wiki (action={action})"-WIKI_MAP_COMPILE_KWD = "wiki_map_extract"WIKI_REDUCE_COMPILE_KWD = "wiki_reduce_result"WIKI_PLAN_COMPILE_KWD = "wiki_compilation_plan"WIKI_DRAFT_COMPILE_KWD = "wiki_page_draft"WIKI_PAGE_COMPILE_KWD = "wiki_page"-WIKI_PAGE_TOPIC_COMPILE_KWD = "wiki_page_topic"WIKI_DERIVED_COMPILE_KWDS = (WIKI_REDUCE_COMPILE_KWD,WIKI_PLAN_COMPILE_KWD,WIKI_DRAFT_COMPILE_KWD,WIKI_PAGE_COMPILE_KWD,- WIKI_PAGE_TOPIC_COMPILE_KWD,+ "wiki_page_topic","wiki_entity","wiki_relation","wiki_page_graph",@@ -335,8 +310,7 @@ def _wiki_eligible_docs(all_docs, tenant_id: str, skip_doc_ids=None) -> list[tuppipeline path is essential for docs uploaded/parsed through a pipeline, whichcarry their compilation templates on the pipeline's compiler rather than in``parser_config``. Returns ``(doc, template_id)`` for the first wiki template- matched per doc. Shared by :func:`run_wiki` and :func:`run_wiki_incremental`- so their eligibility can't drift.+ matched per doc."""from api.db.services.compilation_template_service import CompilationTemplateServicefrom api.apps.restful_apis.chunk_api import _compilation_template_kind@@ -704,374 +678,6 @@ async def _wiki_reset_all_wiki_state(tenant_id: str, kb_id: str) -> None:logging.exception("wiki: failed to reset all wiki state for kb=%s", kb_id)-def _wiki_topic_from_page(page: Dict, fallback: str = "") -> str:- for key in ("topic", "title", "page_type"):- value = page.get(key)- if isinstance(value, str) and value.strip():- return value.strip()- return fallback.strip()---def _wiki_topic_slug(topic: str) -> str:- slug = re.sub(r"[^a-z0-9]+", "-", topic.lower()).strip("-")- if not slug:- slug = xxhash.xxh64(topic.encode("utf-8", "surrogatepass")).hexdigest()[:12]- return f"topic/{slug[:80]}"---async def _ensure_wiki_topic_rows(- ctx: TaskContext,- index: str,- kb_id_str: str,- topics_by_name: dict[str, str],- topic_doc_ids: dict[str, list[str]] | None = None,-) -> None:- if not topics_by_name:- return-- from common.doc_store.doc_store_base import OrderByExpr- from rag.nlp import rag_tokenizer-- topic_doc_ids = topic_doc_ids or {}- topics = list(topics_by_name.keys())- existing: set[str] = set()- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- ["id", "topic_kwd"],- [],- {- "compile_kwd": [WIKI_PAGE_TOPIC_COMPILE_KWD],- "topic_kwd": topics,- },- [],- OrderByExpr(),- 0,- max(len(topics), 1),- index,- [ctx.kb_id],- )- field_map = settings.docStoreConn.get_fields(res, ["id", "topic_kwd"])- for row in (field_map or {}).values():- raw = row.get("topic_kwd")- if isinstance(raw, list):- existing.update(t for t in raw if isinstance(t, str) and t)- elif isinstance(raw, str) and raw:- existing.add(raw)- except Exception:- logging.exception(- "wiki_persist: topic existence read failed for kb=%s; inserting stable topic ids",- kb_id_str,- )-- rows: list[dict] = []- for topic, slug in topics_by_name.items():- if topic in existing:- # Topic row already present — refresh only its doc provenance so- # delete-time ref-counting stays accurate as new docs contribute- # to (or stop contributing to) the topic across recompiles.- doc_ids = topic_doc_ids.get(topic) or []- try:- await thread_pool_exec(- settings.docStoreConn.update,- {"compile_kwd": WIKI_PAGE_TOPIC_COMPILE_KWD, "topic_kwd": topic},- {"source_doc_ids": list(doc_ids)},- index,- ctx.kb_id,- )- except Exception:- logging.exception(- "wiki_persist: topic provenance update failed for kb=%s topic=%s",- kb_id_str,- topic,- )- continue- topic_id = xxhash.xxh64(- f"{kb_id_str}:{WIKI_PAGE_TOPIC_COMPILE_KWD}:{topic}".encode(- "utf-8",- "surrogatepass",- ),- ).hexdigest()- content_ltks = rag_tokenizer.tokenize(topic)- rows.append(- {- "id": topic_id,- "kb_id": kb_id_str,- "doc_id": kb_id_str,- "compile_kwd": WIKI_PAGE_TOPIC_COMPILE_KWD,- "topic_kwd": topic,- "title_kwd": topic,- "slug_kwd": slug,- "source_doc_ids": list(topic_doc_ids.get(topic) or []),- "content_with_weight": topic,- "content_ltks": content_ltks,- "content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),- "available_int": 1,- }- )-- if rows:- await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)---# ----- persistence ------------------------------------------------------async def persist_wiki_pages(- ctx: TaskContext,- pages: List[Dict],- embd_mdl,-) -> None:- """Insert one doc-store row per generated artifact page using the- knowledge-compilation schema:-- id xxh64(kb_id + ":" + slug)- compile_kwd "wiki_page"- slug_kwd page.slug- title_kwd page.title- page_type_kwd page.page_type- topic_kwd page.topic- entity_names_kwd page.entity_names- outlinks_kwd page.outlinks- related_kb_pages_kwd page.related_kb_pages- source_chunk_ids page.source_chunk_ids- source_doc_ids page.source_doc_ids- kb_id ctx.kb_id- content_with_weight rendered markdown- content_ltks /- content_sm_ltks tokenize(content_md + summary)- q_<dim>_vec embed(summary)-- ``action`` is intentionally not stored — it's a planner artifact- and has no meaning post-write.- """- if not pages:- return-- from rag.nlp import rag_tokenizer- from common.doc_store.doc_store_base import OrderByExpr- from api.db.services.file_commit_service import (- FileCommitService as WikiCommitService,- )-- index = search.index_name(ctx.tenant_id)- kb_id_str = str(ctx.kb_id)-- # Capture the prior rendered content for every slug we're about to- # overwrite, so the per-page commit row downstream has a real diff- # baseline. Prefer md_with_weight because that is the canonical field- # consumed by the Wiki page API, with content_with_weight as fallback.- # Single batch read by slug_kwd IN [...] — one round-trip regardless of- # page count. Failures here degrade gracefully.- target_slugs: list[str] = [(p.get("slug") or "").strip() for p in pages if isinstance(p.get("slug"), str) and p.get("slug")]- prior_by_slug: dict[str, str] = {}- if target_slugs:- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- ["id", "slug_kwd", "md_with_weight", "content_with_weight"],- [],- {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)},- [],- OrderByExpr(),- 0,- max(len(target_slugs), 1),- index,- [ctx.kb_id],- )- field_map = settings.docStoreConn.get_fields(- res,- ["id", "slug_kwd", "md_with_weight", "content_with_weight"],- )- for row in (field_map or {}).values():- s = row.get("slug_kwd")- c = row.get("md_with_weight") or row.get("content_with_weight")- if isinstance(s, str) and isinstance(c, str):- prior_by_slug[s] = c- except Exception:- logging.exception(- "wiki_persist: prior-content read failed for kb=%s; commit audit will treat all pages as creations",- kb_id_str,- )-- # Batch the summary embeddings in one model call. Empty strings are- # swapped for a single space so the encoder doesn't reject them —- # they still yield a vector but contribute nothing meaningful.- summaries = [(p.get("summary") or "").strip() for p in pages]- embed_inputs = [s if s else " " for s in summaries]- try:- embeddings, _ = await thread_pool_exec(embd_mdl.encode, embed_inputs)- except Exception:- logging.exception(- "wiki_persist: summary embedding batch failed for kb=%s",- kb_id_str,- )- return- try:- n_emb = len(embeddings) if embeddings is not None else 0- except TypeError:- n_emb = 0- if n_emb != len(pages):- logging.warning(- "wiki_persist: embedding count %d != pages %d for kb=%s; aborting",- n_emb,- len(pages),- kb_id_str,- )- return-- rows: List[Dict] = []- topics_by_name: dict[str, str] = {}- # Per-topic union of contributing doc ids, so topic rows can be- # reference-counted at document-delete time like every other product.- topic_doc_ids: dict[str, list[str]] = {}- for page, vec in zip(pages, embeddings):- slug = page.get("slug") or ""- if not slug:- continue- title = page.get("title") or slug- topic = _wiki_topic_from_page(page, title)- summary = page.get("summary") or ""- content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""-- vec_list = vec.tolist() if hasattr(vec, "tolist") else list(vec)- if not vec_list:- logging.warning(- "wiki_persist: empty embedding for slug=%s; skipping",- slug,- )- continue-- text_for_search = (content_md + "\n\n" + summary).strip()- content_ltks = rag_tokenizer.tokenize(text_for_search) if text_for_search else ""- content_sm_ltks = rag_tokenizer.fine_grained_tokenize(content_ltks) if content_ltks else ""-- row_id = xxhash.xxh64(- f"{kb_id_str}:{slug}".encode("utf-8", "surrogatepass"),- ).hexdigest()-- rows.append(- {- "id": row_id,- "kb_id": kb_id_str,- "doc_id": kb_id_str, # sentinel; KB-scoped row, real provenance in source_doc_ids- "compile_kwd": WIKI_PAGE_COMPILE_KWD,- "slug_kwd": slug,- "title_kwd": title,- "page_type_kwd": page.get("page_type") or "concept",- "topic_kwd": topic,- "entity_names_kwd": list(page.get("entity_names") or []),- "outlinks_kwd": list(page.get("outlinks") or []),- "outlinks_int": len(list(page.get("outlinks") or [])),- "related_kb_pages_kwd": list(page.get("related_kb_pages") or []),- "source_chunk_ids": list(page.get("source_chunk_ids") or []),- "source_doc_ids": list(page.get("source_doc_ids") or []),- "md_with_weight": content_md,- "content_with_weight": content_md,- # Summary kept verbatim alongside the rendered body so the- # viewer can render it as a distinct (smaller) block above- # the main content.- "summary_with_weight": summary,- "content_ltks": content_ltks,- "content_sm_ltks": content_sm_ltks,- f"q_{len(vec_list)}_vec": vec_list,- "available_int": 1,- }- )- if topic:- topics_by_name.setdefault(topic, _wiki_topic_slug(topic))- bucket = topic_doc_ids.setdefault(topic, [])- seen_bucket = set(bucket)- for d in page.get("source_doc_ids") or []:- if isinstance(d, str) and d and d not in seen_bucket:- seen_bucket.add(d)- bucket.append(d)-- if not rows:- return-- try:- insert_errors = await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)- except Exception:- logging.exception(- "wiki_persist: bulk insert failed for kb=%s (rows=%d)",- kb_id_str,- len(rows),- )- return-- if insert_errors:- logging.warning(- "wiki_persist: page insert returned %d error(s) for kb=%s",- len(insert_errors),- kb_id_str,- )-- # Verify the rows that are actually visible after the bulk operation. Some- # backends can report partial bulk failures without raising, so recording a- # version for every input page would create history for pages that were not- # persisted.- try:- persisted_res = await thread_pool_exec(- settings.docStoreConn.search,- ["id", "slug_kwd"],- [],- {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "id": [row["id"] for row in rows]},- [],- OrderByExpr(),- 0,- len(rows),- index,- [ctx.kb_id],- )- persisted_fields = settings.docStoreConn.get_fields(persisted_res, ["id", "slug_kwd"])- persisted_page_slugs = {row.get("slug_kwd") for row in (persisted_fields or {}).values() if isinstance(row.get("slug_kwd"), str)}- except Exception:- logging.exception(- "wiki_persist: failed to verify persisted pages for kb=%s; skipping history records",- kb_id_str,- )- return-- if topics_by_name:- try:- await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name, topic_doc_ids)- except Exception:- logging.exception(- "wiki_persist: topic row insert failed for kb=%s",- kb_id_str,- )-- # Audit trail: one ArtifactCommit row per page whose rendered- # content actually changed (record_edit silently skips empty diffs).- # Best-effort — commit failures log but don't fail the artifact- # compile.- for page in pages:- slug = (page.get("slug") or "").strip()- if not slug or slug not in persisted_page_slugs:- continue- content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""- action = (page.get("action") or "CREATE").upper()- try:- WikiCommitService.record_page_edit(- tenant_id=ctx.tenant_id,- kb_id=ctx.kb_id,- page_type=page.get("page_type") or "concept",- slug=slug,- content_before=prior_by_slug.get(slug, ""),- content_after=content_md,- title=WIKI_REGEN_COMMIT_TITLE,- comments=WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE.format(action=action),- user_id=None, # system commit — no human author- )- except Exception:- logging.exception(- "wiki_persist: commit record failed for kb=%s slug=%s",- kb_id_str,- slug,- )--def build_wiki_page_graph(pages: List[Dict],kb_id: str,@@ -1288,342 +894,6 @@ async def persist_wiki_page_graph()-# ----- main entry -------------------------------------------------------async def run_wiki(- ctx: TaskContext,- embedding_model,- load_chunks_for_doc: Callable[..., AsyncIterator[list[dict]]],-) -> None:- """KB-wide artifact compilation task.-- Runs after the user clicks the "Artifact" button in the dataset- generate menu. Iterates every doc in the KB whose parser_config- has a compilation template group resolving to an artifacts-kind- child, runs MAP per-doc (which uses ES-stored resume rows to skip- chunks already processed in a previous run), then runs REDUCE /- PLAN / REFINE KB-wide and persists pages.-- Batching: each MAP call uses ``batch_size_cap=8`` and- ``window_fraction=0.5`` — i.e. roll over to a new batch when the- current batch reaches 8 chunks OR its accumulated token count- exceeds 50% of the chat model's ``max_length``.- """- # Local imports so this module doesn't drag in the API service- # layer at import time — that's a source of circular-import risk- # given how much lives under ``api.db.services``.- from api.db.services.document_service import DocumentService- from api.db.services.knowledgebase_service import KnowledgebaseService- from api.db.services.compilation_template_service import CompilationTemplateService- from api.db.services.llm_service import LLMBundle- from api.db.joint_services.tenant_model_service import resolve_model_config-- progress = ctx.progress_cb- map_llm_pool = _create_wiki_llm_pool(progress)- progress(0.0, "Loading documents for wiki compilation...")-- # 1. Resolve KB metadata for PLAN.- ok, kb = KnowledgebaseService.get_by_id(ctx.kb_id)- if not ok:- progress(-1, f"KB {ctx.kb_id} not found.")- return- kb_name = kb.name- kb_description = kb.description-- # 2. Pick docs eligible for artifact compilation (those whose- # configured template group resolves to at least one artifacts-kind- # child). The frontend Artifact button targets the KB, but the- # per-doc opt-in is what gates inclusion.- all_docs, _ = await thread_pool_exec(- DocumentService.get_by_kb_id,- kb_id=ctx.kb_id,- page_number=0,- items_per_page=0,- orderby="create_time",- desc=False,- keywords="",- run_status=[],- types=[],- suffix=[],- )- current_doc_ids = {str(d.get("id")) for d in all_docs or [] if d.get("id")}- existing_map_doc_ids = await _wiki_existing_map_doc_ids(ctx.tenant_id, ctx.kb_id)- deleted_doc_ids = existing_map_doc_ids - current_doc_ids- if deleted_doc_ids:- progress(- 0.02,- f"Removing stale wiki state for {len(deleted_doc_ids)} deleted document(s)...",- )- await _wiki_delete_deleted_doc_state(- ctx.tenant_id,- ctx.kb_id,- deleted_doc_ids,- )-- eligible = _wiki_eligible_docs(all_docs, ctx.tenant_id)- if not eligible:- progress(1.0, _wiki_empty_eligible_message(all_docs))- return- pipeline_chat_llm_ids = _validate_wiki_eligible_docs(eligible)-- # 3. Resolve chat models. MAP is per document, so each document uses- # its pipeline Compiler's ``llm_id``. REDUCE / PLAN / REFINE are- # KB-wide and need exactly one model — we pick the first eligible- # pipeline Compiler's ``llm_id`` as the canonical KB chat model.- llm_bundle_cache: dict[str, LLMBundle] = {}-- def _bundle_for(llm_id: str) -> LLMBundle:- key = llm_id.strip()- cached = llm_bundle_cache.get(key)- if cached is not None:- return cached- cfg = resolve_model_config(- ctx.tenant_id,- LLMType.CHAT,- key,- )- bundle = LLMBundle(ctx.tenant_id, cfg, lang=ctx.language)- llm_bundle_cache[key] = bundle- return bundle-- def _stage_cb(prefix: str):- def _cb(*args, **kwargs):- try:- if args and isinstance(args[0], (int, float)):- msg = args[1] if len(args) > 1 else kwargs.get("msg", "")- progress(msg=f"{prefix} {msg}")- else:- msg = kwargs.get("msg") or (args[0] if args else "")- progress(msg=f"{prefix} {msg}")- except Exception:- logging.exception("wiki: progress callback failed")-- return _cb-- # 4. MAP per eligible doc. Historical extraction versions are keyed by- # doc_id + chunk_id + input hash, so unchanged or reverted content can- # reuse prior LLM output.- #- # Resolve templates before starting workers so the first eligible- # template remains the deterministic source for KB-wide REDUCE/PLAN/- # REFINE, independent of which document worker happens to finish first.- resolved_eligible: list[tuple[dict, str, dict]] = []- for doc, template_id in eligible:- template = CompilationTemplateService.get_saved(template_id, ctx.tenant_id)- if not template:- logging.warning(- "artifact: template %s not found for doc %s; skipping",- template_id,- doc["id"],- )- continue- resolved_eligible.append((doc, template_id, template.get("config") or {}))-- if not resolved_eligible:- progress(1.0, "No valid templates resolved for wiki compilation.")- return-- # ``kb_chat_llm_id`` is captured from the first eligible pipeline and- # used as the canonical chat model for KB-wide REDUCE/PLAN/REFINE.- # Writer instructions and page examples follow the same first-template-- # wins rule.- first_doc = resolved_eligible[0][0]- first_parser_cfg = resolved_eligible[0][2]- first_parser_cfg = first_parser_cfg if isinstance(first_parser_cfg, dict) else {}- kb_chat_llm_id = pipeline_chat_llm_ids[str(first_doc.get("id") or "")]- first_instruction = first_parser_cfg.get("instruction")- first_example = first_parser_cfg.get("example")- kb_writer_instruction: Optional[str] = first_instruction if isinstance(first_instruction, str) and first_instruction.strip() else None- kb_writer_example: Optional[str] = first_example if isinstance(first_example, str) and first_example.strip() else None- n_docs = len(resolved_eligible)- map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)- doc_stats = {- i: {- "doc": doc,- "batch_count": 0,- "saw_any": False,- "status": "ok",- "agg": {"entities": 0, "concepts": 0, "claims": 0, "relations": 0},- "map": {"requested": 0, "cache_hits": 0, "extracted": 0},- }- for i, (doc, _, _) in enumerate(resolved_eligible)- }-- async def _produce_document(i: int, job: tuple[dict, str, dict]) -> None:- doc, template_id, parser_cfg = job- doc_id = doc["id"]- stats = doc_stats[i]- progress(0.05 + 0.6 * (i / n_docs), f"MAP {i + 1}/{n_docs}: {doc.get('name', doc_id)}")- try:- async for batch in load_chunks_for_doc(- ctx.tenant_id,- ctx.kb_id,- doc_id,- batch_size=WIKI_MAP_BATCH_CHUNKS,- ):- stats["saw_any"] = True- stats["batch_count"] += 1- await map_queue.put((i, doc, template_id, parser_cfg, batch, stats["batch_count"]))- except Exception:- logging.exception("wiki: loading MAP chunks failed for doc %s", doc_id)- stats["status"] = "error"-- async def _map_worker() -> None:- while True:- item = await map_queue.get()- try:- if item is None:- return- i, doc, template_id, parser_cfg, batch, batch_no = item- doc_id = doc["id"]- stats = doc_stats[i]- map_llm_id = pipeline_chat_llm_ids[str(doc_id)]- phase1 = await wiki_map_from_chunks(- chunks=batch,- chat_mdl=map_llm_pool.wrap(- _bundle_for(map_llm_id),- priority=30,- label=f"wiki-map:{doc_id}",- context=f"{ctx.kb_id}:{doc_id}:map",- ),- embd_mdl=embedding_model,- doc_id=doc_id,- tenant_id=ctx.tenant_id,- kb_id=ctx.kb_id,- language=ctx.language,- callback=_stage_cb(f"[wiki MAP {i + 1}/{n_docs} b{batch_no}]"),- parser_config=parser_cfg,- batch_size_cap=8,- window_fraction=0.5,- # Match the shared pool width. The pool is the single- # admission/concurrency control for actual MAP LLM calls.- max_workers=WIKI_MAP_LLM_POOL_SIZE,- )- for key in stats["agg"]:- stats["agg"][key] += len(phase1.get(key) or [])- meta = phase1.get("_meta") or {}- if isinstance(meta, dict):- for key in stats["map"]:- stats["map"][key] += int(meta.get(key, 0) or 0)- except Exception:- logging.exception(- "wiki: MAP failed for doc %s batch %d",- doc_id,- batch_no,- )- stats["status"] = "error"- finally:- map_queue.task_done()-- producers = [asyncio.create_task(_produce_document(i, job)) for i, job in enumerate(resolved_eligible)]- workers = [asyncio.create_task(_map_worker()) for _ in range(WIKI_MAP_LLM_POOL_SIZE)]- try:- await asyncio.gather(*producers)- await map_queue.join()- finally:- for task in producers + workers:- if not task.done():- task.cancel()- await asyncio.gather(*producers, *workers, return_exceptions=True)-- for i, stats in doc_stats.items():- doc = stats["doc"]- doc_id = doc["id"]- if not stats["saw_any"] and stats["status"] == "ok":- stats["status"] = "empty"- logging.info("wiki: no chunks for doc %s; skipping", doc_id)- agg = stats["agg"]- map_stats = stats["map"]- logging.info(- "wiki: MAP doc=%s entities=%d concepts=%d claims=%d relations=%d (batches=%d, requested=%d cache_hits=%d extracted=%d)",- doc_id,- agg["entities"],- agg["concepts"],- agg["claims"],- agg["relations"],- stats["batch_count"],- map_stats["requested"],- map_stats["cache_hits"],- map_stats["extracted"],- )-- # 5. REDUCE / PLAN / REFINE KB-wide. Each phase has its own- # input_hash gate (REDUCE keys off the MAP-state hash, PLAN off- # REDUCE's hash, REFINE off PLAN's hash) so re-runs without an- # upstream delta short-circuit at the cache layer.- kb_chat_mdl = _bundle_for(kb_chat_llm_id)-- try:- progress(0.65, "Reducing extracts KB-wide...")- await wiki_reduce_from_extracts(- chat_mdl=map_llm_pool.wrap(- kb_chat_mdl,- priority=20,- label="wiki-reduce",- context=f"{ctx.kb_id}:reduce",- ),- embd_mdl=embedding_model,- tenant_id=ctx.tenant_id,- kb_id=ctx.kb_id,- callback=_stage_cb("[wiki REDUCE]"),- )-- progress(0.75, "Planning wiki pages...")- await wiki_plan_from_reduction(- chat_mdl=map_llm_pool.wrap(- kb_chat_mdl,- priority=20,- label="wiki-plan",- context=f"{ctx.kb_id}:plan",- ),- embd_mdl=embedding_model,- tenant_id=ctx.tenant_id,- kb_id=ctx.kb_id,- kb_name=kb_name,- kb_description=kb_description,- callback=_stage_cb("[wiki PLAN]"),- )-- progress(0.85, "Refining pages...")- pages = await wiki_refine_from_plan(- chat_mdl=map_llm_pool.wrap(- kb_chat_mdl,- priority=30,- label="wiki-refine",- context=f"{ctx.kb_id}:refine",- ),- embd_mdl=embedding_model,- tenant_id=ctx.tenant_id,- kb_id=ctx.kb_id,- max_workers=WIKI_REFINE_WORKERS,- callback=_stage_cb("[wiki REFINE]"),- instruction=kb_writer_instruction,- example=kb_writer_example,- )- except Exception:- logging.exception("wiki: REDUCE/PLAN/REFINE failed for kb %s", ctx.kb_id)- progress(-1, "Wiki pipeline failed during REDUCE/PLAN/REFINE.")- return-- # 6. Persist searchable wiki_page rows.- try:- await persist_wiki_pages(ctx=ctx, pages=pages or [], embd_mdl=embedding_model)- except Exception:- logging.exception("wiki: persist failed for kb %s", ctx.kb_id)-- # 7. Materialize the canvas graph from the refined pages.- try:- await persist_wiki_page_graph(ctx=ctx, pages=pages or [])- except Exception:- logging.exception("wiki: page-graph persist failed for kb %s", ctx.kb_id)-- progress(1.0, f"Wiki compiled {len(pages or [])} page(s).")---# ----- dual-mode incremental entry point -----------------------------------async def run_wiki_incremental(ctx: TaskContext,embedding_model,@@ -1788,7 +1058,7 @@ async def run_wiki_incremental(kb_chat_llm_id = Nonefirst_template_found = False- # 4. MAP per doc (same as run_wiki's MAP phase)+ # 4. MAP per eligible document.map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)n_docs = len(eligible)diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.pyindex f7434c9814..1f094978c1 100644--- a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py@@ -173,6 +173,7 @@ _common_mod = sys.modules["rag.advanced_rag.knowlege_compile._common"]_common_mod.build_chunk_batches = lambda *a, **k: ([], {})_common_mod.bulk_dedup_items = lambda items, *a, **k: items_common_mod.ensure_llm_bundle = lambda model: model+_common_mod.env_int = lambda _name, default, **_kwargs: default_common_mod.knowledge_compile_gen_conf = lambda *a, **k: {}_common_mod.run_chunked_pipeline = MagicMock(return_value={})_common_mod.stable_row_id = lambda *a, **k: ""diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py b/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.pyindex 74084d5923..848889aa69 100644--- a/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py+++ b/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py@@ -66,6 +66,7 @@ _STUB_NAMES = ("rag","rag.advanced_rag","rag.advanced_rag.knowlege_compile",+ "rag.advanced_rag.knowlege_compile._common","rag.nlp","rag.utils","rag.utils.es_connection",@@ -112,6 +113,10 @@ def _load_wiki_generator(monkeypatch):# Set __path__ on the knowlege_compile stub so that submodules like# ``structure`` resolve as packages rather than attribute lookups.sys.modules["rag.advanced_rag.knowlege_compile"].__path__ = [os.path.join(_REPO_ROOT, "rag", "advanced_rag", "knowlege_compile")]+ _stub_module(+ "rag.advanced_rag.knowlege_compile._common",+ env_int=lambda _name, default, **_kwargs: default,+ )sys.modules["rag.advanced_rag.knowlege_compile"].structure = ModuleType("rag.advanced_rag.knowlege_compile.structure")sys.modules["rag.advanced_rag.knowlege_compile.structure"] = sys.modules["rag.advanced_rag.knowlege_compile"].structuresys.modules["rag.advanced_rag.knowlege_compile.structure"].LLMCallPool = type("LLMCallPool", (), {})@@ -126,7 +131,6 @@ def _load_wiki_generator(monkeypatch):_wiki_scan_current_chunk_state=lambda *_args, **_kwargs: [],wiki_map_from_chunks=lambda *_args, **_kwargs: [],wiki_plan_from_reduction=lambda *_args, **_kwargs: [],- wiki_reduce_from_extracts=lambda *_args, **_kwargs: [],wiki_refine_from_plan=lambda *_args, **_kwargs: [],)_stub_module("rag.nlp", search=ModuleType("rag.nlp.search"))