Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refa: remove unused knowledge compilation code (#19437)

buua4362aedd79b8fb6
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 0b8c86966a..6de92596af 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -1596,8 +1596,8 @@ async def search_datasets(tenant_id: str, req: dict):
# Artifact (knowledge compilation) page surface
#
# These three helpers power the dataset-level "Artifact" tab. They query rows
-# with ``compile_kwd="wiki_page"`` written by TaskHandler's
-# ``persist_wiki_pages``. The schema fields they rely on are:
+# with ``compile_kwd="wiki_page"`` written by incremental Wiki REFINE. The
+# schema fields they rely on are:
# slug_kwd, title_kwd, page_type_kwd, content_with_weight,
# topic_kwd, entity_names_kwd, outlinks_kwd, related_kb_pages_kwd,
# source_chunk_ids, source_doc_ids
diff --git a/docs/guides/knowledge_compilation/runtime_configuration.md b/docs/guides/knowledge_compilation/runtime_configuration.md
index 1588fabfa1..1223f85008 100644
--- a/docs/guides/knowledge_compilation/runtime_configuration.md
+++ b/docs/guides/knowledge_compilation/runtime_configuration.md
@@ -28,7 +28,6 @@ the valid range and logged.
| `WIKI_REFINE_WORKERS` | `4` | Number of page-refinement workers. Actual LLM concurrency is still limited by the shared pool. |
| `WIKI_MAP_WORKERS` | `20` | Default worker count used by direct Wiki MAP calls. |
| `WIKI_MAP_TIMEOUT` | `600` seconds | Timeout for one Wiki MAP extraction call. |
-| `WIKI_REDUCE_TIMEOUT` | `60` seconds | Timeout for one Wiki REDUCE disambiguation call. |
| `WIKI_PLAN_TIMEOUT` | `600` seconds | Timeout for Wiki PLAN calls, including page planning and MAYBE resolution. |
| `WIKI_REFINE_TIMEOUT` | `300` seconds | Timeout for one Wiki page-writing call. |
| `WIKI_MERGE_TIMEOUT` | `600` seconds | Timeout for merging an existing Wiki page with newly generated content. |
diff --git a/rag/advanced_rag/knowlege_compile/__init__.py b/rag/advanced_rag/knowlege_compile/__init__.py
index 2a848ac2f1..36fd2e1f2b 100644
--- a/rag/advanced_rag/knowlege_compile/__init__.py
+++ b/rag/advanced_rag/knowlege_compile/__init__.py
@@ -23,7 +23,6 @@ from .wiki import (
WIKI_REDUCE_COMPILE_KWD,
wiki_map_from_chunks,
wiki_plan_from_reduction,
- wiki_reduce_from_extracts,
wiki_refine_from_plan,
)
@@ -32,7 +31,6 @@ __all__ = [
"compile_structure_from_text",
"merge_compiled_structures",
"wiki_map_from_chunks",
- "wiki_reduce_from_extracts",
"wiki_plan_from_reduction",
"wiki_refine_from_plan",
"WIKI_MAP_COMPILE_KWD",
diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.py
index 9cd36157c4..f7243c5753 100644
--- a/rag/advanced_rag/knowlege_compile/_common.py
+++ b/rag/advanced_rag/knowlege_compile/_common.py
@@ -197,30 +197,6 @@ def union_ordered(*lists: Optional[Iterable]) -> list[str]:
return seen
-# ---------------------------------------------------------------------------
-# Token-budget calculation for split_chunks
-# ---------------------------------------------------------------------------
-
-
-def make_input_budget(
- chat_mdl,
- *prompts: str,
- floor: int = 1024,
- utilization: float = INPUT_UTILIZATION,
-) -> int:
- """``chat_mdl.max_length * utilization - num_tokens(sum of prompts)``,
- floored at ``floor``.
-
- Mirrors the budget idiom used by ``compile_structure_from_text`` and
- ``wiki_map_from_chunks``: caller passes the constant prompt scaffolding
- (system prompt + user template) — ``split_chunks`` then sizes batches
- to leave that much room.
- """
- overhead = num_tokens_from_string("".join(p or "" for p in prompts))
- budget = int(chat_mdl.max_length * utilization) - overhead
- return max(budget, floor)
-
-
# ---------------------------------------------------------------------------
# Defensive LLMBundle validation
# ---------------------------------------------------------------------------
@@ -256,111 +232,6 @@ def ensure_llm_bundle(mdl, method: str, *, label: str = "model"):
return None
-# ---------------------------------------------------------------------------
-# ES I/O wrappers
-# ---------------------------------------------------------------------------
-
-
-async def doc_storage_search(
- select_fields: list[str],
- condition: dict,
- *,
- tenant_id: str,
- kb_ids: list[str],
- match_expressions: list | None = None,
- offset: int = 0,
- limit: int = 1000,
- label: str = "doc_storage_search",
-) -> dict:
- """Thin wrapper around ``docStoreConn.search`` + ``get_fields``.
-
- Returns ``{row_id: row_dict}``. Returns ``{}`` on failure (with a
- logged exception). ``label`` is included in the failure log so each
- call site is identifiable.
- """
- from common import settings
- from common.doc_store.doc_store_base import OrderByExpr
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- select_fields,
- [],
- condition,
- match_expressions or [],
- OrderByExpr(),
- offset,
- limit,
- index,
- kb_ids,
- )
- return settings.docStoreConn.get_fields(res, select_fields) or {}
- except Exception:
- logging.exception("%s failed (condition=%r)", label, condition)
- return {}
-
-
-async def doc_storage_insert(
- rows: list[dict],
- tenant_id: str,
- kb_id: str,
- *,
- label: str = "doc_storage_insert",
-) -> None:
- """Bulk insert wrapped in ``thread_pool_exec``. Logs on failure."""
- if not rows:
- return
- from common import settings
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- try:
- await thread_pool_exec(settings.docStoreConn.insert, rows, index, kb_id)
- except Exception:
- logging.exception("%s failed (%d row(s))", label, len(rows))
-
-
-async def doc_storage_delete(
- condition: dict,
- tenant_id: str,
- kb_id: str,
- *,
- label: str = "doc_storage_delete",
-) -> None:
- """Bulk delete wrapped in ``thread_pool_exec``. Best-effort; logs on
- failure (some callers rely on id-based upsert as a fallback)."""
- from common import settings
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- try:
- await thread_pool_exec(settings.docStoreConn.delete, condition, index, kb_id)
- except Exception:
- logging.debug("%s failed (condition=%r); caller may rely on id-upsert", label, condition)
-
-
-async def doc_storage_upsert_one(
- filter_condition: dict,
- row: dict,
- tenant_id: str,
- kb_id: str,
- *,
- label: str = "doc_storage_upsert_one",
-) -> None:
- """Delete-by-filter then insert. Used when an in-place update would
- require knowing the existing row's id and we'd rather drop+re-create.
-
- Best-effort delete (failures are debug-logged) followed by the insert.
- Set ``row["id"]`` to a stable value derived from the filter
- (:func:`stable_row_id`) so id-based dedup at the connector catches any
- race that bypasses the delete.
- """
- await doc_storage_delete(filter_condition, tenant_id, kb_id, label=f"{label}.delete")
- await doc_storage_insert([row], tenant_id, kb_id, label=f"{label}.insert")
-
-
# ---------------------------------------------------------------------------
# Doc-vector field discovery
# ---------------------------------------------------------------------------
@@ -1022,12 +893,7 @@ __all__ = [
"encode",
"tokenize_for_search",
"union_ordered",
- "make_input_budget",
"ensure_llm_bundle",
- "doc_storage_search",
- "doc_storage_insert",
- "doc_storage_delete",
- "doc_storage_upsert_one",
"find_vec_field",
# New engines
"normalize_key",
diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py
index 638f99c699..7a447b02ce 100644
--- a/rag/advanced_rag/knowlege_compile/structure.py
+++ b/rag/advanced_rag/knowlege_compile/structure.py
@@ -1699,32 +1699,6 @@ Return ONLY JSON with this exact shape:
}}
"""
-ES_GROUP_BATCH_MERGE_PROMPT = """You are judging multiple independent ES deduplication groups.
-
-For every group, compare every incoming item with that group's existing item.
-You must make a separate duplicated decision for every incoming item. Only
-incoming items marked duplicated=true may contribute to that group's merged
-payload. Incoming items marked duplicated=false must remain separate. Do not
-merge items from different groups and do not invent data.
-
-Return ONLY JSON with this exact shape:
-{{
- "groups": [
- {{
- "group_id": "<group id>",
- "decisions": [
- {{"incoming_index": 0, "duplicated": true}},
- {{"incoming_index": 1, "duplicated": false}}
- ],
- "merged": <merged JSON object when any item is duplicated, otherwise null>
- }}
- ]
-}}
-
-Groups:
-{groups}
-"""
-
ES_GROUP_DECISION_BATCH_PROMPT = """You are judging multiple independent ES deduplication groups.
For every incoming item, independently decide whether it is a duplicate of
@@ -1798,61 +1772,6 @@ async def _struct_judge_doc_storage_group_batch(group_specs: list[dict], chat_md
return result
-async def _struct_merge_doc_storage_group_batch(group_specs: list[dict], chat_mdl) -> dict[str, tuple[list[dict], dict | None]]:
- """Judge multiple old_id groups in one LLM request."""
- prompt_groups = []
- for spec in group_specs:
- old_doc = spec["old_doc"]
- incoming_docs = spec["incoming_docs"]
- try:
- existing_payload = json.loads(old_doc.get("content_with_weight") or "{}")
- incoming_payloads = [json.loads(d.get("content_with_weight") or "{}") for d in incoming_docs]
- except Exception:
- logging.exception("merge: failed to parse grouped content_with_weight")
- continue
- if not isinstance(existing_payload, dict) or not all(isinstance(p, dict) for p in incoming_payloads):
- continue
- prompt_groups.append(
- {
- "group_id": spec["old_id"],
- "existing": existing_payload,
- "incoming": [{"index": i, "item": payload} for i, payload in enumerate(incoming_payloads)],
- }
- )
- if not prompt_groups:
- return {spec["old_id"]: (list(spec["incoming_docs"]), None) for spec in group_specs}
-
- user_prompt = ES_GROUP_BATCH_MERGE_PROMPT.format(groups=json.dumps(prompt_groups, ensure_ascii=False))
- system_prompt = MERGE_SYSTEM_PROMPT + "\n\n" + ES_GROUP_BATCH_MERGE_PROMPT.split("Groups:", 1)[0]
- res = await gen_json(system_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.0}))
- raw_groups = res.get("groups") if isinstance(res, dict) else None
- if not isinstance(raw_groups, list):
- return {spec["old_id"]: (list(spec["incoming_docs"]), None) for spec in group_specs}
-
- result = {}
- by_id = {spec["old_id"]: spec for spec in group_specs}
- for raw in raw_groups:
- if not isinstance(raw, dict) or raw.get("group_id") not in by_id:
- continue
- spec = by_id[raw["group_id"]]
- decisions = raw.get("decisions")
- merged = raw.get("merged")
- if not isinstance(decisions, list):
- result[spec["old_id"]] = (list(spec["incoming_docs"]), None)
- continue
- duplicate_indices = {item.get("incoming_index") for item in decisions if isinstance(item, dict) and item.get("duplicated") is True and isinstance(item.get("incoming_index"), int)}
- duplicate_indices = {i for i in duplicate_indices if 0 <= i < len(spec["incoming_docs"])}
- if not duplicate_indices or not isinstance(merged, dict):
- result[spec["old_id"]] = (list(spec["incoming_docs"]), None)
- continue
- separate = [d for i, d in enumerate(spec["incoming_docs"]) if i not in duplicate_indices]
- result[spec["old_id"]] = (separate, merged)
-
- for spec in group_specs:
- result.setdefault(spec["old_id"], (list(spec["incoming_docs"]), None))
- return result
-
-
async def _struct_merge_doc_storage_group(old_doc: dict, incoming_docs: list[dict], chat_mdl) -> tuple[list[dict], dict | None]:
"""Judge one ES candidate group with one LLM request.
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index b6b277555f..4d01f04c1e 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -49,7 +49,6 @@ import xxhash as _xxhash
from ._common import (
build_chunk_batches as _build_chunk_batches,
- bulk_dedup_items as _bulk_dedup_items,
ensure_llm_bundle as _ensure_llm_bundle,
knowledge_compile_gen_conf as _knowledge_compile_gen_conf,
run_chunked_pipeline as _run_chunked_pipeline,
@@ -1369,99 +1368,9 @@ async def wiki_map_from_chunks(
return merged
-# ---------------------------------------------------------------------------
-# REDUCE phase (KB-scoped)
-# ---------------------------------------------------------------------------
-#
+# PLAN input cache
WIKI_REDUCE_COMPILE_KWD = "wiki_reduce_result"
-DEFAULT_WIKI_REDUCE_MERGE_THRESHOLD = 0.95
-DEFAULT_WIKI_REDUCE_AMBIGUOUS_LOW = 0.75
-DEFAULT_WIKI_REDUCE_AMBIGUOUS_BATCH = 50
-DEFAULT_WIKI_REDUCE_TIMEOUT = _env_int("WIKI_REDUCE_TIMEOUT", 60, minimum=1)
-
-
-# System prompt for the LLM disambiguation batch. The shared engine
-# (``_common.bulk_dedup_items``) defaults to the same wording via
-# ``_common.DEFAULT_DISAMBIGUATE_SYSTEM``; we keep the local alias so the
-# constant name stays usable by call sites and external imports.
-WIKI_REDUCE_DISAMBIGUATE_SYSTEM = "You are a named-entity resolution assistant. Return only JSON."
-
-
-# --- ES I/O ----------------------------------------------------------------
-
-
-async def _wiki_load_all_map_extracts(tenant_id: str, kb_id: str) -> dict:
- """Aggregate every wiki_map_extract row in this KB into one merged dict.
-
- Pages through ES if the KB has more than the per-call cap. Returns a dict
- in the same shape as wiki_map_from_chunks' return value.
- """
- from common import settings
- from common.doc_store.doc_store_base import OrderByExpr
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)
- condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}
- select_fields = ["id", "content_with_weight", "doc_id"]
-
- PAGE_SIZE = 1000
- offset = 0
- merged = _wiki_empty_extract()
- seen_topics: set[str] = set()
-
- while True:
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- select_fields,
- [],
- condition,
- [],
- OrderByExpr(),
- offset,
- PAGE_SIZE,
- index,
- [kb_id],
- )
- field_map = settings.docStoreConn.get_fields(res, select_fields)
- except Exception:
- logging.exception("wiki_reduce: failed to page wiki_map_extract rows")
- break
-
- if not field_map:
- break
-
- for row in field_map.values():
- if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:
- continue
- content = row.get("content_with_weight")
- if not isinstance(content, str) or not content:
- continue
- try:
- payload = json.loads(content)
- except Exception:
- logging.debug("wiki_reduce: skipping unparseable extract row")
- continue
- if not isinstance(payload, dict):
- continue
- for key in _EXTRACT_LIST_KEYS:
- items = payload.get(key)
- if isinstance(items, list):
- merged[key].extend(item for item in items if isinstance(item, dict))
- topics = payload.get("topics")
- if isinstance(topics, list):
- for t in topics:
- if isinstance(t, str) and t and t not in seen_topics:
- seen_topics.add(t)
- merged["topics"].append(t)
-
- if len(field_map) < PAGE_SIZE:
- break
- offset += PAGE_SIZE
-
- return merged
async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:
@@ -1516,84 +1425,6 @@ async def _wiki_all_map_doc_ids(tenant_id: str, kb_id: str) -> list[str]:
return doc_ids
-async def _wiki_compute_map_input_hash(tenant_id: str, kb_id: str) -> str:
- """xxh64 fingerprint of the **current** ``wiki_map_extract`` rows for
- this KB — used by REDUCE / PLAN to cache-bust when MAP changed.
-
- Built from ``sorted((chunk_id, chunk_hash))`` so:
- * adding a new chunk → new pair appears → hash flips.
- * editing a chunk → MAP row deleted + re-inserted with new hash → flips.
- * deleting a chunk → MAP row gone → its pair drops → flips.
- * everything stable → identical hash.
-
- Empty / missing ``chunk_hash_kwd`` (legacy rows) defaults to '' so a
- legacy KB still produces a stable hash; once those rows are touched
- by an incremental MAP run, the hash naturally upgrades.
-
- Pages through ES in windows of ``PAGE_SIZE`` rows — single-shot
- "give me everything" reads hit doc-store limits on KBs with many
- chunks. The accumulated ``pairs`` are sorted once at the end so the
- fingerprint is independent of page order.
- """
- from common import settings
- from common.doc_store.doc_store_base import OrderByExpr
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- disabled_doc_ids = await _wiki_disabled_doc_ids(kb_id)
- condition = {"compile_kwd": [WIKI_MAP_COMPILE_KWD]}
- select_fields = ["id", "doc_id", "source_chunk_ids", "chunk_hash_kwd"]
-
- PAGE_SIZE = 128
- offset = 0
- pairs: list[tuple[str, str]] = []
- while True:
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- select_fields,
- [],
- condition,
- [],
- OrderByExpr(),
- offset,
- PAGE_SIZE,
- index,
- [kb_id],
- )
- field_map = settings.docStoreConn.get_fields(res, select_fields)
- except Exception:
- logging.exception(
- "wiki: failed to compute MAP input hash for kb=%s (offset=%d)",
- kb_id,
- offset,
- )
- # Partial scan → cannot trust the resulting hash; return ""
- # so REDUCE / PLAN fall through to a full re-run rather than
- # cache-hitting against an incomplete fingerprint.
- return ""
- if not field_map:
- break
- for row in field_map.values():
- if _wiki_doc_ids(row.get("doc_id")) & disabled_doc_ids:
- continue
- hh = row.get("chunk_hash_kwd")
- if not isinstance(hh, str):
- hh = ""
- src = row.get("source_chunk_ids") or []
- if isinstance(src, list):
- for cid in src:
- if isinstance(cid, str) and cid:
- pairs.append((cid, hh))
- if len(field_map) < PAGE_SIZE:
- break
- offset += PAGE_SIZE
-
- pairs.sort()
- body = "|".join(f"{cid}:{hh}" for cid, hh in pairs) + "|" + _WIKI_PIPELINE_REV
- return _xxhash.xxh64(body.encode("utf-8", "surrogatepass")).hexdigest()
-
-
async def _wiki_load_reduce_resume(
tenant_id: str,
kb_id: str,
@@ -1642,232 +1473,6 @@ async def _wiki_load_reduce_resume(
return cached, stored_hash
-async def _wiki_persist_reduce(
- reduced: dict,
- tenant_id: str,
- kb_id: str,
- input_hash: str = "",
- source_doc_ids: Optional[list[str]] = None,
-) -> None:
- """Upsert the single non-searchable wiki_reduce_result row for this KB.
-
- ``input_hash`` records the MAP-state fingerprint this reduction was
- computed from; the next call compares it before re-running.
- ``source_doc_ids`` is the set of documents that fed this reduction, used
- for delete-time reference counting.
- """
- from common import settings
- from rag.nlp import search as _rag_search
-
- index = _rag_search.index_name(tenant_id)
- kb_id_str = str(kb_id)
- content_with_weight = json.dumps(reduced, ensure_ascii=False)
- # Stable id per KB so a re-run upserts the same row.
- row_id = _stable_row_id(WIKI_REDUCE_COMPILE_KWD, kb_id_str)
- doc = {
- "id": row_id,
- "doc_id": kb_id_str, # sentinel — KB-scoped row, not a real document
- "compile_kwd": WIKI_REDUCE_COMPILE_KWD,
- "source_id": [kb_id_str],
- "source_doc_ids": list(source_doc_ids or []),
- "input_hash_kwd": input_hash,
- "content_with_weight": content_with_weight,
- "available_int": 0,
- }
- try:
- # Best-effort delete then insert so re-runs replace cleanly.
- try:
- await thread_pool_exec(
- settings.docStoreConn.delete,
- {"compile_kwd": WIKI_REDUCE_COMPILE_KWD},
- index,
- kb_id,
- )
- except Exception:
- logging.debug("wiki_reduce: prior result delete failed; will overwrite by id")
- await thread_pool_exec(settings.docStoreConn.insert, [doc], index, kb_id)
- except Exception:
- logging.exception("wiki_reduce: failed to persist result row")
-
-
-# --- public entry ----------------------------------------------------------
-
-
-async def wiki_reduce_from_extracts(
- chat_mdl,
- embd_mdl,
- tenant_id: str,
- kb_id: str,
- merge_threshold: float = DEFAULT_WIKI_REDUCE_MERGE_THRESHOLD,
- ambiguous_low: float = DEFAULT_WIKI_REDUCE_AMBIGUOUS_LOW,
- ambiguous_batch_size: int = DEFAULT_WIKI_REDUCE_AMBIGUOUS_BATCH,
- llm_timeout: int = DEFAULT_WIKI_REDUCE_TIMEOUT,
- force_rerun: bool = False,
- callback: Optional[Callable] = None,
-) -> dict:
- """Phase 2 (REDUCE/Dedup) — KB-scoped.
-
- Loads every ``wiki_map_extract`` row in this KB (across all documents) and
- produces a single canonical dict of entities/concepts via:
- 1. Exact dedup by ``(normalize(name), type)`` for entities and by
- ``normalize(term)`` for concepts.
- 2. Embedding dedup of entity names: vectorized pairwise cosine over
- ``embd_mdl.encode(...)`` output. Pairs of the same type with
- similarity ≥ ``merge_threshold`` auto-merge; pairs in
- ``[ambiguous_low, merge_threshold)`` go to step 3.
- 3. LLM disambiguation: batches of ambiguous pairs are sent to
- ``chat_mdl`` via ``gen_json``; true verdicts collapse via union-find.
- 4. Apply merges: sum ``mention_count``, union ``aliases`` and
- ``chunk_ids`` per canonical entity.
-
- The result is persisted to ES as a single non-searchable
- ``wiki_reduce_result`` row per KB. Subsequent calls with
- ``force_rerun=False`` (default) return the cached row immediately; pass
- ``force_rerun=True`` after new ``wiki_map_extract`` rows have been added.
-
- Args:
- chat_mdl, embd_mdl: ragflow LLMBundle instances.
- tenant_id, kb_id: address the doc-store index.
- merge_threshold: cosine ≥ this auto-merges. Default 0.90.
- ambiguous_low: cosine in [ambiguous_low, merge_threshold) goes to LLM.
- ambiguous_batch_size: max pairs per LLM disambiguation call.
- llm_timeout: seconds per LLM disambiguation batch.
- force_rerun: bypass the cached wiki_reduce_result.
- callback: optional ``(progress: float, msg: str)`` callback.
-
- Returns the canonical extract dict::
-
- {
- "entities": [{"name","type","aliases","mention_count","chunk_ids"}, ...],
- "concepts": [{"term","definition_excerpt","mention_count","chunk_ids"}, ...],
- "claims": [...], # pass-through from MAP
- "relations": [...], # pass-through from MAP
- "topics": [...], # pass-through from MAP
- }
- """
- # Incremental gate: the current MAP-state fingerprint is the union
- # of every MAP row's (chunk_id, chunk_hash). If a cached REDUCE row
- # exists AND its stored input_hash equals the current fingerprint,
- # the upstream chunks haven't changed → cached output is still
- # correct. ``force_rerun=True`` bypasses both checks for the
- # legacy / admin "rebuild from scratch" path.
- current_input_hash = await _wiki_compute_map_input_hash(tenant_id, kb_id)
- reduce_source_doc_ids = await _wiki_all_map_doc_ids(tenant_id, kb_id)
- if not force_rerun:
- cached_pair = await _wiki_load_reduce_resume(tenant_id, kb_id)
- if cached_pair is not None:
- cached, stored_hash = cached_pair
- if stored_hash and stored_hash == current_input_hash:
- if callback:
- try:
- callback(1.0, "wiki REDUCE: cache hit (input unchanged)")
- except Exception:
- pass
- return cached
- # Cache present but stale (no hash, or hash mismatch). Fall
- # through to a full re-reduce and write a fresh stamp.
-
- if callback:
- try:
- callback(0.05, "wiki REDUCE: loading MAP extracts")
- except Exception:
- pass
-
- raw = await _wiki_load_all_map_extracts(tenant_id, kb_id)
- raw_entities = raw.get("entities") or []
- raw_concepts = raw.get("concepts") or []
- logging.info(
- "wiki_reduce: kb=%s loaded raw entities=%d concepts=%d claims=%d relations=%d",
- kb_id,
- len(raw_entities),
- len(raw_concepts),
- len(raw.get("claims") or []),
- len(raw.get("relations") or []),
- )
-
- if not raw_entities and not raw_concepts:
- # Nothing to reduce; persist an empty result so resume can short-circuit.
- empty = _wiki_empty_extract()
- await _wiki_persist_reduce(empty, tenant_id, kb_id, input_hash=current_input_hash, source_doc_ids=reduce_source_doc_ids)
- return empty
-
- if callback:
- try:
- callback(0.25, "wiki REDUCE: dedup (exact + embedding + LLM)")
- except Exception:
- pass
-
- # Entities: full three-phase dedup keyed by (normalized name, type).
- canonical_entities = await _bulk_dedup_items(
- raw_entities,
- name_key="name",
- type_key="type",
- chat_mdl=chat_mdl,
- embd_mdl=embd_mdl,
- merge_threshold=merge_threshold,
- ambiguous_low=ambiguous_low,
- ambiguous_batch_size=ambiguous_batch_size,
- disambiguate_system_prompt=WIKI_REDUCE_DISAMBIGUATE_SYSTEM,
- llm_timeout=llm_timeout,
- )
-
- # Concepts: exact-dedup only (current behaviour); keep the longest
- # definition_excerpt across the group via aggregate_extra.
- def _concept_extras(group: list[dict]) -> dict:
- best_def = max(
- ((c.get("definition_excerpt") or "") for c in group if isinstance(c, dict)),
- key=lambda s: len(s) if isinstance(s, str) else 0,
- default="",
- )
- return {"definition_excerpt": best_def}
-
- canonical_concepts = await _bulk_dedup_items(
- raw_concepts,
- name_key="term",
- type_key=None,
- aggregate_extra=_concept_extras,
- )
-
- logging.info(
- "wiki_reduce: after dedup entities=%d concepts=%d",
- len(canonical_entities),
- len(canonical_concepts),
- )
-
- reduced = {
- "entities": canonical_entities,
- "concepts": canonical_concepts,
- "claims": list(raw.get("claims") or []),
- "relations": list(raw.get("relations") or []),
- "topics": list(raw.get("topics") or []),
- }
-
- if callback:
- try:
- callback(0.9, "wiki REDUCE: persisting result")
- except Exception:
- pass
- await _wiki_persist_reduce(reduced, tenant_id, kb_id, input_hash=current_input_hash, source_doc_ids=reduce_source_doc_ids)
-
- logging.info(
- "wiki_reduce: kb=%s done — entities=%d concepts=%d claims=%d relations=%d topics=%d",
- kb_id,
- len(reduced["entities"]),
- len(reduced["concepts"]),
- len(reduced["claims"]),
- len(reduced["relations"]),
- len(reduced["topics"]),
- )
-
- if callback:
- try:
- callback(1.0, "wiki REDUCE: done")
- except Exception:
- pass
-
- return reduced
-
-
# ---------------------------------------------------------------------------
# PLAN phase (KB-scoped)
# ---------------------------------------------------------------------------
@@ -2844,9 +2449,7 @@ WIKI_TEMPLATE_EXAMPLE = (
# placeholder is filled in at request time so each artifact compilation
# template can override the page-structure section without touching the
# rest of the writer's guidance. Use ``_build_refine_writer_system`` to
-# materialize a concrete prompt; ``WIKI_REFINE_WRITER_SYSTEM`` is
-# kept as the default-filled value for back-compat with any code that
-# still imports it.
+# materialize a concrete prompt.
WIKI_REFINE_WRITER_SYSTEM_TEMPLATE = (
"You are an enterprise knowledge compilation writer. Your job is to write a single, "
"high-quality wiki page by reading the SOURCE TEXT provided and using the "
@@ -2897,9 +2500,6 @@ def _build_refine_writer_system(instruction: str | None = None, example: str | N
"""Return the writer system prompt with separate instruction and page
example overrides. Empty values use the built-in defaults.
- The default-filled form is also exposed as
- ``WIKI_REFINE_WRITER_SYSTEM`` for callers that don't have an
- override to apply.
"""
instruction_body = (instruction or "").strip() or "Follow the page structure and writing requirements below."
example_body = (example or "").strip() or WIKI_TEMPLATE_EXAMPLE
@@ -2909,9 +2509,6 @@ def _build_refine_writer_system(instruction: str | None = None, example: str | N
)
-WIKI_REFINE_WRITER_SYSTEM = _build_refine_writer_system(None)
-
-
WIKI_REFINE_WRITER_USER_TEMPLATE = """\
## Task
{action} the following wiki page.
@@ -4148,7 +3745,6 @@ __all__ = [
"WIKI_PAGE_COMPILE_KWD",
"WIKI_DRAFT_COMPILE_KWD",
"wiki_map_from_chunks",
- "wiki_reduce_from_extracts",
"wiki_plan_from_reduction",
"wiki_refine_from_plan",
]
diff --git a/rag/advanced_rag/knowlege_compile/wiki_incremental.py b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
index 02392f68b6..c7535aa0e5 100644
--- a/rag/advanced_rag/knowlege_compile/wiki_incremental.py
+++ b/rag/advanced_rag/knowlege_compile/wiki_incremental.py
@@ -521,56 +521,6 @@ def _build_canonical_entity_doc(
return doc
-async def _save_canonical_entity(
- tenant_id: str,
- kb_id: str,
- entity_name: str,
- entity_type: str,
- aliases: list[str],
- source_doc_ids: list[str],
- claim_count: int,
- embedding: list[float] | None = None,
- source_chunk_ids: list[str] | None = None,
-) -> None:
- """Insert or update a canonical entity row."""
- index = search.index_name(tenant_id)
- doc = _build_canonical_entity_doc(
- tenant_id,
- kb_id,
- entity_name,
- entity_type,
- aliases,
- source_doc_ids,
- claim_count,
- embedding,
- source_chunk_ids,
- )
-
- condition = {"compile_kwd": [WIKI_CANONICAL_ENTITY_COMPILE_KWD], "entity_kwd": [entity_name]}
- existing = await thread_pool_exec(
- settings.docStoreConn.search,
- ["entity_kwd"],
- [],
- condition,
- [],
- OrderByExpr(),
- 0,
- 1,
- index,
- [kb_id],
- )
- if settings.docStoreConn.get_fields(existing, ["entity_kwd"]):
- await thread_pool_exec(
- settings.docStoreConn.update,
- {"compile_kwd": [WIKI_CANONICAL_ENTITY_COMPILE_KWD], "entity_kwd": entity_name},
- doc,
- index,
- kb_id,
- )
- else:
- await thread_pool_exec(settings.docStoreConn.insert, [doc], index, kb_id)
-
-
async def _update_canonical_entity(
tenant_id: str,
kb_id: str,
@@ -1935,24 +1885,6 @@ async def _wiki_load_doc_page_source(
return None
-async def _wiki_delete_doc_page_source(
- tenant_id: str,
- kb_id: str,
- doc_id: str,
-) -> None:
- """Delete doc_page_source record when a document is removed."""
- index = search.index_name(tenant_id)
- await thread_pool_exec(
- settings.docStoreConn.delete,
- {
- "compile_kwd": [WIKI_DOC_PAGE_SOURCE_COMPILE_KWD],
- "doc_id": [doc_id],
- },
- index,
- kb_id,
- )
-
-
# ----- Mode A: no-plan REFINE -----------------------------------------------
@@ -4805,191 +4737,12 @@ async def _wiki_load_plan_group_members(tenant_id: str, kb_id: str) -> dict[str,
return result
-async def wiki_handle_document_deleted(
- tenant_id: str,
- kb_id: str,
- doc_id: str,
- chat_mdl,
- embd_mdl,
- mode: str,
-) -> dict:
- """Clean up wiki pages + canonical entities when a document is deleted.
-
- Args:
- mode: ``topic`` updates plan groups; ``entity`` does not.
-
- Returns: {pages_modified, pages_deleted, errors}
- """
- summary = {"pages_modified": 0, "pages_deleted": 0, "errors": []}
-
- # Step 1: Update canonical entity index (decrement claim_count)
- dps = await _wiki_load_doc_page_source(tenant_id, kb_id, doc_id)
- if not dps:
- return summary
-
- entity_names = dps.get("entity_names", [])
- if entity_names:
- canonical_index = await _load_canonical_entities(tenant_id, kb_id)
- for ename in entity_names:
- centry = canonical_index.get(ename)
- if centry:
- src_ids = centry.get("source_doc_ids", [])
- if isinstance(src_ids, str):
- try:
- src_ids = json.loads(src_ids) if src_ids else []
- except (json.JSONDecodeError, TypeError):
- src_ids = []
- if doc_id in src_ids:
- src_ids.remove(doc_id)
-
- if not src_ids:
- await _delete_canonical_entity(tenant_id, kb_id, ename)
- else:
- # Keep existing mention_count_int; the next REFINE phase
- # will recalculate claims precisely from wiki pages.
- # Removing the doc_id from source_doc_ids prevents future
- # incremental runs from re-tracking this deletion.
- await _save_canonical_entity(
- tenant_id,
- kb_id,
- ename,
- centry.get("entity_type_kwd", "entity"),
- centry.get("aliases", []),
- src_ids,
- centry.get("mention_count_int", len(src_ids)),
- source_chunk_ids=centry.get("source_chunk_ids", []),
- )
-
- affected_page_ids = dps.get("page_ids", [])
- if not affected_page_ids:
- return summary
-
- # Step 2: Update wiki pages
- all_existing_pages = await _search_existing_pages(
- tenant_id,
- kb_id,
- ["slug_kwd", "title_kwd", "md_with_weight", "claims", "source_doc_ids", "page_version_int", "entity_names_kwd", "page_type_kwd", "topic_kwd"],
- )
-
- for page_id in affected_page_ids:
- try:
- existing = all_existing_pages.get(page_id)
- if not existing:
- continue
-
- source_doc_ids = existing.get("source_doc_ids", [])
- if isinstance(source_doc_ids, str):
- source_doc_ids = json.loads(source_doc_ids) if source_doc_ids else []
-
- if doc_id in source_doc_ids:
- source_doc_ids.remove(doc_id)
-
- page_type = existing.get("page_type_kwd", "concept" if mode == "entity" else "entity")
-
- if not source_doc_ids:
- await _wiki_refine_page(
- mode="delete",
- page_id=page_id,
- page_title=existing.get("title_kwd", page_id),
- existing_page=existing,
- page_type_kwd=page_type,
- additions=None,
- retractions=None,
- source_chunks=[],
- claims=[],
- available_pages=[],
- contextual_hints="",
- chat_mdl=chat_mdl,
- embd_mdl=embd_mdl,
- tenant_id=tenant_id,
- kb_id=kb_id,
- page_version=existing.get("page_version_int", 0),
- source_doc_ids=source_doc_ids,
- )
- summary["pages_deleted"] += 1
- else:
- existing_claims = existing.get("claims", [])
- if isinstance(existing_claims, str):
- existing_claims = json.loads(existing_claims) if existing_claims else []
-
- retractions = [c for c in existing_claims if c.get("source_doc_id") == doc_id]
- retained = [c for c in existing_claims if c.get("source_doc_id") != doc_id]
-
- await _wiki_refine_page(
- mode="modify",
- page_id=page_id,
- page_title=existing.get("title_kwd", page_id),
- existing_page=existing,
- page_type_kwd=page_type,
- additions=[],
- retractions=retractions,
- source_chunks=[],
- claims=retained,
- available_pages=list(all_existing_pages.keys()),
- contextual_hints=_wiki_build_contextual_hints(page_id, existing, {}),
- chat_mdl=chat_mdl,
- embd_mdl=embd_mdl,
- tenant_id=tenant_id,
- kb_id=kb_id,
- page_version=existing.get("page_version_int", 0),
- )
- summary["pages_modified"] += 1
-
- if mode == "topic":
- plan_condition = {
- "compile_kwd": [WIKI_PLAN_GROUP_COMPILE_KWD],
- "page_id": [page_id],
- }
- index = search.index_name(tenant_id)
- res_pg = await thread_pool_exec(
- settings.docStoreConn.search,
- ["entity_names", "source_doc_ids"],
- [],
- plan_condition,
- [],
- OrderByExpr(),
- 0,
- 1,
- index,
- [kb_id],
- )
- pg_map = settings.docStoreConn.get_fields(res_pg, ["entity_names", "source_doc_ids"])
- for pg_row in pg_map.values():
- pg_src_ids = pg_row.get("source_doc_ids", [])
- if isinstance(pg_src_ids, str):
- pg_src_ids = json.loads(pg_src_ids)
- if doc_id in pg_src_ids:
- pg_src_ids.remove(doc_id)
- await _wiki_update_plan_group(
- tenant_id,
- kb_id,
- page_id,
- entity_names=json.loads(pg_row.get("entity_names", "[]")) if isinstance(pg_row.get("entity_names"), str) else pg_row.get("entity_names", []),
- page_version=existing.get("page_version_int", 0),
- )
- break
-
- except Exception:
- logging.exception("wiki: document deletion cleanup failed for page=%s doc=%s", page_id, doc_id)
- summary["errors"].append(f"CLEANUP_FAILED:{page_id}")
-
- await _wiki_delete_doc_page_source(tenant_id, kb_id, doc_id)
-
- try:
- await _wiki_finalize(tenant_id, kb_id, embd_mdl)
- except Exception:
- logging.exception("wiki: FINALIZE after deletion failed")
-
- return summary
-
-
__all__ = [
"WIKI_PAGE_COMPILE_KWD",
"WIKI_PLAN_GROUP_COMPILE_KWD",
"WIKI_DOC_PAGE_SOURCE_COMPILE_KWD",
"WIKI_CANONICAL_ENTITY_COMPILE_KWD",
"wiki_compile_incremental",
- "wiki_handle_document_deleted",
"_wiki_reduce_entity",
"_wiki_reduce_batch",
"_wiki_match_entities",
@@ -4998,7 +4751,6 @@ __all__ = [
"_wiki_refine_page",
"_wiki_update_doc_page_source",
"_load_canonical_entities",
- "_save_canonical_entity",
"_delete_canonical_entity",
"_extract_raw_entities",
]
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index 4480e7983f..492773406c 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -355,19 +355,6 @@ async def apply_tags(docs: list[dict], ctx: TaskContext) -> None:
ctx.progress_cb(msg=f"Tagging {len(docs)} chunks completed in {timer() - st:.2f}s")
-def count_with_key(docs: list[dict], key: str) -> int:
- """Count docs that have a specific key.
-
- Args:
- docs: List of chunk dictionaries.
- key: The key to check for.
-
- Returns:
- Count of docs that have the key.
- """
- return sum(1 for d in docs if d.get(key))
-
-
# =====================================================================
# Document post-chunking pipeline
# ---------------------------------------------------------------------
diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
index fbd827391d..312800c408 100644
--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py
+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py
@@ -14,19 +14,12 @@
# limitations under the License.
#
-"""KB-wide wiki / artifact compilation.
+"""KB-wide incremental wiki / artifact compilation.
-Extracted from ``rag.svr.task_executor_refactor.task_handler`` where the
-same pipeline previously lived as a set of ``_wiki_*`` / ``_persist_wiki_*``
-methods and one ``_run_wiki`` orchestrator. The public entry point is
-:func:`run_wiki`.
-
-The pipeline runs MAP per (doc, template) — each MAP call resumes from
-its own ``wiki_map_extract`` ES rows — then REDUCE / PLAN / REFINE
-KB-wide via ``rag.advanced_rag.knowlege_compile.wiki``. Refined pages
-land in ES as searchable ``wiki_page`` rows, one
-``wiki_page_topic`` row per topic, and ``wiki_entity`` /
-``wiki_relation`` rows for the dataset Artifact tab's canvas graph.
+The public entry point is :func:`run_wiki_incremental`. MAP extraction and
+its resume state live in ``rag.advanced_rag.knowlege_compile.wiki``; the
+incremental REDUCE / REFINE / FINALIZE stages live in
+``rag.advanced_rag.knowlege_compile.wiki_incremental``.
Design notes:
@@ -36,9 +29,6 @@ Design notes:
``parser_config.compilation_template_group_id`` to a template list
via the shared parser-config helper and
``CompilationTemplateGroupService.resolve_template_ids``.
-* The persistence helpers (``persist_wiki_pages`` etc.) are
- exposed at module level for testing but are only called from
- :func:`run_wiki` in production.
"""
from __future__ import annotations
@@ -46,8 +36,7 @@ from __future__ import annotations
import asyncio
import json
import logging
-import re
-from typing import AsyncIterator, Callable, Dict, List, Optional
+from typing import AsyncIterator, Callable, Dict, List
import xxhash
@@ -66,9 +55,6 @@ from rag.advanced_rag.knowlege_compile.wiki import (
_wiki_load_map_extracts_for_state,
_wiki_scan_current_chunk_state,
wiki_map_from_chunks,
- wiki_plan_from_reduction,
- wiki_reduce_from_extracts,
- wiki_refine_from_plan,
)
from rag.svr.task_executor_refactor.task_context import TaskContext
@@ -93,10 +79,6 @@ WIKI_MAP_MAX_PENDING = env_int("WIKI_MAP_MAX_PENDING", 25, minimum=WIKI_MAP_LLM_
# configured worker and pending limits, this bounds in-memory MAP work.
WIKI_MAP_QUEUE_SIZE = 5
-# REFINE pages are independent. Keep enough page workers to feed the shared
-# pool; the pool itself still caps actual LLM requests at the configured size.
-WIKI_REFINE_WORKERS = env_int("WIKI_REFINE_WORKERS", 4, minimum=1)
-
def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:
def _on_concurrency_change(old: int, new: int, reason: str) -> None:
@@ -123,23 +105,16 @@ def _create_wiki_llm_pool(progress: Callable) -> LLMCallPool:
# ``wiki_page`` row the UI deep-links into.
WIKI_GRAPH_MAX_CHUNK_IDS_PER_NODE = 64
-# Title + comments stamped on every ``wiki_commit`` row produced by
-# :func:`run_wiki`'s regeneration path (as opposed to the dialog-edit
-# path, which carries the user's own title/comments).
-WIKI_REGEN_COMMIT_TITLE = "Regenerated by artifact compilation"
-WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE = "Auto-update via run_wiki (action={action})"
-WIKI_MAP_COMPILE_KWD = "wiki_map_extract"
WIKI_REDUCE_COMPILE_KWD = "wiki_reduce_result"
WIKI_PLAN_COMPILE_KWD = "wiki_compilation_plan"
WIKI_DRAFT_COMPILE_KWD = "wiki_page_draft"
WIKI_PAGE_COMPILE_KWD = "wiki_page"
-WIKI_PAGE_TOPIC_COMPILE_KWD = "wiki_page_topic"
WIKI_DERIVED_COMPILE_KWDS = (
WIKI_REDUCE_COMPILE_KWD,
WIKI_PLAN_COMPILE_KWD,
WIKI_DRAFT_COMPILE_KWD,
WIKI_PAGE_COMPILE_KWD,
- WIKI_PAGE_TOPIC_COMPILE_KWD,
+ "wiki_page_topic",
"wiki_entity",
"wiki_relation",
"wiki_page_graph",
@@ -335,8 +310,7 @@ def _wiki_eligible_docs(all_docs, tenant_id: str, skip_doc_ids=None) -> list[tup
pipeline path is essential for docs uploaded/parsed through a pipeline, which
carry their compilation templates on the pipeline's compiler rather than in
``parser_config``. Returns ``(doc, template_id)`` for the first wiki template
- matched per doc. Shared by :func:`run_wiki` and :func:`run_wiki_incremental`
- so their eligibility can't drift.
+ matched per doc.
"""
from api.db.services.compilation_template_service import CompilationTemplateService
from api.apps.restful_apis.chunk_api import _compilation_template_kind
@@ -704,374 +678,6 @@ async def _wiki_reset_all_wiki_state(tenant_id: str, kb_id: str) -> None:
logging.exception("wiki: failed to reset all wiki state for kb=%s", kb_id)
-def _wiki_topic_from_page(page: Dict, fallback: str = "") -> str:
- for key in ("topic", "title", "page_type"):
- value = page.get(key)
- if isinstance(value, str) and value.strip():
- return value.strip()
- return fallback.strip()
-
-
-def _wiki_topic_slug(topic: str) -> str:
- slug = re.sub(r"[^a-z0-9]+", "-", topic.lower()).strip("-")
- if not slug:
- slug = xxhash.xxh64(topic.encode("utf-8", "surrogatepass")).hexdigest()[:12]
- return f"topic/{slug[:80]}"
-
-
-async def _ensure_wiki_topic_rows(
- ctx: TaskContext,
- index: str,
- kb_id_str: str,
- topics_by_name: dict[str, str],
- topic_doc_ids: dict[str, list[str]] | None = None,
-) -> None:
- if not topics_by_name:
- return
-
- from common.doc_store.doc_store_base import OrderByExpr
- from rag.nlp import rag_tokenizer
-
- topic_doc_ids = topic_doc_ids or {}
- topics = list(topics_by_name.keys())
- existing: set[str] = set()
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- ["id", "topic_kwd"],
- [],
- {
- "compile_kwd": [WIKI_PAGE_TOPIC_COMPILE_KWD],
- "topic_kwd": topics,
- },
- [],
- OrderByExpr(),
- 0,
- max(len(topics), 1),
- index,
- [ctx.kb_id],
- )
- field_map = settings.docStoreConn.get_fields(res, ["id", "topic_kwd"])
- for row in (field_map or {}).values():
- raw = row.get("topic_kwd")
- if isinstance(raw, list):
- existing.update(t for t in raw if isinstance(t, str) and t)
- elif isinstance(raw, str) and raw:
- existing.add(raw)
- except Exception:
- logging.exception(
- "wiki_persist: topic existence read failed for kb=%s; inserting stable topic ids",
- kb_id_str,
- )
-
- rows: list[dict] = []
- for topic, slug in topics_by_name.items():
- if topic in existing:
- # Topic row already present — refresh only its doc provenance so
- # delete-time ref-counting stays accurate as new docs contribute
- # to (or stop contributing to) the topic across recompiles.
- doc_ids = topic_doc_ids.get(topic) or []
- try:
- await thread_pool_exec(
- settings.docStoreConn.update,
- {"compile_kwd": WIKI_PAGE_TOPIC_COMPILE_KWD, "topic_kwd": topic},
- {"source_doc_ids": list(doc_ids)},
- index,
- ctx.kb_id,
- )
- except Exception:
- logging.exception(
- "wiki_persist: topic provenance update failed for kb=%s topic=%s",
- kb_id_str,
- topic,
- )
- continue
- topic_id = xxhash.xxh64(
- f"{kb_id_str}:{WIKI_PAGE_TOPIC_COMPILE_KWD}:{topic}".encode(
- "utf-8",
- "surrogatepass",
- ),
- ).hexdigest()
- content_ltks = rag_tokenizer.tokenize(topic)
- rows.append(
- {
- "id": topic_id,
- "kb_id": kb_id_str,
- "doc_id": kb_id_str,
- "compile_kwd": WIKI_PAGE_TOPIC_COMPILE_KWD,
- "topic_kwd": topic,
- "title_kwd": topic,
- "slug_kwd": slug,
- "source_doc_ids": list(topic_doc_ids.get(topic) or []),
- "content_with_weight": topic,
- "content_ltks": content_ltks,
- "content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),
- "available_int": 1,
- }
- )
-
- if rows:
- await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)
-
-
-# ----- persistence ---------------------------------------------------
-
-
-async def persist_wiki_pages(
- ctx: TaskContext,
- pages: List[Dict],
- embd_mdl,
-) -> None:
- """Insert one doc-store row per generated artifact page using the
- knowledge-compilation schema:
-
- id xxh64(kb_id + ":" + slug)
- compile_kwd "wiki_page"
- slug_kwd page.slug
- title_kwd page.title
- page_type_kwd page.page_type
- topic_kwd page.topic
- entity_names_kwd page.entity_names
- outlinks_kwd page.outlinks
- related_kb_pages_kwd page.related_kb_pages
- source_chunk_ids page.source_chunk_ids
- source_doc_ids page.source_doc_ids
- kb_id ctx.kb_id
- content_with_weight rendered markdown
- content_ltks /
- content_sm_ltks tokenize(content_md + summary)
- q_<dim>_vec embed(summary)
-
- ``action`` is intentionally not stored — it's a planner artifact
- and has no meaning post-write.
- """
- if not pages:
- return
-
- from rag.nlp import rag_tokenizer
- from common.doc_store.doc_store_base import OrderByExpr
- from api.db.services.file_commit_service import (
- FileCommitService as WikiCommitService,
- )
-
- index = search.index_name(ctx.tenant_id)
- kb_id_str = str(ctx.kb_id)
-
- # Capture the prior rendered content for every slug we're about to
- # overwrite, so the per-page commit row downstream has a real diff
- # baseline. Prefer md_with_weight because that is the canonical field
- # consumed by the Wiki page API, with content_with_weight as fallback.
- # Single batch read by slug_kwd IN [...] — one round-trip regardless of
- # page count. Failures here degrade gracefully.
- target_slugs: list[str] = [(p.get("slug") or "").strip() for p in pages if isinstance(p.get("slug"), str) and p.get("slug")]
- prior_by_slug: dict[str, str] = {}
- if target_slugs:
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- ["id", "slug_kwd", "md_with_weight", "content_with_weight"],
- [],
- {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)},
- [],
- OrderByExpr(),
- 0,
- max(len(target_slugs), 1),
- index,
- [ctx.kb_id],
- )
- field_map = settings.docStoreConn.get_fields(
- res,
- ["id", "slug_kwd", "md_with_weight", "content_with_weight"],
- )
- for row in (field_map or {}).values():
- s = row.get("slug_kwd")
- c = row.get("md_with_weight") or row.get("content_with_weight")
- if isinstance(s, str) and isinstance(c, str):
- prior_by_slug[s] = c
- except Exception:
- logging.exception(
- "wiki_persist: prior-content read failed for kb=%s; commit audit will treat all pages as creations",
- kb_id_str,
- )
-
- # Batch the summary embeddings in one model call. Empty strings are
- # swapped for a single space so the encoder doesn't reject them —
- # they still yield a vector but contribute nothing meaningful.
- summaries = [(p.get("summary") or "").strip() for p in pages]
- embed_inputs = [s if s else " " for s in summaries]
- try:
- embeddings, _ = await thread_pool_exec(embd_mdl.encode, embed_inputs)
- except Exception:
- logging.exception(
- "wiki_persist: summary embedding batch failed for kb=%s",
- kb_id_str,
- )
- return
- try:
- n_emb = len(embeddings) if embeddings is not None else 0
- except TypeError:
- n_emb = 0
- if n_emb != len(pages):
- logging.warning(
- "wiki_persist: embedding count %d != pages %d for kb=%s; aborting",
- n_emb,
- len(pages),
- kb_id_str,
- )
- return
-
- rows: List[Dict] = []
- topics_by_name: dict[str, str] = {}
- # Per-topic union of contributing doc ids, so topic rows can be
- # reference-counted at document-delete time like every other product.
- topic_doc_ids: dict[str, list[str]] = {}
- for page, vec in zip(pages, embeddings):
- slug = page.get("slug") or ""
- if not slug:
- continue
- title = page.get("title") or slug
- topic = _wiki_topic_from_page(page, title)
- summary = page.get("summary") or ""
- content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""
-
- vec_list = vec.tolist() if hasattr(vec, "tolist") else list(vec)
- if not vec_list:
- logging.warning(
- "wiki_persist: empty embedding for slug=%s; skipping",
- slug,
- )
- continue
-
- text_for_search = (content_md + "\n\n" + summary).strip()
- content_ltks = rag_tokenizer.tokenize(text_for_search) if text_for_search else ""
- content_sm_ltks = rag_tokenizer.fine_grained_tokenize(content_ltks) if content_ltks else ""
-
- row_id = xxhash.xxh64(
- f"{kb_id_str}:{slug}".encode("utf-8", "surrogatepass"),
- ).hexdigest()
-
- rows.append(
- {
- "id": row_id,
- "kb_id": kb_id_str,
- "doc_id": kb_id_str, # sentinel; KB-scoped row, real provenance in source_doc_ids
- "compile_kwd": WIKI_PAGE_COMPILE_KWD,
- "slug_kwd": slug,
- "title_kwd": title,
- "page_type_kwd": page.get("page_type") or "concept",
- "topic_kwd": topic,
- "entity_names_kwd": list(page.get("entity_names") or []),
- "outlinks_kwd": list(page.get("outlinks") or []),
- "outlinks_int": len(list(page.get("outlinks") or [])),
- "related_kb_pages_kwd": list(page.get("related_kb_pages") or []),
- "source_chunk_ids": list(page.get("source_chunk_ids") or []),
- "source_doc_ids": list(page.get("source_doc_ids") or []),
- "md_with_weight": content_md,
- "content_with_weight": content_md,
- # Summary kept verbatim alongside the rendered body so the
- # viewer can render it as a distinct (smaller) block above
- # the main content.
- "summary_with_weight": summary,
- "content_ltks": content_ltks,
- "content_sm_ltks": content_sm_ltks,
- f"q_{len(vec_list)}_vec": vec_list,
- "available_int": 1,
- }
- )
- if topic:
- topics_by_name.setdefault(topic, _wiki_topic_slug(topic))
- bucket = topic_doc_ids.setdefault(topic, [])
- seen_bucket = set(bucket)
- for d in page.get("source_doc_ids") or []:
- if isinstance(d, str) and d and d not in seen_bucket:
- seen_bucket.add(d)
- bucket.append(d)
-
- if not rows:
- return
-
- try:
- insert_errors = await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)
- except Exception:
- logging.exception(
- "wiki_persist: bulk insert failed for kb=%s (rows=%d)",
- kb_id_str,
- len(rows),
- )
- return
-
- if insert_errors:
- logging.warning(
- "wiki_persist: page insert returned %d error(s) for kb=%s",
- len(insert_errors),
- kb_id_str,
- )
-
- # Verify the rows that are actually visible after the bulk operation. Some
- # backends can report partial bulk failures without raising, so recording a
- # version for every input page would create history for pages that were not
- # persisted.
- try:
- persisted_res = await thread_pool_exec(
- settings.docStoreConn.search,
- ["id", "slug_kwd"],
- [],
- {"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "id": [row["id"] for row in rows]},
- [],
- OrderByExpr(),
- 0,
- len(rows),
- index,
- [ctx.kb_id],
- )
- persisted_fields = settings.docStoreConn.get_fields(persisted_res, ["id", "slug_kwd"])
- persisted_page_slugs = {row.get("slug_kwd") for row in (persisted_fields or {}).values() if isinstance(row.get("slug_kwd"), str)}
- except Exception:
- logging.exception(
- "wiki_persist: failed to verify persisted pages for kb=%s; skipping history records",
- kb_id_str,
- )
- return
-
- if topics_by_name:
- try:
- await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name, topic_doc_ids)
- except Exception:
- logging.exception(
- "wiki_persist: topic row insert failed for kb=%s",
- kb_id_str,
- )
-
- # Audit trail: one ArtifactCommit row per page whose rendered
- # content actually changed (record_edit silently skips empty diffs).
- # Best-effort — commit failures log but don't fail the artifact
- # compile.
- for page in pages:
- slug = (page.get("slug") or "").strip()
- if not slug or slug not in persisted_page_slugs:
- continue
- content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""
- action = (page.get("action") or "CREATE").upper()
- try:
- WikiCommitService.record_page_edit(
- tenant_id=ctx.tenant_id,
- kb_id=ctx.kb_id,
- page_type=page.get("page_type") or "concept",
- slug=slug,
- content_before=prior_by_slug.get(slug, ""),
- content_after=content_md,
- title=WIKI_REGEN_COMMIT_TITLE,
- comments=WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE.format(action=action),
- user_id=None, # system commit — no human author
- )
- except Exception:
- logging.exception(
- "wiki_persist: commit record failed for kb=%s slug=%s",
- kb_id_str,
- slug,
- )
-
-
def build_wiki_page_graph(
pages: List[Dict],
kb_id: str,
@@ -1288,342 +894,6 @@ async def persist_wiki_page_graph(
)
-# ----- main entry ----------------------------------------------------
-
-
-async def run_wiki(
- ctx: TaskContext,
- embedding_model,
- load_chunks_for_doc: Callable[..., AsyncIterator[list[dict]]],
-) -> None:
- """KB-wide artifact compilation task.
-
- Runs after the user clicks the "Artifact" button in the dataset
- generate menu. Iterates every doc in the KB whose parser_config
- has a compilation template group resolving to an artifacts-kind
- child, runs MAP per-doc (which uses ES-stored resume rows to skip
- chunks already processed in a previous run), then runs REDUCE /
- PLAN / REFINE KB-wide and persists pages.
-
- Batching: each MAP call uses ``batch_size_cap=8`` and
- ``window_fraction=0.5`` — i.e. roll over to a new batch when the
- current batch reaches 8 chunks OR its accumulated token count
- exceeds 50% of the chat model's ``max_length``.
- """
- # Local imports so this module doesn't drag in the API service
- # layer at import time — that's a source of circular-import risk
- # given how much lives under ``api.db.services``.
- from api.db.services.document_service import DocumentService
- from api.db.services.knowledgebase_service import KnowledgebaseService
- from api.db.services.compilation_template_service import CompilationTemplateService
- from api.db.services.llm_service import LLMBundle
- from api.db.joint_services.tenant_model_service import resolve_model_config
-
- progress = ctx.progress_cb
- map_llm_pool = _create_wiki_llm_pool(progress)
- progress(0.0, "Loading documents for wiki compilation...")
-
- # 1. Resolve KB metadata for PLAN.
- ok, kb = KnowledgebaseService.get_by_id(ctx.kb_id)
- if not ok:
- progress(-1, f"KB {ctx.kb_id} not found.")
- return
- kb_name = kb.name
- kb_description = kb.description
-
- # 2. Pick docs eligible for artifact compilation (those whose
- # configured template group resolves to at least one artifacts-kind
- # child). The frontend Artifact button targets the KB, but the
- # per-doc opt-in is what gates inclusion.
- all_docs, _ = await thread_pool_exec(
- DocumentService.get_by_kb_id,
- kb_id=ctx.kb_id,
- page_number=0,
- items_per_page=0,
- orderby="create_time",
- desc=False,
- keywords="",
- run_status=[],
- types=[],
- suffix=[],
- )
- current_doc_ids = {str(d.get("id")) for d in all_docs or [] if d.get("id")}
- existing_map_doc_ids = await _wiki_existing_map_doc_ids(ctx.tenant_id, ctx.kb_id)
- deleted_doc_ids = existing_map_doc_ids - current_doc_ids
- if deleted_doc_ids:
- progress(
- 0.02,
- f"Removing stale wiki state for {len(deleted_doc_ids)} deleted document(s)...",
- )
- await _wiki_delete_deleted_doc_state(
- ctx.tenant_id,
- ctx.kb_id,
- deleted_doc_ids,
- )
-
- eligible = _wiki_eligible_docs(all_docs, ctx.tenant_id)
- if not eligible:
- progress(1.0, _wiki_empty_eligible_message(all_docs))
- return
- pipeline_chat_llm_ids = _validate_wiki_eligible_docs(eligible)
-
- # 3. Resolve chat models. MAP is per document, so each document uses
- # its pipeline Compiler's ``llm_id``. REDUCE / PLAN / REFINE are
- # KB-wide and need exactly one model — we pick the first eligible
- # pipeline Compiler's ``llm_id`` as the canonical KB chat model.
- llm_bundle_cache: dict[str, LLMBundle] = {}
-
- def _bundle_for(llm_id: str) -> LLMBundle:
- key = llm_id.strip()
- cached = llm_bundle_cache.get(key)
- if cached is not None:
- return cached
- cfg = resolve_model_config(
- ctx.tenant_id,
- LLMType.CHAT,
- key,
- )
- bundle = LLMBundle(ctx.tenant_id, cfg, lang=ctx.language)
- llm_bundle_cache[key] = bundle
- return bundle
-
- def _stage_cb(prefix: str):
- def _cb(*args, **kwargs):
- try:
- if args and isinstance(args[0], (int, float)):
- msg = args[1] if len(args) > 1 else kwargs.get("msg", "")
- progress(msg=f"{prefix} {msg}")
- else:
- msg = kwargs.get("msg") or (args[0] if args else "")
- progress(msg=f"{prefix} {msg}")
- except Exception:
- logging.exception("wiki: progress callback failed")
-
- return _cb
-
- # 4. MAP per eligible doc. Historical extraction versions are keyed by
- # doc_id + chunk_id + input hash, so unchanged or reverted content can
- # reuse prior LLM output.
- #
- # Resolve templates before starting workers so the first eligible
- # template remains the deterministic source for KB-wide REDUCE/PLAN/
- # REFINE, independent of which document worker happens to finish first.
- resolved_eligible: list[tuple[dict, str, dict]] = []
- for doc, template_id in eligible:
- template = CompilationTemplateService.get_saved(template_id, ctx.tenant_id)
- if not template:
- logging.warning(
- "artifact: template %s not found for doc %s; skipping",
- template_id,
- doc["id"],
- )
- continue
- resolved_eligible.append((doc, template_id, template.get("config") or {}))
-
- if not resolved_eligible:
- progress(1.0, "No valid templates resolved for wiki compilation.")
- return
-
- # ``kb_chat_llm_id`` is captured from the first eligible pipeline and
- # used as the canonical chat model for KB-wide REDUCE/PLAN/REFINE.
- # Writer instructions and page examples follow the same first-template-
- # wins rule.
- first_doc = resolved_eligible[0][0]
- first_parser_cfg = resolved_eligible[0][2]
- first_parser_cfg = first_parser_cfg if isinstance(first_parser_cfg, dict) else {}
- kb_chat_llm_id = pipeline_chat_llm_ids[str(first_doc.get("id") or "")]
- first_instruction = first_parser_cfg.get("instruction")
- first_example = first_parser_cfg.get("example")
- kb_writer_instruction: Optional[str] = first_instruction if isinstance(first_instruction, str) and first_instruction.strip() else None
- kb_writer_example: Optional[str] = first_example if isinstance(first_example, str) and first_example.strip() else None
- n_docs = len(resolved_eligible)
- map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)
- doc_stats = {
- i: {
- "doc": doc,
- "batch_count": 0,
- "saw_any": False,
- "status": "ok",
- "agg": {"entities": 0, "concepts": 0, "claims": 0, "relations": 0},
- "map": {"requested": 0, "cache_hits": 0, "extracted": 0},
- }
- for i, (doc, _, _) in enumerate(resolved_eligible)
- }
-
- async def _produce_document(i: int, job: tuple[dict, str, dict]) -> None:
- doc, template_id, parser_cfg = job
- doc_id = doc["id"]
- stats = doc_stats[i]
- progress(0.05 + 0.6 * (i / n_docs), f"MAP {i + 1}/{n_docs}: {doc.get('name', doc_id)}")
- try:
- async for batch in load_chunks_for_doc(
- ctx.tenant_id,
- ctx.kb_id,
- doc_id,
- batch_size=WIKI_MAP_BATCH_CHUNKS,
- ):
- stats["saw_any"] = True
- stats["batch_count"] += 1
- await map_queue.put((i, doc, template_id, parser_cfg, batch, stats["batch_count"]))
- except Exception:
- logging.exception("wiki: loading MAP chunks failed for doc %s", doc_id)
- stats["status"] = "error"
-
- async def _map_worker() -> None:
- while True:
- item = await map_queue.get()
- try:
- if item is None:
- return
- i, doc, template_id, parser_cfg, batch, batch_no = item
- doc_id = doc["id"]
- stats = doc_stats[i]
- map_llm_id = pipeline_chat_llm_ids[str(doc_id)]
- phase1 = await wiki_map_from_chunks(
- chunks=batch,
- chat_mdl=map_llm_pool.wrap(
- _bundle_for(map_llm_id),
- priority=30,
- label=f"wiki-map:{doc_id}",
- context=f"{ctx.kb_id}:{doc_id}:map",
- ),
- embd_mdl=embedding_model,
- doc_id=doc_id,
- tenant_id=ctx.tenant_id,
- kb_id=ctx.kb_id,
- language=ctx.language,
- callback=_stage_cb(f"[wiki MAP {i + 1}/{n_docs} b{batch_no}]"),
- parser_config=parser_cfg,
- batch_size_cap=8,
- window_fraction=0.5,
- # Match the shared pool width. The pool is the single
- # admission/concurrency control for actual MAP LLM calls.
- max_workers=WIKI_MAP_LLM_POOL_SIZE,
- )
- for key in stats["agg"]:
- stats["agg"][key] += len(phase1.get(key) or [])
- meta = phase1.get("_meta") or {}
- if isinstance(meta, dict):
- for key in stats["map"]:
- stats["map"][key] += int(meta.get(key, 0) or 0)
- except Exception:
- logging.exception(
- "wiki: MAP failed for doc %s batch %d",
- doc_id,
- batch_no,
- )
- stats["status"] = "error"
- finally:
- map_queue.task_done()
-
- producers = [asyncio.create_task(_produce_document(i, job)) for i, job in enumerate(resolved_eligible)]
- workers = [asyncio.create_task(_map_worker()) for _ in range(WIKI_MAP_LLM_POOL_SIZE)]
- try:
- await asyncio.gather(*producers)
- await map_queue.join()
- finally:
- for task in producers + workers:
- if not task.done():
- task.cancel()
- await asyncio.gather(*producers, *workers, return_exceptions=True)
-
- for i, stats in doc_stats.items():
- doc = stats["doc"]
- doc_id = doc["id"]
- if not stats["saw_any"] and stats["status"] == "ok":
- stats["status"] = "empty"
- logging.info("wiki: no chunks for doc %s; skipping", doc_id)
- agg = stats["agg"]
- map_stats = stats["map"]
- logging.info(
- "wiki: MAP doc=%s entities=%d concepts=%d claims=%d relations=%d (batches=%d, requested=%d cache_hits=%d extracted=%d)",
- doc_id,
- agg["entities"],
- agg["concepts"],
- agg["claims"],
- agg["relations"],
- stats["batch_count"],
- map_stats["requested"],
- map_stats["cache_hits"],
- map_stats["extracted"],
- )
-
- # 5. REDUCE / PLAN / REFINE KB-wide. Each phase has its own
- # input_hash gate (REDUCE keys off the MAP-state hash, PLAN off
- # REDUCE's hash, REFINE off PLAN's hash) so re-runs without an
- # upstream delta short-circuit at the cache layer.
- kb_chat_mdl = _bundle_for(kb_chat_llm_id)
-
- try:
- progress(0.65, "Reducing extracts KB-wide...")
- await wiki_reduce_from_extracts(
- chat_mdl=map_llm_pool.wrap(
- kb_chat_mdl,
- priority=20,
- label="wiki-reduce",
- context=f"{ctx.kb_id}:reduce",
- ),
- embd_mdl=embedding_model,
- tenant_id=ctx.tenant_id,
- kb_id=ctx.kb_id,
- callback=_stage_cb("[wiki REDUCE]"),
- )
-
- progress(0.75, "Planning wiki pages...")
- await wiki_plan_from_reduction(
- chat_mdl=map_llm_pool.wrap(
- kb_chat_mdl,
- priority=20,
- label="wiki-plan",
- context=f"{ctx.kb_id}:plan",
- ),
- embd_mdl=embedding_model,
- tenant_id=ctx.tenant_id,
- kb_id=ctx.kb_id,
- kb_name=kb_name,
- kb_description=kb_description,
- callback=_stage_cb("[wiki PLAN]"),
- )
-
- progress(0.85, "Refining pages...")
- pages = await wiki_refine_from_plan(
- chat_mdl=map_llm_pool.wrap(
- kb_chat_mdl,
- priority=30,
- label="wiki-refine",
- context=f"{ctx.kb_id}:refine",
- ),
- embd_mdl=embedding_model,
- tenant_id=ctx.tenant_id,
- kb_id=ctx.kb_id,
- max_workers=WIKI_REFINE_WORKERS,
- callback=_stage_cb("[wiki REFINE]"),
- instruction=kb_writer_instruction,
- example=kb_writer_example,
- )
- except Exception:
- logging.exception("wiki: REDUCE/PLAN/REFINE failed for kb %s", ctx.kb_id)
- progress(-1, "Wiki pipeline failed during REDUCE/PLAN/REFINE.")
- return
-
- # 6. Persist searchable wiki_page rows.
- try:
- await persist_wiki_pages(ctx=ctx, pages=pages or [], embd_mdl=embedding_model)
- except Exception:
- logging.exception("wiki: persist failed for kb %s", ctx.kb_id)
-
- # 7. Materialize the canvas graph from the refined pages.
- try:
- await persist_wiki_page_graph(ctx=ctx, pages=pages or [])
- except Exception:
- logging.exception("wiki: page-graph persist failed for kb %s", ctx.kb_id)
-
- progress(1.0, f"Wiki compiled {len(pages or [])} page(s).")
-
-
-# ----- dual-mode incremental entry point ---------------------------------
-
-
async def run_wiki_incremental(
ctx: TaskContext,
embedding_model,
@@ -1788,7 +1058,7 @@ async def run_wiki_incremental(
kb_chat_llm_id = None
first_template_found = False
- # 4. MAP per doc (same as run_wiki's MAP phase)
+ # 4. MAP per eligible document.
map_queue: asyncio.Queue = asyncio.Queue(maxsize=WIKI_MAP_QUEUE_SIZE)
n_docs = len(eligible)
diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py
index f7434c9814..1f094978c1 100644
--- a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py
+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py
@@ -173,6 +173,7 @@ _common_mod = sys.modules["rag.advanced_rag.knowlege_compile._common"]
_common_mod.build_chunk_batches = lambda *a, **k: ([], {})
_common_mod.bulk_dedup_items = lambda items, *a, **k: items
_common_mod.ensure_llm_bundle = lambda model: model
+_common_mod.env_int = lambda _name, default, **_kwargs: default
_common_mod.knowledge_compile_gen_conf = lambda *a, **k: {}
_common_mod.run_chunked_pipeline = MagicMock(return_value={})
_common_mod.stable_row_id = lambda *a, **k: ""
diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py b/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py
index 74084d5923..848889aa69 100644
--- a/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py
+++ b/test/unit_test/rag/svr/task_executor_refactor/test_wiki_empty_eligible_message.py
@@ -66,6 +66,7 @@ _STUB_NAMES = (
"rag",
"rag.advanced_rag",
"rag.advanced_rag.knowlege_compile",
+ "rag.advanced_rag.knowlege_compile._common",
"rag.nlp",
"rag.utils",
"rag.utils.es_connection",
@@ -112,6 +113,10 @@ def _load_wiki_generator(monkeypatch):
# Set __path__ on the knowlege_compile stub so that submodules like
# ``structure`` resolve as packages rather than attribute lookups.
sys.modules["rag.advanced_rag.knowlege_compile"].__path__ = [os.path.join(_REPO_ROOT, "rag", "advanced_rag", "knowlege_compile")]
+ _stub_module(
+ "rag.advanced_rag.knowlege_compile._common",
+ env_int=lambda _name, default, **_kwargs: default,
+ )
sys.modules["rag.advanced_rag.knowlege_compile"].structure = ModuleType("rag.advanced_rag.knowlege_compile.structure")
sys.modules["rag.advanced_rag.knowlege_compile.structure"] = sys.modules["rag.advanced_rag.knowlege_compile"].structure
sys.modules["rag.advanced_rag.knowlege_compile.structure"].LLMCallPool = type("LLMCallPool", (), {})
@@ -126,7 +131,6 @@ def _load_wiki_generator(monkeypatch):
_wiki_scan_current_chunk_state=lambda *_args, **_kwargs: [],
wiki_map_from_chunks=lambda *_args, **_kwargs: [],
wiki_plan_from_reduction=lambda *_args, **_kwargs: [],
- wiki_reduce_from_extracts=lambda *_args, **_kwargs: [],
wiki_refine_from_plan=lambda *_args, **_kwargs: [],
)
_stub_module("rag.nlp", search=ModuleType("rag.nlp.search"))