RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat: Compilation benefit naive rag. (#17555)
Kevin Hu3f8a3dfcff76
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex 63e7a3dff7..c639521624 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -637,7 +637,7 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):config = template.get("config") if isinstance(template.get("config"), dict) else {}raw_kind = (config.get("kind") if isinstance(config, dict) else "") or template.get("kind") or ""kind_norm = _compilation_template_kind(raw_kind)- if kind_norm == "artifacts":+ if kind_norm == "wiki":continueseen_configured_ids.add(template_id)configured_ids.append(template_id)diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex 732d3df4f2..eaaf0d44d3 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -792,16 +792,21 @@ def delete_dataset_structure(tenant_id, dataset_id):@login_required@add_tenant_id_to_kwargsasync def get_wiki_alteration(tenant_id, dataset_id):- """Return document drift for the dataset Artifact wiki.+ """Return document drift for a compiled dataset product.- GET /api/v1/datasets/<dataset_id>/artifacts/alteration+ GET /api/v1/datasets/<dataset_id>/artifacts/alteration?kind=<kind>+ ``kind`` (default ``wiki``) is one of: wiki | graph | mindmap | timeline |+ tree (tree covers both ``tree`` and ``page_index``).Success: {"code": 0, "data": {"removed": int, "newly_uploaded": int, ...}}"""+ kind = (request.args.get("kind") or "wiki").strip().lower()+ if kind not in {"wiki", "graph", "mindmap", "timeline", "tree"}:+ return get_error_data_result(message=f"Unsupported kind: {kind!r}. Expected one of: wiki, graph, mindmap, timeline, tree.")try:- success, result = await dataset_api_service.get_wiki_alteration(- dataset_id,- tenant_id,- )+ if kind == "wiki":+ success, result = await dataset_api_service.get_wiki_alteration(dataset_id, tenant_id)+ else:+ success, result = await dataset_api_service.get_structure_alteration(dataset_id, tenant_id, kind)if success:return get_result(data=result)return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)diff --git a/api/apps/restful_apis/utils/compilation_template_validation.py b/api/apps/restful_apis/utils/compilation_template_validation.pyindex 0ee378fcc8..1ac3b6a3c6 100644--- a/api/apps/restful_apis/utils/compilation_template_validation.py+++ b/api/apps/restful_apis/utils/compilation_template_validation.py@@ -56,7 +56,7 @@ def validate_template_payload(req: dict, require_all: bool = True) -> str:return f"{section.capitalize()} field description is too long."if len(str((field or {}).get("rule") or "")) > 1024:return f"{section.capitalize()} field rule is too long."- if config.get("kind") == "artifacts" or req.get("kind") == "artifacts":+ if config.get("kind") == "wiki" or req.get("kind") == "wiki":for field in (config.get("claim") or {}).get("fields") or []:if not str((field or {}).get("statement") or "").strip():return "Claim statement is required."diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 52965bbaf2..264f6dccfb 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -50,13 +50,13 @@ _STRUCTURE_INDEX_TYPE_TO_KIND = {}_STRUCTURE_INDEX_TYPES = frozenset(_STRUCTURE_INDEX_TYPE_TO_KIND)-_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap", "artifact", "skill"} | set(_STRUCTURE_INDEX_TYPES)+_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap", "wiki", "skill"} | set(_STRUCTURE_INDEX_TYPES)_INDEX_TYPE_TO_TASK_TYPE = {"graph": "structure_graph","raptor": "raptor","mindmap": "structure_mindmap",- "artifact": "artifact",+ "wiki": "wiki","skill": "skill",# Structure merge types carry their own task_type (== index_type) so the# executor can resolve which kind to merge from the task body.@@ -67,7 +67,7 @@ _INDEX_TYPE_TO_TASK_ID_FIELD = {"graph": "graphrag_task_id","raptor": "raptor_task_id","mindmap": "mindmap_task_id",- "artifact": "wiki_task_id",+ "wiki": "wiki_task_id","skill": "skill_task_id",**{t: f"{t}_task_id" for t in _STRUCTURE_INDEX_TYPES},}@@ -76,7 +76,7 @@ _INDEX_TYPE_TO_DISPLAY_NAME = {"graph": "Graph","raptor": "RAPTOR","mindmap": "Mindmap",- "artifact": "Artifact",+ "wiki": "Wiki","skill": "Skill","structure_graph": "Structure Graph","structure_mindmap": "Structure Mindmap",@@ -1614,40 +1614,53 @@ def _extract_pipeline_compiler_group_ids(dsl) -> list[str]:return group_ids-def _template_is_wiki(template: dict | None) -> bool:+def _normalize_template_kind(raw) -> str:+ """Lowercase a template's raw kind WITHOUT folding.++ Unlike :func:`_compilation_template_kind` (which folds ``page_index`` /+ ``knowledge_graph`` into ``timeline``), this keeps kinds distinct so the+ alteration API can tell ``graph`` / ``timeline`` / ``tree`` apart.+ """+ if not isinstance(raw, str):+ return ""+ return raw.strip().lower().replace("-", "_")+++def _template_matches_kind(template: dict | None, accepted: set[str]) -> bool:if not isinstance(template, dict):return Falseconfig = template.get("config") if isinstance(template.get("config"), dict) else {}raw_kind = config.get("kind") or template.get("kind") or ""- return _compilation_template_kind(raw_kind) == "artifacts"+ return _normalize_template_kind(raw_kind) in accepted-def _group_has_wiki_template(group_id: str, tenant_id: str, group_cache: dict[str, bool]) -> bool:+def _group_has_template_of_kind(group_id: str, tenant_id: str, accepted: set[str], group_cache: dict[str, bool]) -> bool:if group_id in group_cache:return group_cache[group_id]from api.db.services.compilation_template_group_service import CompilationTemplateGroupServicegroup = CompilationTemplateGroupService.get_saved(group_id, tenant_id)- has_wiki = any(_template_is_wiki(template) for template in (group or {}).get("templates") or [])- group_cache[group_id] = has_wiki- return has_wiki+ matched = any(_template_matches_kind(template, accepted) for template in (group or {}).get("templates") or [])+ group_cache[group_id] = matched+ return matched-def _parser_config_has_wiki_template(parser_config, tenant_id: str, template_cache: dict[str, bool]) -> bool:+def _parser_config_has_template_of_kind(parser_config, tenant_id: str, accepted: set[str], template_cache: dict[str, bool]) -> bool:from api.db.services.compilation_template_service import CompilationTemplateServicefrom rag.svr.task_executor_refactor.chunk_post_processor import _parser_config_compilation_template_idsfor template_id in _parser_config_compilation_template_ids(parser_config, tenant_id):if template_id not in template_cache:- template_cache[template_id] = _template_is_wiki(CompilationTemplateService.get_saved(template_id, tenant_id))+ template_cache[template_id] = _template_matches_kind(CompilationTemplateService.get_saved(template_id, tenant_id), accepted)if template_cache[template_id]:return Truereturn False-def _pipeline_has_wiki_compiler(+def _pipeline_has_compiler_of_kind(pipeline_id: str,tenant_id: str,+ accepted: set[str],pipeline_cache: dict[str, bool],group_cache: dict[str, bool],) -> bool:@@ -1665,9 +1678,9 @@ def _pipeline_has_wiki_compiler(return Falsegroup_ids = _extract_pipeline_compiler_group_ids(getattr(canvas, "dsl", None))- has_wiki = any(_group_has_wiki_template(group_id, tenant_id, group_cache) for group_id in group_ids)- pipeline_cache[pipeline_id] = has_wiki- return has_wiki+ matched = any(_group_has_template_of_kind(group_id, tenant_id, accepted, group_cache) for group_id in group_ids)+ pipeline_cache[pipeline_id] = matched+ return matcheddef _skill_index_or_none(tenant_id: str, kb_id: str):@@ -1841,48 +1854,63 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw# fields only) to enumerate the distinct template ids whose top-level kind# matches the request; ``build_bucket`` below then reads each template's rows.meta_fields = ["id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]- kind_template_ids: list[str] = []- seen_tid: set[str] = set()- has_templateless = False- offset = 0page_size = 1000- pages = 0- while True:- pages += 1- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- meta_fields,- [],- {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": ["dataset"]},- [],- OrderByExpr(),- offset,- page_size,- index_nm,- [dataset_id],- )- meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}- except Exception:- logging.exception("get_dataset_structure: docStore discovery failed for kb=%s", dataset_id)- return True, empty- if not meta_rows:- break- for row in meta_rows.values():- tid = _row_template_id(row)- stamped_kind = (row.get("compilation_template_kind_kwd") or "").strip()- row_kind = stamped_kind or _template_meta(tid) or ""- if _resolve_dataset_structure_kind(row_kind) != resolved_kind:- continue- if tid:- if tid not in seen_tid:- seen_tid.add(tid)- kind_template_ids.append(tid)- else:- has_templateless = True- if len(meta_rows) < page_size:- break- offset += page_size++ async def _discover_scope_templates(scope_kwd: str) -> tuple[list[str], bool, int]:+ scope_template_ids: list[str] = []+ seen_scope_tid: set[str] = set()+ scope_has_templateless = False+ offset = 0+ pages = 0+ while True:+ pages += 1+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ meta_fields,+ [],+ {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": [scope_kwd]},+ [],+ OrderByExpr(),+ offset,+ page_size,+ index_nm,+ [dataset_id],+ )+ meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}+ except Exception:+ logging.exception("get_dataset_structure: docStore discovery failed for kb=%s scope=%s", dataset_id, scope_kwd)+ return [], False, pages+ if not meta_rows:+ break+ for row in meta_rows.values():+ tid = _row_template_id(row)+ stamped_kind = row.get("compilation_template_kind_kwd") or ""+ if isinstance(stamped_kind, list):+ stamped_kind = stamped_kind[0].strip()+ row_kind = stamped_kind or _template_meta(tid) or ""+ if _resolve_dataset_structure_kind(row_kind) != resolved_kind:+ continue+ if tid:+ if tid not in seen_scope_tid:+ seen_scope_tid.add(tid)+ scope_template_ids.append(tid)+ else:+ scope_has_templateless = True+ if len(meta_rows) < page_size:+ break+ offset += page_size+ return scope_template_ids, scope_has_templateless, pages++ dataset_template_ids, has_templateless, dataset_pages = await _discover_scope_templates("dataset")+ kind_template_ids: list[str] = list(dataset_template_ids)+ template_scope_by_id: dict[str, str] = {tid: "dataset" for tid in dataset_template_ids}++ doc_template_ids, _, doc_pages = await _discover_scope_templates("doc")+ for tid in doc_template_ids:+ if tid not in template_scope_by_id:+ template_scope_by_id[tid] = "doc"+ kind_template_ids.append(tid)# Detect datasets that have ONLY the legacy dataset_graph blob (no# entity/relation rows yet) so the fallback path below handles them.@@ -1907,9 +1935,11 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywpasslogging.debug(- "get_dataset_structure: discovered %d template(s) in %d page(s) for kb=%s kind=%s",- len(kind_template_ids),- pages,+ "get_dataset_structure: discovered %d dataset template(s) and %d doc template(s) in %d/%d page(s) for kb=%s kind=%s",+ len(dataset_template_ids),+ len(doc_template_ids),+ dataset_pages,+ doc_pages,dataset_id,resolved_kind,)@@ -1927,17 +1957,20 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywdef _scope_for_template(row: dict):tid = _row_template_id(row) or ""- stamped = (row.get("compilation_template_kind_kwd") or "").strip()+ stamped = row.get("compilation_template_kind_kwd") or ""+ if isinstance(stamped, list):+ stamped = stamped[0].strip()+ scope_kwd = template_scope_by_id.get(tid, "dataset") if tid else "dataset"meta = _bucket_meta_for(tid, stamped) if tid else {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind}if tid:- return meta, {"compilation_template_ids": [tid], "scope_kwd": ["dataset"]}- return meta, {"compilation_template_kind_kwd": [stamped], "scope_kwd": ["dataset"]}+ return meta, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]}+ return meta, {"compilation_template_kind_kwd": [stamped], "scope_kwd": [scope_kwd]}bucket_meta, kw_entities, kw_relations = await sgc.keyword_subgraph(index_nm,dataset_id,embd_mdl,- {"compilation_template_ids": kind_template_ids, "knowledge_graph_kwd": ["entity"], "scope_kwd": ["dataset"]},+ {"compilation_template_ids": kind_template_ids, "knowledge_graph_kwd": ["entity"], "scope_kwd": ["dataset", "doc"]},keywords,_scope_for_template,log_ctx=f"kb={dataset_id}",@@ -1956,11 +1989,18 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw# ── normal mode: per-template subgraph sampling from raw KB-wide rows. ──templates_out: list[dict] = []for tid in kind_template_ids:+ scope_kwd = template_scope_by_id.get(tid, "dataset")try:- entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["dataset"]})+ entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]})except Exception:logging.exception("get_dataset_structure: bucket build failed for kb=%s template=%s", dataset_id, tid)continue+ if scope_kwd == "dataset" and not entities and not relations:+ try:+ entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["doc"]})+ except Exception:+ logging.exception("get_dataset_structure: doc fallback bucket build failed for kb=%s template=%s", dataset_id, tid)+ continueif resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:entities = sgc.filter_entities_with_relations(entities, relations)if not entities:@@ -1993,7 +2033,10 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywlegacy_rows = {}legacy_bucket = {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind, "entities": [], "relations": []}for row in legacy_rows.values():- if _resolve_dataset_structure_kind((row.get("compilation_template_kind_kwd") or "").strip()) != resolved_kind:+ stamped_kind = row.get("compilation_template_kind_kwd") or ""+ if isinstance(stamped_kind, list):+ stamped_kind = stamped_kind[0].strip()+ if _resolve_dataset_structure_kind((stamped_kind or "").strip()) != resolved_kind:continuetry:graph = json.loads(row.get("content_with_weight") or "{}")@@ -2012,14 +2055,39 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywreturn True, {"kind": kind, "templates": templates_out}-async def get_wiki_alteration(dataset_id: str, tenant_id: str):- """Return doc-level drift between current dataset docs and compiled wiki provenance."""- if not KnowledgebaseService.accessible(dataset_id, tenant_id):- return False, "no authorization"- ok, kb = KnowledgebaseService.get_by_id(dataset_id)- if not ok:- return False, "Invalid Dataset ID"+# ── artifacts/alteration: per-``kind`` provenance & eligibility mapping ──+#+# Alteration is a pure set operation shared by every kind:+# removed = involved (product provenance) − current (dataset docs)+# newly_uploaded = eligible (should be compiled) − involved+# Only two inputs vary by kind: how ``involved`` provenance is gathered, and+# which template kinds make a doc ``eligible``.++# Non-folded template kinds that make a doc eligible for each API kind.+_ALTERATION_ELIGIBLE_TEMPLATE_KINDS = {+ "wiki": {"artifacts"},+ "graph": {"knowledge_graph"},+ "mindmap": {"mind_map"},+ "timeline": {"timeline"},+ "tree": {"tree", "page_index"},+}++# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` on+# ``scope_kwd="dataset"`` rows, discovered by their stamped top-level kind.+_ALTERATION_KIND_TO_MERGED_ROW_KIND = {+ "graph": "knowledge_graph",+ "mindmap": "mind_map",+ "timeline": "timeline",+}++# Per-document structure kinds keep one product row per document; provenance is+# the row's own ``doc_id`` (no dataset merge, no ``source_doc_ids``). ``tree`` and+# ``page_index`` write distinct ``compile_kwd`` values.+_ALTERATION_TREE_COMPILE_KWDS = ["tree", "page_index"]++async def _current_dataset_docs(dataset_id: str):+ """Return ``(docs, current_doc_id_set)`` for the dataset."""docs, _ = await thread_pool_exec(DocumentService.get_by_kb_id,kb_id=dataset_id,@@ -2032,81 +2100,147 @@ async def get_wiki_alteration(dataset_id: str, tenant_id: str):types=[],suffix=[],)- current_doc_ids = {str(doc.get("id")) for doc in docs or [] if doc.get("id")}+ docs = docs or []+ current_doc_ids = {str(doc.get("id")) for doc in docs if doc.get("id")}+ return docs, current_doc_ids- wiki_involved_doc_ids: set[str] = set()- pack = _wiki_index_or_none(kb.tenant_id, dataset_id)- if pack is not None:- index_nm, _ = pack- from common.doc_store.doc_store_base import OrderByExpr- select_fields = ["id", "source_doc_ids"]- offset = 0- page_size = 1000- while True:- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields=select_fields,- highlight_fields=[],- condition={"compile_kwd": [WIKI_PAGE_COMPILE_KWD]},- match_expressions=[],- order_by=OrderByExpr(),- offset=offset,- limit=page_size,- index_names=index_nm,- knowledgebase_ids=[dataset_id],- )- rows = settings.docStoreConn.get_fields(res, select_fields) or {}- except Exception:- logging.exception("get_wiki_alteration: docStore search failed for kb=%s", dataset_id)- rows = {}-- if not rows:- break- for row in rows.values():- source_doc_ids = row.get("source_doc_ids")- if isinstance(source_doc_ids, str):- source_doc_ids = [source_doc_ids]- if not isinstance(source_doc_ids, list):- continue- wiki_involved_doc_ids.update(str(doc_id) for doc_id in source_doc_ids if doc_id)+def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc_ids: set) -> dict:+ """Build the drift response dict shared by every ``kind``."""+ removed_doc_ids = sorted(involved_doc_ids - current_doc_ids)+ newly_uploaded_doc_ids = sorted(eligible_doc_ids - involved_doc_ids)+ return {+ "removed": len(removed_doc_ids),+ "newly_uploaded": len(newly_uploaded_doc_ids),+ "removed_doc_ids": removed_doc_ids,+ "newly_uploaded_doc_ids": newly_uploaded_doc_ids,+ "involved_doc_ids": sorted(involved_doc_ids),+ "eligible_doc_ids": sorted(eligible_doc_ids),+ }- offset += page_size- total = settings.docStoreConn.get_total(res)- if not total or offset >= int(total):- break+def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:+ """Doc ids whose parser_config or pipeline carries a template of ``kind``."""+ accepted = _ALTERATION_ELIGIBLE_TEMPLATE_KINDS.get(kind) or set()template_cache: dict[str, bool] = {}group_cache: dict[str, bool] = {}pipeline_cache: dict[str, bool] = {}- eligible_wiki_doc_ids: set[str] = set()+ eligible: set[str] = set()for doc in docs or []:doc_id = str(doc.get("id") or "")if not doc_id:continueparser_config = doc.get("parser_config") or {}- if _parser_config_has_wiki_template(parser_config, kb.tenant_id, template_cache):- eligible_wiki_doc_ids.add(doc_id)+ if _parser_config_has_template_of_kind(parser_config, tenant_id, accepted, template_cache):+ eligible.add(doc_id)continue- if _pipeline_has_wiki_compiler(- doc.get("pipeline_id") or "",- kb.tenant_id,- pipeline_cache,- group_cache,- ):- eligible_wiki_doc_ids.add(doc_id)-- removed_doc_ids = sorted(wiki_involved_doc_ids - current_doc_ids)- newly_uploaded_doc_ids = sorted(eligible_wiki_doc_ids - wiki_involved_doc_ids)- return True, {- "removed": len(removed_doc_ids),- "newly_uploaded": len(newly_uploaded_doc_ids),- "removed_doc_ids": removed_doc_ids,- "newly_uploaded_doc_ids": newly_uploaded_doc_ids,- "involved_doc_ids": sorted(wiki_involved_doc_ids),- "eligible_doc_ids": sorted(eligible_wiki_doc_ids),- }+ if _pipeline_has_compiler_of_kind(doc.get("pipeline_id") or "", tenant_id, accepted, pipeline_cache, group_cache):+ eligible.add(doc_id)+ return eligible+++async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str, from_list: bool) -> set:+ """Page a docStore search, folding each row's ``field`` into a doc-id set.++ ``from_list`` reads ``field`` as ``source_doc_ids`` (a list of provenance doc+ ids); otherwise ``field`` is the row's own ``doc_id`` scalar.+ """+ from common.doc_store.doc_store_base import OrderByExpr++ involved: set[str] = set()+ select_fields = ["id", field]+ offset = 0+ page_size = 1000+ while True:+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields=select_fields,+ highlight_fields=[],+ condition=condition,+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=offset,+ limit=page_size,+ index_names=index_nm,+ knowledgebase_ids=[dataset_id],+ )+ rows = settings.docStoreConn.get_fields(res, select_fields) or {}+ except Exception:+ logging.exception("alteration: docStore search failed for kb=%s cond=%s", dataset_id, condition)+ rows = {}++ if not rows:+ break+ for row in rows.values():+ value = row.get(field)+ if from_list:+ if isinstance(value, str):+ value = [value]+ if isinstance(value, list):+ involved.update(str(d) for d in value if d)+ elif value:+ involved.add(str(value))++ offset += page_size+ total = settings.docStoreConn.get_total(res)+ if not total or offset >= int(total):+ break+ return involved+++async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> set:+ """Gather the doc ids baked into the compiled product for ``kind``."""+ if kind == "wiki":+ return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": [WIKI_PAGE_COMPILE_KWD]}, "source_doc_ids", from_list=True)+ if kind in _ALTERATION_KIND_TO_MERGED_ROW_KIND:+ condition = {+ "knowledge_graph_kwd": ["entity", "relation"],+ "scope_kwd": ["dataset"],+ "compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],+ }+ return await _involved_doc_ids_paged(index_nm, dataset_id, condition, "source_doc_ids", from_list=True)+ if kind == "tree":+ return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": list(_ALTERATION_TREE_COMPILE_KWDS)}, "doc_id", from_list=False)+ return set()+++async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):+ """Shared driver: doc-level drift between the ``kind`` product and the dataset."""+ if not KnowledgebaseService.accessible(dataset_id, tenant_id):+ return False, "no authorization"+ ok, kb = KnowledgebaseService.get_by_id(dataset_id)+ if not ok:+ return False, "Invalid Dataset ID"++ docs, current_doc_ids = await _current_dataset_docs(dataset_id)++ involved_doc_ids: set[str] = set()+ pack = _compiled_index_or_none(kb.tenant_id, dataset_id)+ if pack is not None:+ index_nm, _ = pack+ involved_doc_ids = await _involved_doc_ids_for_kind(index_nm, dataset_id, kind)++ eligible_doc_ids = _eligible_doc_ids_for_kind(docs, kb.tenant_id, kind)+ return True, _alteration_result(current_doc_ids, involved_doc_ids, eligible_doc_ids)+++async def get_wiki_alteration(dataset_id: str, tenant_id: str):+ """Return doc-level drift between current dataset docs and compiled wiki provenance."""+ return await _get_alteration(dataset_id, tenant_id, "wiki")+++async def get_structure_alteration(dataset_id: str, tenant_id: str, kind: str):+ """Return doc-level drift for a non-wiki structure ``kind``.++ ``kind`` is one of ``graph`` / ``mindmap`` / ``timeline`` (dataset-merged+ products, provenance from ``source_doc_ids``) or ``tree`` (per-document+ products covering ``tree`` + ``page_index``, provenance from ``doc_id``).+ """+ kind = (kind or "").strip().lower()+ if kind not in _ALTERATION_KIND_TO_MERGED_ROW_KIND and kind != "tree":+ return False, f"Unsupported structure kind: {kind!r}. Expected one of: graph, mindmap, timeline, tree."+ return await _get_alteration(dataset_id, tenant_id, kind)async def list_wiki_pages(diff --git a/api/db/init_data/compilation_templates/wiki.yaml b/api/db/init_data/compilation_templates/wiki.yamlindex b0484c4b1c..dc2949d060 100644--- a/api/db/init_data/compilation_templates/wiki.yaml+++ b/api/db/init_data/compilation_templates/wiki.yaml@@ -1,7 +1,7 @@-kind: artifacts-display_name: Artifacts — Graph-based wiki+kind: wiki+display_name: Wiki — Graph-based wikiconfig:- kind: artifacts+ kind: wikiexample: |- Each page must be a proper encyclopedic article, NOT a flat bullet list:- 1. Opening paragraph (2-4 sentences defining what this is). No heading.diff --git a/api/db/services/compilation_template_group_service.py b/api/db/services/compilation_template_group_service.pyindex 00bf0e09dc..7ea104c79a 100644--- a/api/db/services/compilation_template_group_service.py+++ b/api/db/services/compilation_template_group_service.py@@ -39,10 +39,10 @@ def _derive_scope(templates: list[dict]) -> str:if not templates:raise GroupValidationError("A template group must contain at least one template.")kinds = [str((t or {}).get("kind") or "").strip() for t in templates]- artifact_count = sum(1 for k in kinds if k == "artifacts")+ artifact_count = sum(1 for k in kinds if k == "wiki")if artifact_count > 0:if artifact_count != 1 or len(templates) != 1:- raise GroupValidationError("An artifacts template cannot be combined with other templates in the same group.")+ raise GroupValidationError("A wiki template cannot be combined with other templates in the same group.")return SCOPE_DATASET_enforce_single_rechunk_tree(templates)diff --git a/api/db/services/document_service.py b/api/db/services/document_service.pyindex 48167aff6c..b639bb291c 100644--- a/api/db/services/document_service.py+++ b/api/db/services/document_service.py@@ -1225,7 +1225,7 @@ def queue_raptor_o_graphrag_tasks(sample_doc, ty, priority, fake_doc_id="", doc_"graphrag","raptor","mindmap",- "artifact",+ "wiki","skill",# KB-wide structure-graph merge task types (rebuild dataset_graph rows)."structure_graph",diff --git a/common/constants.py b/common/constants.pyindex 139612e42d..51fb3c7e17 100644--- a/common/constants.py+++ b/common/constants.py@@ -185,7 +185,9 @@ class PipelineTaskType(StrEnum):GRAPH_RAG = "GraphRAG"MINDMAP = "Mindmap"MEMORY = "Memory"- ARTIFACT = "Artifact"+ # Member name kept as ARTIFACT for back-compat; value is "Wiki" so the+ # runtime task_type (``.lower()`` == "wiki") matches the wiki index/task type.+ ARTIFACT = "Wiki"SKILL = "Skill"# KB-wide structure-graph merge tasks (rebuild_dataset_structure_graph_json).STRUCTURE_GRAPH = "StructureGraph"diff --git a/rag/advanced_rag/harness/orchestrator/agentic.py b/rag/advanced_rag/harness/orchestrator/agentic.pyindex 19bb9424f0..212cbc110c 100644--- a/rag/advanced_rag/harness/orchestrator/agentic.py+++ b/rag/advanced_rag/harness/orchestrator/agentic.py@@ -310,7 +310,7 @@ async def _add_template_group_compilations(comps: set[str], parser_config: dict,comps.add("page_index")elif kind in {"mindmap", "mind_map"}:comps.add("mindmap")- elif kind == "artifacts":+ elif kind == "wiki":comps.add("wiki")diff --git a/rag/advanced_rag/harness/orchestrator/direct.py b/rag/advanced_rag/harness/orchestrator/direct.pyindex e7cdc32704..d7c1fc43b3 100644--- a/rag/advanced_rag/harness/orchestrator/direct.py+++ b/rag/advanced_rag/harness/orchestrator/direct.py@@ -11,9 +11,9 @@ async def direct_search(state: dict, tools) -> dict:"""Single hybrid search → merge into kbinfos."""question = state.get("question", "")keywords = state.get("keywords", "")- _LOG.info("[Direct search] Looking up the knowledge base for: \"%s\" (keywords: %s)", question, keywords)+ _LOG.info('[Direct search] Looking up the knowledge base for: "%s" (keywords: %s)', question, keywords)- result = await hybrid_search(tools, query=question, keywords=keywords)+ result = await hybrid_search(tools, query=question, keywords=keywords, use_compiled=True)_merge_kbinfos(tools, result)if not _has_chunks(tools):diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex 4d52cc9e8d..f3127452f5 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -543,13 +543,22 @@ async def dataset_navigation_by_tree(tools, topic: str, keywords: str = "", doc_# its way to a small subgraph: seed entities by the question, hop out over# relations to the 2nd-degree neighbours, then answer from that subgraph.-_KG_SEEDS = 3 # entities matched directly to the question-_KG_HOPS = 1 # relation hops out from the seeds (1 => "2nd degree")-_KG_NEIGHBORS = 10 # neighbour entity rows resolved per hop+# Scope of the compiled KG rows we search. Without a doc_scope we read the+# dataset-merged rows (one graph per dataset); with a doc_scope we read the+# per-document rows and filter by doc_id. Kept local rather than imported from+# the task-executor layer so the harness has no dependency on it.+_SCOPE_KWD_DATASET = "dataset"+_SCOPE_KWD_DOC = "doc"++_KG_SEEDS = 2 # top-N entities matched directly to the question+_KG_SEED_POOL = 64 # KNN candidate pool before the mention_count_int re-sort+_KG_SEED_SIM = 0.8 # dense-similarity floor for seed entities+_KG_HOPS = 2 # relation hops out from the seeds (1 => "2nd degree")+_KG_NEIGHBORS = 128 # cap on neighbour entity rows resolved per hop_KG_REL_LIMIT = 32 # relations fetched per endpoint filter-async def _kg_scopes(tools, doc_scope: list[str] | None):+async def _kg_scopes(tools, doc_scope: list[str] | None = None):"""Resolve the (kb_id, tenant_id, doc_ids|None) groups to search.With a ``doc_scope`` the graph is limited to those docs (grouped by their@@ -567,29 +576,59 @@ async def _kg_scopes(tools, doc_scope: list[str] | None):return [(kb.id, kb.tenant_id, None) for kb in getattr(tools, "kbs", []) or []]-async def _kg_search(tools, kb_id: str, tenant_id: str, doc_ids, kind: str, text: str = "", top_n: int = 8, extra: dict | None = None) -> list[dict]:- """Search the compiled KG rows of one KB and return the raw field maps."""+async def _kg_search(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids,+ kind: str,+ text: str = "",+ top_n: int = 8,+ extra: dict | None = None,+ scope_kwd: str | None = None,+ order_desc: str | None = None,+ pool: int | None = None,+ similarity: float = 0.6,+) -> list[dict]:+ """Search the compiled KG rows of one KB and return the raw field maps.++ ``scope_kwd`` narrows to dataset-merged (``"dataset"``) or per-doc (``"doc"``)+ rows. ``order_desc`` sorts the hits by that field descending; when combined+ with a dense ``text`` match, ``pool`` sets the KNN candidate count so the+ re-sort ranks a wider pool than the ``top_n`` finally kept. ``similarity`` is+ the dense-match floor.+ """from common import settingsfrom common.doc_store.doc_store_base import MatchTextExpr, OrderByExprfrom common.misc_utils import thread_pool_execfrom rag.nlp import searchcondition: dict = {"knowledge_graph_kwd": [kind]}+ if scope_kwd:+ condition["scope_kwd"] = [scope_kwd]if doc_ids:condition["doc_id"] = list(doc_ids)if extra:condition.update(extra)- fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "from_entity_kwd", "to_entity_kwd"]+ fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "name_kwd", "mention_count_int", "from_entity_kwd", "to_entity_kwd"]exprs = []if text:+ knn_topn = pool or top_nif getattr(tools, "embed_mdl", None):try:- exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, top_n, 0.1))+ exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, knn_topn, similarity))except Exception:_LOG.exception("[Graph exploration] vector build failed; using keyword match")if not exprs:- exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, top_n))+ exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, knn_topn))++ order_by = OrderByExpr()+ if order_desc:+ try:+ order_by.desc(order_desc)+ except Exception:+ order_by = OrderByExpr()try:res = await thread_pool_exec(@@ -598,7 +637,7 @@ async def _kg_search(tools, kb_id: str, tenant_id: str, doc_ids, kind: str, text[],condition,exprs,- OrderByExpr(),+ order_by,0,top_n,search.index_name(tenant_id),@@ -651,6 +690,25 @@ def _kg_parse_relation(row: dict) -> dict | None:}+def _endpoint_terms(names) -> list[str]:+ """Case variants for matching relation endpoints.++ ``dataset_structure_merger`` lowercases ``from_entity_kwd``/``to_entity_kwd``+ on merged rows while entity names keep their original case, so hop queries+ must try both forms. Accepts a single name or an iterable and returns the+ sorted union of each name's original and lowercased form.+ """+ if isinstance(names, str):+ names = [names]+ terms: set[str] = set()+ for n in names or []:+ n = (n or "").strip()+ if n:+ terms.add(n)+ terms.add(n.lower())+ return sorted(terms)++def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_names: list[str]) -> dict:"""Group the source_chunk_ids of the relevant entities AND relations by doc.@@ -684,23 +742,31 @@ def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[str] | None = None) -> dict:"""Explore the compiled knowledge graph to answer ``query``.- Searches seed entities for the question, hops out over their relations to the- 2nd-degree neighbours, asks the chat model to answer from that subgraph, then- pulls the source passages behind the entities/relations it found relevant and- narrows them with ``keywords``. Falls back to a plain hybrid search when the- KB has no compiled graph or no source text is behind the relevant nodes.+ Seeds the top-``_KG_SEEDS`` entities for the question (dense match above+ ``_KG_SEED_SIM``, ranked by ``mention_count_int``), hops ``_KG_HOPS`` out over+ their relations, then asks the chat model whether that subgraph answers the+ question. When it does, the answer is returned directly; when it doesn't, the+ source passages behind the entities/relations the model found relevant are+ returned as evidence (narrowed by ``keywords``) so the caller can continue.- :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}``+ Without ``doc_scope`` the dataset-merged graph is searched+ (``scope_kwd="dataset"``); with it, the per-document rows of those docs+ (``scope_kwd="doc"``, filtered by ``doc_id``).++ :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}`` — exactly one+ of ``answer`` / ``chunks`` is populated."""- from rag.advanced_rag.harness.tools.search import _narrow_by_keywords, hybrid_search+ from rag.advanced_rag.harness.tools.search import _narrow_by_keywords+ _empty = {"answer": "", "chunks": [], "doc_aggs": []}_LOG.info(f'[Graph exploration] Exploring the knowledge graph for "{query}" (keywords: {keywords})')scopes = await _kg_scopes(tools, doc_scope)if not scopes:- _LOG.info("[Graph exploration] No knowledge base in scope — falling back to a normal search.")- return await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)+ _LOG.info("[Graph exploration] No knowledge base in scope.")+ return _empty+ scope_kwd = _SCOPE_KWD_DOC if doc_scope else _SCOPE_KWD_DATASETtext = f"{query} {keywords}".strip()entities: list[dict] = []relations: list[dict] = []@@ -718,52 +784,79 @@ async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[sreturn addedfor kb_id, tenant_id, doc_ids in scopes:- seed_rows = await _kg_search(tools, kb_id, tenant_id, doc_ids, "entity", text=text, top_n=_KG_SEEDS)+ # (1) Seeds: condition C — dense match (>= _KG_SEED_SIM) over the scoped+ # entity rows, ranked by mention_count_int desc, top _KG_SEEDS.+ seed_rows = await _kg_search(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "entity",+ text=text,+ top_n=_KG_SEEDS,+ scope_kwd=scope_kwd,+ order_desc="mention_count_int",+ pool=_KG_SEED_POOL,+ similarity=_KG_SEED_SIM,+ )seeds = [e for e in (_kg_parse_entity(r) for r in seed_rows.values()) if e]frontier = _add_entities(seeds, kb_id)_LOG.info("[Graph exploration] Seeded %d entity(ies): %s", len(frontier), ", ".join(frontier) or "none")+ # (2) Expand _KG_HOPS out, collecting relations and neighbour entities.for _hop in range(_KG_HOPS):if not frontier:break- # Relations touching the current frontier (as source OR target).+ terms = _endpoint_terms(frontier) # case variants — merged rows lowercase endpointsrel_rows: dict = {}- rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, extra={"from_entity_kwd": frontier}))- rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, extra={"to_entity_kwd": frontier}))+ rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"from_entity_kwd": terms}))+ rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"to_entity_kwd": terms}))hop_relations = [r for r in (_kg_parse_relation(x) for x in rel_rows.values()) if r]relations.extend(hop_relations)- neighbour_names = {n for r in hop_relations for n in (r["from"], r["to"]) if n.lower() not in ent_names}- if not neighbour_names:+ seen_lower = {k.split(":", 1)[1] for k in ent_names if k.startswith(f"{kb_id}:")}+ neigh_names = {n.strip() for r in hop_relations for n in (r["from"], r["to"]) if n and n.strip()}+ neigh_lower_set = {n.lower() for n in neigh_names} - seen_lower+ if not neigh_lower_set:break- neigh_rows = await _kg_search(tools, kb_id, tenant_id, doc_ids, "entity", text=" ".join(neighbour_names), top_n=_KG_NEIGHBORS)- wanted = {n.lower() for n in neighbour_names}- neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e and e["name"].lower() in wanted]+ neigh_filtered = {n for n in neigh_names if n.lower() in neigh_lower_set}+ neigh_rows = await _kg_search(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "entity",+ top_n=min(max(len(neigh_filtered), 1), _KG_NEIGHBORS),+ scope_kwd=scope_kwd,+ extra={"name_kwd": _endpoint_terms(neigh_filtered)},+ )+ neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e]frontier = _add_entities(neighbours, kb_id)_LOG.info("[Graph exploration] Hop %d reached %d neighbour entity(ies).", _hop + 1, len(frontier))if not entities and not relations:- _LOG.info("[Graph exploration] No compiled knowledge graph here — falling back to a normal search.")- return await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)+ _LOG.info("[Graph exploration] No compiled knowledge graph in scope.")+ return _empty_LOG.info("[Graph exploration] Built a subgraph of %d entity(ies) and %d relation(s).", len(entities), len(relations))+ # (3) Does the subgraph answer the question?answer, relevant = await _ask_structure(tools, query, entities, relations, "knowledge graph", "Graph exploration")+ # (4a) Sufficient — return the answer, no chunks.+ if answer:+ _LOG.info("[Graph exploration] The subgraph answered the question directly.")+ return {"answer": answer, "chunks": [], "doc_aggs": []}++ # (4b) Insufficient — return the source passages behind the relevant nodes.evidence = _collect_evidence_ids(entities, relations, relevant)chunks: list[dict] = []for doc_id, ids in evidence.items():if doc_id and ids:chunks.extend(await _load_chunks_by_ids(tools, doc_id, ids))- if not chunks:- _LOG.info("[Graph exploration] No source text behind those nodes — falling back to a normal search.")- fallback = await hybrid_search(tools, query=query, keywords=keywords, doc_scope=doc_scope)- fallback["answer"] = answer- return fallback-before = len(chunks)chunks = _narrow_by_keywords(chunks, keywords)- _LOG.info("[Graph exploration] Pulled %d source passage(s) behind those nodes, kept %d after keyword filtering.", before, len(chunks))+ _LOG.info("[Graph exploration] Insufficient; returning %d evidence passage(s) (%d before keyword filtering).", len(chunks), before)- return {"answer": answer, "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}+ return {"answer": "", "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex 351cbba44c..822c7812b1 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -4,6 +4,7 @@ import loggingimport reimport hashlibfrom common import settings+from .navigation import _kg_scopes_LOG = logging.getLogger(__name__)@@ -179,7 +180,7 @@ def _normalize(kbinfos: dict, tenant_ids: list[str] | str | None) -> dict:return kbinfos-async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "") -> dict:+async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "", use_compiled: bool = False) -> dict:if not tools.kb_ids and not kb_ids:return {"chunks": [], "doc_aggs": []}target_ids = kb_ids or tools.kb_ids@@ -220,6 +221,9 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_length = len(kbinfos["chunks"])kbinfos["chunks"] = _narrow_by_keywords(kbinfos.get("chunks", []), keywords)_LOG.info(f"[Hybrid search] Kept {len(kbinfos['chunks'])} of {length} passage(s) that actually mention the keywords.")+ if use_compiled and kbinfos.get("chunks"):+ _LOG.info("[Hybrid search] Compiled expansion enabled — enriching with page_index/tree/KG navigation.")+ await _expand_with_compiled(tools, query, keywords, kbinfos)if cache is not None:cache[cache_key] = kbinfosreturn kbinfos@@ -277,6 +281,473 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:return kbinfos+# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───+++async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict) -> None:+ """Zero-LLM compiled-product expansion: page_index → tree → KG.++ For each bound KB, searches compiled entity rows matching the query,+ hops 1-hop via relations to find neighbour entities, then appends+ their source passages to ``kbinfos["chunks"]``.+ """+ before = len(kbinfos.get("chunks", []))+ seen_ids = {c.get("chunk_id") or c.get("id") for c in kbinfos.get("chunks", [])}++ scopes = await _kg_scopes(tools)+ if not scopes:+ return++ for kb_id, tenant_id, doc_ids in scopes:+ # 1-hop entity-graph expansion per template kind.+ # Each template writes entity/relation rows tagged with+ # ``compilation_template_kind_kwd`` — search them independently.+ for label, template_kind in (+ ("knowledge_graph", "knowledge_graph"),+ ("mind_map", "mind_map"),+ ("timeline", "timeline"),+ ("page_index", "page_index"),+ ):+ chunks = await _expand_compiled_strategy(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ query,+ seen_ids,+ template_kind=template_kind,+ max_chunks=5,+ )+ if chunks:+ kbinfos.setdefault("chunks", []).extend(chunks)+ _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))++ # Tree structure graph (uses ``compile_kwd``, not template kind).+ chunks = await _expand_compiled_strategy(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ query,+ seen_ids,+ compile_kwd="tree",+ max_chunks=5,+ )+ if chunks:+ kbinfos.setdefault("chunks", []).extend(chunks)+ _LOG.debug("[Compiled expand] tree: +%d chunks", len(chunks))++ # Synthesis pages — standalone rendered articles from wiki / session+ # graph / session essence templates. Searched directly (no entity-graph nav).+ for label, ckwd in (+ ("wiki_page", "wiki_page"),+ ("artifact_page", "artifact_page"),+ ("essence", "essence"),+ ):+ chunks = await _expand_wiki_page_strategy(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ query,+ seen_ids,+ compile_kwd=ckwd,+ max_chunks=5,+ )+ if chunks:+ kbinfos.setdefault("chunks", []).extend(chunks)+ _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))++ # Re-sort so compiled-expansion chunks blend by similarity with regular ones.+ chunks = kbinfos.get("chunks", [])+ if chunks:+ chunks.sort(key=lambda c: c.get("similarity", 0.0), reverse=True)++ after = len(chunks)+ _LOG.info("[Hybrid search] Compiled expansion added %d chunks.", after - before)+++async def _search_compiled_rows(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids: list[str] | None,+ kind: str,+ *,+ text: str = "",+ top_n: int = 8,+ extra: dict | None = None,+ compile_kwd: str | None = None,+ template_kind: str | None = None,+) -> dict:+ """Search compiled KG rows in one KB, returning raw field maps.++ *compile_kwd* filters by the ``compile_kwd`` field (e.g. "tree" for tree+ structure nodes). *template_kind* filters by ``compilation_template_kind_kwd``+ (e.g. "knowledge_graph", "mind_map"). Leave both ``None`` to scan all rows.+ """+ from common import settings+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ condition: dict = {"knowledge_graph_kwd": [kind]}+ if compile_kwd:+ condition["compile_kwd"] = compile_kwd+ if template_kind:+ condition["compilation_template_kind_kwd"] = template_kind+ if doc_ids:+ condition["doc_id"] = list(doc_ids)+ if extra:+ condition.update(extra)++ fields = [+ "content_with_weight",+ "source_chunk_ids",+ "doc_id",+ "docnm_kwd",+ "from_entity_kwd",+ "to_entity_kwd",+ "name_kwd",+ ]+ exprs = []+ if text:+ embd_mdl = getattr(tools, "embed_mdl", None)+ if embd_mdl:+ try:+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))+ except Exception:+ _LOG.exception("[Compiled expand] vector build failed; using keyword match")+ if not exprs:+ exprs.append(+ MatchTextExpr(+ ["content_ltks", "content_sm_ltks"],+ text,+ top_n,+ )+ )++ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ exprs,+ OrderByExpr(),+ 0,+ top_n,+ search.index_name(tenant_id),+ [kb_id],+ )+ return settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ _LOG.exception("[Compiled expand] search failed (kind=%s compile_kwd=%s)", kind, compile_kwd)+ return {}+++async def _load_chunks_for_doc(tools, doc_id: str, chunk_ids: list[str]) -> list[dict]:+ """Load chunks by their IDs from the doc store."""+ if not chunk_ids:+ return []+ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)+ if not resolved:+ return []+ kb_id, tenant_id = resolved++ fields = ["content_with_weight", "docnm_kwd", "doc_id", "id"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"id": list(chunk_ids)},+ [],+ OrderByExpr(),+ 0,+ len(chunk_ids),+ search.index_name(tenant_id),+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields)+ if not rows:+ return []+ return [{**v, "chunk_id": k} for k, v in rows.items()]+ except Exception:+ _LOG.exception("[Compiled expand] failed to load chunks for doc_id=%s", doc_id)+ return []+++async def _expand_compiled_strategy(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids: list[str] | None,+ query: str,+ seen_ids: set[str],+ *,+ compile_kwd: str | None = None,+ template_kind: str | None = None,+ max_chunks: int = 5,+) -> list[dict]:+ """Generic 1-hop compiled expansion: entity search → relation nav → chunk load.++ 1. Embedding-match seed entities (filtered by *compile_kwd* or *template_kind*).+ 2. Fetch relations adjacent to seed entities (forward + backward).+ 3. Collect neighbour entity names (1-hop away).+ 4. Look up neighbour entities to get ``source_chunk_ids``.+ 5. Load actual chunks, deduplicate, respect *max_chunks*.++ *compile_kwd* is used for structure graphs (e.g. "tree").+ *template_kind* is used for entity extraction rows (e.g. "knowledge_graph").+ """+ import json++ # -- 1. Seed entities --+ seed_rows = await _search_compiled_rows(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "entity",+ text=query,+ top_n=5,+ compile_kwd=compile_kwd,+ template_kind=template_kind,+ )+ if not seed_rows:+ return []++ seed_names: set[str] = set()+ for r in seed_rows.values():+ try:+ payload = json.loads(r.get("content_with_weight") or "{}")+ except Exception:+ continue+ name = (payload.get("name") or payload.get("title") or "").strip()+ if name:+ seed_names.add(name)+ if not seed_names:+ return []++ # -- 2. Adjacent relations (outgoing + incoming) --+ # Provide both original and lowercased names — dataset_structure_merger+ # lowercases merged-row endpoints while per-doc rows keep original case.+ seed_list = sorted({n.lower() for n in seed_names} | seed_names)+ fwd = await _search_compiled_rows(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "relation",+ top_n=50,+ compile_kwd=compile_kwd,+ template_kind=template_kind,+ extra={"from_entity_kwd": seed_list},+ )+ bwd = await _search_compiled_rows(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "relation",+ top_n=50,+ compile_kwd=compile_kwd,+ template_kind=template_kind,+ extra={"to_entity_kwd": seed_list},+ )+ all_rels = {**fwd, **bwd}++ # -- 3. Neighbour names (1-hop, exclude seeds) --+ seed_lower = {n.lower() for n in seed_names}+ neighbour_names: set[str] = set()+ for r in all_rels.values():+ frm = (r.get("from_entity_kwd") or "").strip()+ frm_lower = frm.lower()+ to = (r.get("to_entity_kwd") or "").strip()+ to_lower = to.lower()+ if frm_lower in seed_lower and to and to_lower not in seed_lower:+ neighbour_names.add(to)+ if to_lower in seed_lower and frm and frm_lower not in seed_lower:+ neighbour_names.add(frm)+ if not neighbour_names:+ return []++ # -- 4. Neighbour entity source_chunk_ids --+ # Provide both original and lowercased — same as seed_list above.+ neigh_list = sorted({n.lower() for n in neighbour_names} | neighbour_names)+ if len(neigh_list) > 100:+ neigh_list = neigh_list[:100] # reasonable cap for name_kwd search+ neigh_rows = await _search_compiled_rows(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ "entity",+ top_n=len(neigh_list),+ compile_kwd=compile_kwd,+ template_kind=template_kind,+ extra={"name_kwd": neigh_list},+ )++ # Group chunk IDs by doc+ by_doc: dict[str, set[str]] = {}+ for r in neigh_rows.values():+ doc_id = r.get("doc_id") or ""+ for cid in r.get("source_chunk_ids") or []:+ if cid and cid not in seen_ids:+ by_doc.setdefault(doc_id, set()).add(cid)++ # -- 5. Load and return --+ new_chunks: list[dict] = []+ for doc_id, cids in by_doc.items():+ if len(new_chunks) >= max_chunks:+ break+ limit = max_chunks - len(new_chunks)+ chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])+ for c in chunks:+ cid = c.get("chunk_id") or c.get("id")+ if cid and cid not in seen_ids:+ seen_ids.add(cid)+ new_chunks.append(c)++ return new_chunks+++async def _search_synthesis_pages(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids: list[str] | None,+ text: str,+ *,+ compile_kwd: str = "wiki_page",+ top_n: int = 8,+) -> dict:+ """Search synthesis-compiled page rows (no knowledge_graph_kwd filter).++ Synthesis pages are standalone articles (wiki_page, artifact_page,+ essence, etc.) with ``content_with_weight``, keyword index, and+ vector. They do NOT carry the ``knowledge_graph_kwd`` field (unlike+ entity/relation rows from extraction).+ """+ from common import settings+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ condition: dict = {"compile_kwd": compile_kwd, "available_int": 1}+ if doc_ids:+ condition["source_doc_ids"] = list(doc_ids)++ fields = [+ "content_with_weight",+ "summary_with_weight",+ "source_chunk_ids",+ "doc_id",+ "title_kwd",+ "topic_kwd",+ ]++ exprs = []+ if text:+ embd_mdl = getattr(tools, "embed_mdl", None)+ if embd_mdl:+ try:+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))+ except Exception:+ _LOG.exception("[Wiki expand] vector build failed; using keyword match")+ if not exprs:+ exprs.append(+ MatchTextExpr(+ ["content_ltks", "content_sm_ltks"],+ text,+ top_n,+ )+ )++ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ exprs,+ OrderByExpr(),+ 0,+ top_n,+ search.index_name(tenant_id),+ [kb_id],+ )+ return settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ _LOG.exception("[Wiki expand] search failed for kb=%s", kb_id)+ return {}+++async def _expand_wiki_page_strategy(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids: list[str] | None,+ query: str,+ seen_ids: set[str],+ *,+ compile_kwd: str = "wiki_page",+ max_chunks: int = 5,+) -> list[dict]:+ """Expand synthesis-compiled pages: semantic search → load source chunks.++ Unlike ``_expand_compiled_strategy`` (which does 1-hop entity-graph+ navigation), synthesis pages are standalone rendered articles — we search+ them directly and load the referenced source chunks as context.++ *compile_kwd* selects which synthesis type: "wiki_page" (Wiki template),+ "artifact_page" (Session Graph synthesis), "essence" (Session Essence).+ """+ # -- 1. Search synthesis pages --+ wiki_rows = await _search_synthesis_pages(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ query,+ compile_kwd=compile_kwd,+ top_n=5,+ )+ if not wiki_rows:+ return []++ # -- 2. Collect source_chunk_ids from matching pages --+ by_doc: dict[str, set[str]] = {}+ for r in wiki_rows.values():+ doc_id = r.get("doc_id") or ""+ for cid in r.get("source_chunk_ids") or []:+ if cid and cid not in seen_ids:+ by_doc.setdefault(doc_id, set()).add(cid)++ # -- 3. Load chunks, assign high similarity for priority ranking --+ new_chunks: list[dict] = []+ for doc_id, cids in by_doc.items():+ if len(new_chunks) >= max_chunks:+ break+ limit = max_chunks - len(new_chunks)+ chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])+ for c in chunks:+ cid = c.get("chunk_id") or c.get("id")+ if cid and cid not in seen_ids:+ seen_ids.add(cid)+ c.setdefault("similarity", 0.9) # wiki pages rank high+ new_chunks.append(c)++ return new_chunks++async def web_search(tools, query: str, keywords: str = "") -> dict:if not tools.has_web():return {"chunks": [], "doc_aggs": []}diff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.pyindex d87dca810d..61d37cf6de 100644--- a/rag/advanced_rag/knowlege_compile/runner.py+++ b/rag/advanced_rag/knowlege_compile/runner.py@@ -135,7 +135,7 @@ def load_active_templates(template_ids, tenant_id: str) -> list[tuple[str, dict]logging.warning("document_structure_compile: template %s config is invalid", template_id)continuekind = _compilation_template_kind(parser_cfg.get("kind"))- if not kind or kind == "artifacts":+ if not kind or kind == "wiki":continueactive_templates.append((template_id, parser_cfg))return active_templatesdiff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.pyindex 626070375c..4f3ac2361c 100644--- a/rag/svr/task_executor.py+++ b/rag/svr/task_executor.py@@ -136,7 +136,7 @@ TASK_TYPE_TO_PIPELINE_TASK_TYPE = {"graphrag": PipelineTaskType.GRAPH_RAG,"mindmap": PipelineTaskType.MINDMAP,"memory": PipelineTaskType.MEMORY,- "artifact": PipelineTaskType.ARTIFACT,+ "wiki": PipelineTaskType.ARTIFACT,"skill": PipelineTaskType.SKILL,"structure_graph": PipelineTaskType.STRUCTURE_GRAPH,"structure_mindmap": PipelineTaskType.STRUCTURE_MINDMAP,@@ -152,7 +152,7 @@ _KB_FANOUT_TASK_TYPES = ["graphrag","raptor","mindmap",- "artifact",+ "wiki","skill","structure_graph","structure_mindmap",diff --git a/rag/svr/task_executor_refactor/dataset_wiki_generator.py b/rag/svr/task_executor_refactor/dataset_wiki_generator.pyindex 8b6e8c3ed3..b81919d0cc 100644--- a/rag/svr/task_executor_refactor/dataset_wiki_generator.py+++ b/rag/svr/task_executor_refactor/dataset_wiki_generator.py@@ -1028,7 +1028,7 @@ async def run_wiki(template = CompilationTemplateService.get_saved(template_id, ctx.tenant_id)config = template.get("config") if template else {}kind = _compilation_template_kind(config.get("kind") if isinstance(config, dict) else "")- if kind == "artifacts":+ if kind == "wiki":eligible.append((d, template_id))breakif not eligible:diff --git a/rag/svr/task_executor_refactor/task_handler.py b/rag/svr/task_executor_refactor/task_handler.pyindex 5bb0cb0cf4..250523b5b1 100644--- a/rag/svr/task_executor_refactor/task_handler.py+++ b/rag/svr/task_executor_refactor/task_handler.py@@ -255,7 +255,7 @@ class TaskHandler:await self._run_graphrag(embedding_model)elif task_type == "mindmap":ctx.progress_cb(1, "place holder")- elif task_type == "artifact":+ elif task_type == "wiki":from rag.svr.task_executor_refactor.dataset_wiki_generator import (run_wiki,)