RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: handle disabled documents in dataset structures (#18033)
buua436c0582b8e1834
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex ffee50c209..a465bf2993 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -949,15 +949,19 @@ def delete_index(dataset_id: str, tenant_id: str, index_type: str, wipe: bool =elif wipe and index_type in _STRUCTURE_INDEX_TYPES:from rag.nlp import search- # Wipe the merged KB-wide dataset_graph rows for the requested kind- # (all kinds for the merge-all "structure" type). The per-document- # entity/relation rows the merge reads from are left intact.+ # Wipe the merged KB-wide metadata and entity/relation rows for the+ # requested kind (all kinds for the merge-all "structure" type). The+ # per-document entity/relation rows the merge reads from are left intact.friendly = _STRUCTURE_INDEX_TYPE_TO_KIND.get(index_type)resolved_kind = _resolve_dataset_structure_kind(friendly) if friendly else None- condition: dict = {"knowledge_graph_kwd": ["dataset_graph"]}- if resolved_kind:- condition["compilation_template_kind_kwd"] = [resolved_kind]- settings.docStoreConn.delete(condition, search.index_name(kb.tenant_id), dataset_id)+ conditions = [+ {"knowledge_graph_kwd": ["dataset_graph"]},+ {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": ["dataset"]},+ ]+ for condition in conditions:+ if resolved_kind:+ condition["compilation_template_kind_kwd"] = [resolved_kind]+ settings.docStoreConn.delete(condition, search.index_name(kb.tenant_id), dataset_id)KnowledgebaseService.update_by_id(kb.id, {task_id_field: "", task_finish_at_field: None})return True, {}@@ -1841,6 +1845,14 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywfrom api.apps.services import structure_graph_common as sgckeywords = (keywords or "").strip()+ _, active_doc_ids = await _current_dataset_docs(dataset_id)+ disabled_doc_ids = await _disabled_dataset_doc_ids(dataset_id)+ if not active_doc_ids:+ return True, empty+ # Dataset rows use the KB id as ``doc_id``. If a historical row has no+ # source provenance, exclude it while documents are disabled and fall back+ # to active document rows instead of returning potentially stale content.+ dataset_excluded_doc_ids = disabled_doc_ids | ({dataset_id} if disabled_doc_ids else set())def _row_template_id(row: dict) -> str | None:raw = row.get("compilation_template_ids")@@ -2011,6 +2023,7 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywkeywords,_scope_for_template,log_ctx=f"kb={dataset_id}",+ excluded_doc_ids=dataset_excluded_doc_ids,)if resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:kw_entities = sgc.filter_entities_with_relations(kw_entities, kw_relations)@@ -2028,13 +2041,26 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywfor tid in kind_template_ids:scope_kwd = template_scope_by_id.get(tid, "dataset")try:- entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]})+ scope = {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]}+ if scope_kwd == "doc":+ scope["doc_id"] = sorted(active_doc_ids)+ entities, relations = await sgc.build_bucket(+ index_nm,+ dataset_id,+ scope,+ excluded_doc_ids=dataset_excluded_doc_ids if scope_kwd == "dataset" else disabled_doc_ids,+ )except Exception:logging.exception("get_dataset_structure: bucket build failed for kb=%s template=%s", dataset_id, tid)continueif scope_kwd == "dataset" and not entities and not relations:try:- entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["doc"]})+ entities, relations = await sgc.build_bucket(+ index_nm,+ dataset_id,+ {"compilation_template_ids": [tid], "scope_kwd": ["doc"], "doc_id": sorted(active_doc_ids)},+ excluded_doc_ids=disabled_doc_ids,+ )except Exception:logging.exception("get_dataset_structure: doc fallback bucket build failed for kb=%s template=%s", dataset_id, tid)continue@@ -2054,7 +2080,7 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywtry:res_l = await thread_pool_exec(settings.docStoreConn.search,- ["content_with_weight", "compilation_template_kind_kwd"],+ ["content_with_weight", "compilation_template_kind_kwd", "compile_kwd"],[],{"knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD], "must_not": {"exists": "compilation_template_ids"}},[],@@ -2064,25 +2090,43 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywindex_nm,[dataset_id],)- legacy_rows = settings.docStoreConn.get_fields(res_l, ["content_with_weight", "compilation_template_kind_kwd"]) or {}+ legacy_rows = settings.docStoreConn.get_fields(res_l, ["content_with_weight", "compilation_template_kind_kwd", "compile_kwd"]) or {}except Exception:logging.exception("get_dataset_structure: legacy blob fetch failed for kb=%s", dataset_id)legacy_rows = {}legacy_bucket = {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind, "entities": [], "relations": []}+ reconstructed_compile_kwds: set[str] = set()for row in legacy_rows.values():stamped_kind = row.get("compilation_template_kind_kwd") or ""if isinstance(stamped_kind, list):stamped_kind = stamped_kind[0].strip()if _resolve_dataset_structure_kind((stamped_kind or "").strip()) != resolved_kind:continue+ if disabled_doc_ids:+ compile_kwd = _scalar(row.get("compile_kwd"))+ if not compile_kwd or compile_kwd in reconstructed_compile_kwds:+ continue+ reconstructed_compile_kwds.add(compile_kwd)+ try:+ entities, relations = await sgc.build_bucket(+ index_nm,+ dataset_id,+ {"compile_kwd": [compile_kwd], "doc_id": sorted(active_doc_ids)},+ excluded_doc_ids=disabled_doc_ids,+ )+ except Exception:+ continue+ legacy_bucket["entities"].extend(entities)+ legacy_bucket["relations"].extend(relations)+ continuetry:graph = json.loads(row.get("content_with_weight") or "{}")except Exception:continueif not isinstance(graph, dict):continue- legacy_bucket["entities"].extend(graph.get("entities") or [])- legacy_bucket["relations"].extend(graph.get("relations") or [])+ legacy_bucket["entities"].extend(item for item in (graph.get("entities") or []) if isinstance(item, dict))+ legacy_bucket["relations"].extend(item for item in (graph.get("relations") or []) if isinstance(item, dict))if resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:legacy_bucket["entities"] = sgc.filter_entities_with_relations(legacy_bucket["entities"], legacy_bucket["relations"])if legacy_bucket["entities"] or legacy_bucket["relations"]:@@ -2109,8 +2153,9 @@ _ALTERATION_ELIGIBLE_TEMPLATE_KINDS = {"tree": {"tree", "page_index"},}-# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` on-# ``scope_kwd="dataset"`` rows, discovered by their stamped top-level kind.+# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` or+# ``doc_ids_kwd`` on ``scope_kwd="dataset"`` rows, discovered by their stamped+# top-level kind._ALTERATION_KIND_TO_MERGED_ROW_KIND = {"graph": "knowledge_graph","mindmap": "mind_map",@@ -2137,11 +2182,16 @@ async def _current_dataset_docs(dataset_id: str):types=[],suffix=[],)+ docs = [doc for doc in docs if str(doc.get("status", "1")) != "0"]docs = docs or []current_doc_ids = {str(doc.get("id")) for doc in docs if doc.get("id")}return docs, current_doc_ids+async def _disabled_dataset_doc_ids(dataset_id: str) -> set[str]:+ return await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, dataset_id)++def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc_ids: set) -> dict:"""Build the drift response dict shared by every ``kind``."""removed_doc_ids = sorted(involved_doc_ids - current_doc_ids)@@ -2184,6 +2234,8 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:pipeline_cache: dict[str, bool] = {}eligible: set[str] = set()for doc in docs or []:+ if str(doc.get("status", "1")) == "0":+ continuedoc_id = str(doc.get("id") or "")if not doc_id:continue@@ -2196,16 +2248,17 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:return eligible-async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str, from_list: bool) -> set:- """Page a docStore search, folding each row's ``field`` into a doc-id set.+async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str | list[str], from_list: bool) -> set:+ """Page a docStore search, folding provenance fields into a doc-id set.- ``from_list`` reads ``field`` as ``source_doc_ids`` (a list of provenance doc- ids); otherwise ``field`` is the row's own ``doc_id`` scalar.+ ``from_list`` reads the selected fields as lists of provenance document IDs;+ otherwise the selected field is the row's own scalar document ID."""from common.doc_store.doc_store_base import OrderByExprinvolved: set[str] = set()- select_fields = ["id", field]+ fields = [field] if isinstance(field, str) else field+ select_fields = ["id", *fields]offset = 0page_size = 1000while True:@@ -2230,11 +2283,11 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fiif not rows:breakfor row in rows.values():- value = row.get(field)if from_list:- involved.update(_flatten_provenance_doc_ids(value))- elif value:- involved.add(str(value))+ for field_name in fields:+ involved.update(_flatten_provenance_doc_ids(row.get(field_name)))+ else:+ involved.update(_flatten_provenance_doc_ids(row.get(fields[0])))offset += page_sizetotal = settings.docStoreConn.get_total(res)@@ -2253,7 +2306,33 @@ async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> se"scope_kwd": ["dataset"],"compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],}- return await _involved_doc_ids_paged(index_nm, dataset_id, condition, "source_doc_ids", from_list=True)+ involved = await _involved_doc_ids_paged(index_nm, dataset_id, condition, ["source_doc_ids", "doc_ids_kwd"], from_list=True)+ if involved:+ return involved++ # Historical dataset rows may predate provenance columns. Recover the+ # source document set from their template/compile identity so alteration+ # still reports disabled documents instead of treating every active+ # document as newly uploaded.+ template_ids = await _involved_doc_ids_paged(index_nm, dataset_id, condition, "compilation_template_ids", from_list=True)+ compile_kwds = await _involved_doc_ids_paged(index_nm, dataset_id, condition, "compile_kwd", from_list=True)+ if not template_ids and not compile_kwds:+ legacy_condition = {+ "knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD],+ "compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],+ }+ template_ids = await _involved_doc_ids_paged(index_nm, dataset_id, legacy_condition, "compilation_template_ids", from_list=True)+ compile_kwds = await _involved_doc_ids_paged(index_nm, dataset_id, legacy_condition, "compile_kwd", from_list=True)+ source_condition: dict = {"knowledge_graph_kwd": ["entity", "relation"]}+ if template_ids:+ source_condition["compilation_template_ids"] = sorted(template_ids)+ elif compile_kwds:+ source_condition["compile_kwd"] = sorted(compile_kwds)+ else:+ return set()+ involved = await _involved_doc_ids_paged(index_nm, dataset_id, source_condition, "doc_id", from_list=False)+ involved.discard(dataset_id)+ return involvedif kind == "tree":return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": list(_ALTERATION_TREE_COMPILE_KWDS)}, "doc_id", from_list=False)return set()diff --git a/api/apps/services/structure_graph_common.py b/api/apps/services/structure_graph_common.pyindex fd862d2dbd..249ad5910a 100644--- a/api/apps/services/structure_graph_common.py+++ b/api/apps/services/structure_graph_common.py@@ -47,12 +47,24 @@ GRAPH_TOP_ENTITIES = 256# blow up the response.GRAPH_EXPANSION_CAP = 4096-GRAPH_ENTITY_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids"]-GRAPH_RELATION_FIELDS = ["id", "content_with_weight", "from_entity_kwd", "to_entity_kwd"]-GRAPH_ALL_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd"]---async def graph_search(index_name, kb_id, select_fields, condition, order_by, limit, match_expressions=None):+GRAPH_ENTITY_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "doc_id", "doc_ids_kwd", "source_doc_ids"]+GRAPH_RELATION_FIELDS = ["id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"]+GRAPH_ALL_FIELDS = [+ "id",+ "content_with_weight",+ "name_kwd",+ "mention_count_int",+ "source_chunk_ids",+ "from_entity_kwd",+ "to_entity_kwd",+ "knowledge_graph_kwd",+ "doc_id",+ "doc_ids_kwd",+ "source_doc_ids",+]+++async def graph_search(index_name, kb_id, select_fields, condition, order_by, limit, match_expressions=None, offset=0):"""One raw-row search. Returns ``(field_map, total)`` where ``total`` is thefull match count (not the returned slice)."""res = await thread_pool_exec(@@ -62,7 +74,7 @@ async def graph_search(index_name, kb_id, select_fields, condition, order_by, licondition,match_expressions or [],order_by,- 0,+ offset,max(int(limit or 0), 1),index_name,[kb_id],@@ -226,7 +238,38 @@ def filter_entities_with_relations(entities: list[dict], relations: list[dict])return filtered-async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list[dict]]:+def _row_has_enabled_source(row: dict, excluded_doc_ids: set[str]) -> bool:+ if not excluded_doc_ids:+ return True++ def _flatten_ids(value) -> set[str]:+ if value is None:+ return set()+ if isinstance(value, str):+ raw = value.strip()+ if not raw:+ return set()+ try:+ return _flatten_ids(json.loads(raw))+ except (json.JSONDecodeError, TypeError):+ return {raw}+ if isinstance(value, (list, tuple, set)):+ result: set[str] = set()+ for item in value:+ result.update(_flatten_ids(item))+ return result+ return {str(value)}++ source_ids: set[str] = set()+ for field in ("doc_ids_kwd", "source_doc_ids"):+ source_ids.update(_flatten_ids(row.get(field)))+ if source_ids:+ return bool(source_ids - excluded_doc_ids)+ doc_ids = _flatten_ids(row.get("doc_id"))+ return not doc_ids or bool(doc_ids - excluded_doc_ids)+++async def build_bucket(index_name, kb_id, scope: dict, excluded_doc_ids: set[str] | None = None) -> tuple[list[dict], list[dict]]:"""Build one bucket's ``(entities, relations)`` from raw rows.``scope`` is the filter WITHOUT ``knowledge_graph_kwd`` — e.g.@@ -236,6 +279,7 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], listby ``mention_count_int``, the relations sourced from them, and thoserelations' target entities."""+ excluded_doc_ids = excluded_doc_ids or set()both_cond = dict(scope, knowledge_graph_kwd=["entity", "relation"])_, total = await graph_search(index_name, kb_id, ["id"], both_cond, OrderByExpr(), 1)@@ -244,6 +288,8 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], listentities: list[dict] = []relations: list[dict] = []for row in field_map.values():+ if not _row_has_enabled_source(row, excluded_doc_ids):+ continueif row.get("knowledge_graph_kwd") == "relation":edge = project_relation(row)if edge:@@ -261,8 +307,24 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], listorder_by.desc("mention_count_int")except Exception:order_by = OrderByExpr()- ent_a_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"]), order_by, GRAPH_TOP_ENTITIES)- set_a = [n for n in (project_entity(r) for r in ent_a_map.values()) if n]+ set_a: list[dict] = []+ entity_offset = 0+ entity_total = None+ while len(set_a) < GRAPH_TOP_ENTITIES and (entity_total is None or entity_offset < entity_total):+ ent_a_map, entity_total = await graph_search(+ index_name,+ kb_id,+ GRAPH_ENTITY_FIELDS,+ dict(scope, knowledge_graph_kwd=["entity"]),+ order_by,+ GRAPH_TOP_ENTITIES,+ offset=entity_offset,+ )+ if not ent_a_map:+ break+ set_a.extend(n for n in (project_entity(r) for r in ent_a_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n)+ entity_offset += len(ent_a_map)+ set_a = set_a[:GRAPH_TOP_ENTITIES]a_names = sorted({str(e.get("name") or "").strip() for e in set_a if str(e.get("name") or "").strip()})a_name_terms = sorted({term for name in a_names for term in _endpoint_terms(name)})@@ -272,6 +334,8 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], listif a_name_terms:rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], from_entity_kwd=a_name_terms), OrderByExpr(), GRAPH_EXPANSION_CAP)for row in rel_map.values():+ if not _row_has_enabled_source(row, excluded_doc_ids):+ continueedge = project_relation(row)if edge:relations.append(edge)@@ -283,13 +347,22 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], listset_t = []if target_names_lower:tgt_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(target_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)- set_t = [n for n in (project_entity(r) for r in tgt_map.values()) if n]+ set_t = [n for n in (project_entity(r) for r in tgt_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n]entities = dedup_entities(set_a + set_t)return entities, normalize_relation_endpoints(entities, relations)-async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, keywords, scope_for_template, log_ctx="") -> tuple[dict | None, list[dict], list[dict]]:+async def keyword_subgraph(+ index_name,+ kb_id,+ embd_mdl,+ base_entity_condition,+ keywords,+ scope_for_template,+ log_ctx="",+ excluded_doc_ids: set[str] | None = None,+) -> tuple[dict | None, list[dict], list[dict]]:"""KNN the entity rows matching ``base_entity_condition`` for ``keywords``;return ``(top1_bucket_meta, entities, relations)`` for the top-1 entity's1-hop subgraph (top-1 + neighbors + touching relations). ``(None, [], [])``@@ -303,6 +376,8 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, k"""from common.doc_store.doc_store_base import MatchDenseExpr+ excluded_doc_ids = excluded_doc_ids or set()+try:qv, _ = await thread_pool_exec(embd_mdl.encode_queries, keywords)vec = list(qv)@@ -317,14 +392,16 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, kembedding_data=vec,embedding_data_type="float",distance_type="cosine",- topn=1,+ topn=GRAPH_TOP_ENTITIES,extra_options={"similarity": 0.0},)top_fields = GRAPH_ENTITY_FIELDS + ["compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd"]- top_map, _ = await graph_search(index_name, kb_id, top_fields, base_entity_condition, OrderByExpr(), 1, match_expressions=[match_expr])+ top_map, _ = await graph_search(index_name, kb_id, top_fields, base_entity_condition, OrderByExpr(), GRAPH_TOP_ENTITIES, match_expressions=[match_expr])if not top_map:return None, [], []- top_row = next(iter(top_map.values()))+ top_row = next((row for row in top_map.values() if _row_has_enabled_source(row, excluded_doc_ids)), None)+ if top_row is None:+ return None, [], []top_node = project_entity(top_row)if not top_node:return None, [], []@@ -341,6 +418,8 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, kfor field in ("from_entity_kwd", "to_entity_kwd"):rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], **{field: top_name_terms}), OrderByExpr(), GRAPH_EXPANSION_CAP)for row in rel_map.values():+ if not _row_has_enabled_source(row, excluded_doc_ids):+ continueedge = project_relation(row)if not edge:continue@@ -357,7 +436,7 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, kentities = [top_node]if neighbor_names_lower:nb_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(neighbor_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)- entities.extend(n for n in (project_entity(r) for r in nb_map.values()) if n)+ entities.extend(n for n in (project_entity(r) for r in nb_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n)entities = dedup_entities(entities)return bucket_meta, entities, normalize_relation_endpoints(entities, relations)diff --git a/api/db/services/document_service.py b/api/db/services/document_service.pyindex 0a88e85935..0d583cdf26 100644--- a/api/db/services/document_service.py+++ b/api/db/services/document_service.py@@ -42,6 +42,11 @@ from rag.utils.redis_conn import REDIS_CONNclass DocumentService(CommonService):model = Document+ @classmethod+ @DB.connection_context()+ def get_disabled_doc_ids_by_kb_id(cls, kb_id) -> set[str]:+ return {str(doc_id) for doc_id in cls.model.select(cls.model.id).where((cls.model.kb_id == kb_id) & (cls.model.status == "0")).tuples()}+@classmethoddef get_cls_model_fields(cls):return [diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex 6b07b68e53..558dc427fe 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -648,6 +648,38 @@ def _struct_merge_graph_entities(entities: list[dict]) -> list[dict]:target.get("source_chunk_ids"),entity.get("source_chunk_ids"),)+ doc_ids = _union_ordered(+ target.get("doc_ids_kwd"),+ entity.get("doc_ids_kwd"),+ )+ if doc_ids:+ target["doc_ids_kwd"] = doc_ids+ return [merged[key] for key in order]+++def _struct_merge_graph_relations(relations: list[dict]) -> list[dict]:+ """Deduplicate relation payloads while preserving source provenance."""+ merged: dict[tuple[str, str, str], dict] = {}+ order: list[tuple[str, str, str]] = []+ for relation in relations:+ key = (+ str(relation.get("from") or "").strip().casefold(),+ str(relation.get("to") or "").strip().casefold(),+ str(relation.get("type") or "related").strip().casefold(),+ )+ if not key[0] or not key[1]:+ continue+ if key not in merged:+ merged[key] = relation+ order.append(key)+ continue+ target = merged[key]+ doc_ids = _union_ordered(+ target.get("doc_ids_kwd"),+ relation.get("doc_ids_kwd"),+ )+ if doc_ids:+ target["doc_ids_kwd"] = doc_idsreturn [merged[key] for key in order]@@ -2200,7 +2232,7 @@ async def _struct_rebuild_graph_json(from common.doc_store.doc_store_base import OrderByExprindex = _rag_search.index_name(tenant_id)- fields = ["content_with_weight", "knowledge_graph_kwd", "source_chunk_ids"]+ fields = ["content_with_weight", "knowledge_graph_kwd", "source_chunk_ids", "doc_id"]# ``doc_id is None`` collects every document's entities/relations in the KB# for the dataset-level graph; a concrete id keeps it document-scoped.condition: dict = {@@ -2225,22 +2257,35 @@ async def _struct_rebuild_graph_json()rows = settings.docStoreConn.get_fields(res, fields)+ disabled_doc_ids: set[str] = set()+ if doc_id is None:+ from api.db.services.document_service import DocumentService++ disabled_doc_ids = await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)+entities: list[dict] = []relations: list[dict] = []for row in rows.values():+ if str(row.get("doc_id") or "") in disabled_doc_ids:+ continue+ source_doc_id = str(row.get("doc_id") or "").strip()payload = _struct_load_payload(row)if row.get("knowledge_graph_kwd") == "relation":relation = _struct_graph_relation(payload)if relation:+ if doc_id is None and source_doc_id:+ relation["doc_ids_kwd"] = [source_doc_id]relations.append(relation)else:entity = _struct_graph_entity(payload, row.get("source_chunk_ids"))if entity:+ if doc_id is None and source_doc_id:+ entity["doc_ids_kwd"] = [source_doc_id]entities.append(entity)return {"entities": _struct_merge_graph_entities(entities),- "relations": relations,+ "relations": _struct_merge_graph_relations(relations) if doc_id is None else relations,}diff --git a/rag/svr/task_executor_refactor/dataset_structure_merger.py b/rag/svr/task_executor_refactor/dataset_structure_merger.pyindex edc06c075a..c0769a9971 100644--- a/rag/svr/task_executor_refactor/dataset_structure_merger.py+++ b/rag/svr/task_executor_refactor/dataset_structure_merger.py@@ -31,8 +31,11 @@ Key design:from __future__ import annotations+import asyncio+import inspectimport jsonimport logging+from functools import lru_cachefrom typing import Optionalimport xxhash@@ -57,10 +60,22 @@ _PAGE_SIZE = 1000_SCOPE_KWD_DOC = "doc"_SCOPE_KWD_DATASET = "dataset"_DELETION_META_KWD = "doc_deleted"+_EMBED_BATCH_SIZE = 32+_EMBED_MAX_IN_FLIGHT = 4+_BUCKET_MERGE_MAX_IN_FLIGHT = 4# Row id seeds for metadata_META_ROW_KWD = "kg_build_meta"++@lru_cache(maxsize=None)+def _supports_bulk_refresh(connection_type: type) -> bool:+ try:+ return "refresh" in inspect.signature(connection_type.insert).parameters+ except (TypeError, ValueError):+ return False++STRUCTURE_MERGE_TASK_TYPES = frozenset({"structure_graph",@@ -152,11 +167,51 @@ async def _index_insert(tenant_id: str, kb_id: str, rows: list[dict]) -> None:returnindex = _index_name(tenant_id)try:- await thread_pool_exec(settings.docStoreConn.insert, rows, index, kb_id)+ insert = settings.docStoreConn.insert+ if _supports_bulk_refresh(type(settings.docStoreConn)):+ await thread_pool_exec(insert, rows, index, kb_id, refresh=False)+ else:+ await thread_pool_exec(insert, rows, index, kb_id)except Exception:logging.exception("structure_merge: insert failed for kb=%s", kb_id)+async def _refresh_index(tenant_id: str, kb_id: str) -> None:+ """Refresh search visibility once after a merge, instead of per bulk write."""+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)+ if not callable(refresh_idx):+ return+ try:+ await thread_pool_exec(refresh_idx, _index_name(tenant_id))+ except Exception:+ logging.exception("structure_merge: final index refresh failed for kb=%s", kb_id)+++async def _embed_rows(embd_mdl, rows: list[dict], texts: list[str]) -> None:+ """Embed rows in bounded concurrent batches and attach vectors in order."""+ if not embd_mdl or not rows:+ return++ semaphore = asyncio.Semaphore(_EMBED_MAX_IN_FLIGHT)++ async def _embed_batch(start: int):+ async with semaphore:+ return start, await _encode(embd_mdl, texts[start : start + _EMBED_BATCH_SIZE])++ tasks = [_embed_batch(start) for start in range(0, len(rows), _EMBED_BATCH_SIZE)]+ results = await asyncio.gather(*tasks)+ for start, vectors in results:+ for offset, vector in enumerate(vectors):+ if vector is not None and len(vector) > 0:+ rows[start + offset][f"q_{len(vector)}_vec"] = list(vector)+++async def _disabled_doc_ids(kb_id: str) -> set[str]:+ from api.db.services.document_service import DocumentService++ return await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)++# ---------------------------------------------------------------------------# Document deletion tracking (sync, called from DocumentService.remove_document)# ---------------------------------------------------------------------------@@ -268,6 +323,7 @@ async def _merge_bucket(groups.setdefault(key, []).append(row)rows_out: list[dict] = []+ embedding_texts: list[str] = []for (name_lower, typ), rows in groups.items():# Collect all source_chunk_ids, doc_ids, and original-cased namesall_chunks: list[str] = []@@ -319,13 +375,10 @@ async def _merge_bucket(row["compilation_template_ids"] = [template_id]if structure_kind:row["compilation_template_kind_kwd"] = str(structure_kind)- # Re-embed- if embd_mdl:- vecs = await _encode(embd_mdl, [best_desc])- if vecs and len(vecs[0]) > 0:- dim = len(vecs[0])- row[f"q_{dim}_vec"] = list(vecs[0])rows_out.append(row)+ embedding_texts.append(best_desc)++ await _embed_rows(embd_mdl, rows_out, embedding_texts)return rows_out@@ -356,6 +409,7 @@ async def _merge_relations(groups.setdefault(key, []).append(row)rows_out: list[dict] = []+ embedding_texts: list[str] = []for (src, rel_type, tgt), rows in groups.items():all_chunks: list[str] = []all_docs: list[str] = []@@ -397,13 +451,10 @@ async def _merge_relations(row["compilation_template_ids"] = [template_id]if structure_kind:row["compilation_template_kind_kwd"] = str(structure_kind)- # Re-embed (matching _merge_bucket)- if embd_mdl:- vecs = await _encode(embd_mdl, [best_desc])- if vecs and len(vecs[0]) > 0:- dim = len(vecs[0])- row[f"q_{dim}_vec"] = list(vecs[0])rows_out.append(row)+ embedding_texts.append(best_desc)++ await _embed_rows(embd_mdl, rows_out, embedding_texts)return rows_out@@ -614,6 +665,7 @@ async def _do_build(structure_kind: str | None,embd_mdl,incremental: bool = True,+ disabled_doc_ids: set[str] | None = None,) -> bool:"""Core build logic — read doc_graph rows, bucket, merge, write dataset rows.@@ -629,6 +681,7 @@ async def _do_build(markers existed), ``False`` if a store error prevented cleanup."""cleanup_ok = True+ disabled_doc_ids = disabled_doc_ids or set()index = _index_name(tenant_id)@@ -722,6 +775,8 @@ async def _do_build(if not rows:breakfor row in rows:+ if str(row.get("doc_id") or "") in disabled_doc_ids:+ continue# Time filter (incremental mode)if ts_cond:ts = row.get("create_timestamp_flt", 0)@@ -749,37 +804,25 @@ async def _do_build(else:offset += limit- # Merge entity buckets into dataset rows- for bi in range(_BUCKET_COUNT):- if buckets[bi]:- out = await _merge_bucket(- tenant_id,- kb_id,- compile_kwd,- template_id,- buckets[bi],- embd_mdl,- structure_kind,- )- if out:- for start in range(0, len(out), _BUCKET_FLUSH_THRESHOLD):- await _index_insert(tenant_id, kb_id, out[start : start + _BUCKET_FLUSH_THRESHOLD])-- # Merge relation buckets into dataset rows- for bi in range(_BUCKET_COUNT):- if rel_buckets[bi]:- rel_out = await _merge_relations(- tenant_id,- kb_id,- compile_kwd,- template_id,- rel_buckets[bi],- embd_mdl,- structure_kind,- )- if rel_out:- for start in range(0, len(rel_out), _BUCKET_FLUSH_THRESHOLD):- await _index_insert(tenant_id, kb_id, rel_out[start : start + _BUCKET_FLUSH_THRESHOLD])+ async def _merge_buckets(bucket_rows, merge_fn):+ non_empty = [rows for rows in bucket_rows if rows]+ for start in range(0, len(non_empty), _BUCKET_MERGE_MAX_IN_FLIGHT):+ merged = await asyncio.gather(*(merge_fn(rows) for rows in non_empty[start : start + _BUCKET_MERGE_MAX_IN_FLIGHT]))+ for rows in merged:+ for batch_start in range(0, len(rows), _BUCKET_FLUSH_THRESHOLD):+ await _index_insert(tenant_id, kb_id, rows[batch_start : batch_start + _BUCKET_FLUSH_THRESHOLD])++ # Merge entity and relation buckets concurrently in bounded groups. Each+ # bucket embeds its rows in batches, so neither the bucket loop nor the+ # individual row loop is a serial embedding bottleneck.+ await _merge_buckets(+ buckets,+ lambda rows: _merge_bucket(tenant_id, kb_id, compile_kwd, template_id, rows, embd_mdl, structure_kind),+ )+ await _merge_buckets(+ rel_buckets,+ lambda rows: _merge_relations(tenant_id, kb_id, compile_kwd, template_id, rows, embd_mdl, structure_kind),+ )# ── Save build timestamp ─────────────────────────────────────────import datetime@@ -860,6 +903,11 @@ async def run_structure_merge(ctx: TaskContext) -> None:progress(1.0, f"No eligible templates for {kind_label}.")return+ disabled_doc_ids = await _disabled_doc_ids(ctx.kb_id)+ # A disabled document does not trigger an immediate rebuild. On the next+ # explicit merge, rebuild from all active source rows so shared entities+ # and relations are recalculated without the disabled document's data.+ rebuild_all = bool(disabled_doc_ids)total = len(eligible)rebuilt = 0all_cleanup_ok = True@@ -869,7 +917,16 @@ async def run_structure_merge(ctx: TaskContext) -> None:returnprogress(0.05 + 0.9 * (i / total), f"Building dataset graph {i + 1}/{total} ...")try:- cleanup_ok = await _do_build(ctx.tenant_id, ctx.kb_id, compile_kwd, template_id, structure_kind, embd_mdl)+ cleanup_ok = await _do_build(+ ctx.tenant_id,+ ctx.kb_id,+ compile_kwd,+ template_id,+ structure_kind,+ embd_mdl,+ incremental=not rebuild_all,+ disabled_doc_ids=disabled_doc_ids,+ )if not cleanup_ok:all_cleanup_ok = Falserebuilt += 1@@ -877,6 +934,10 @@ async def run_structure_merge(ctx: TaskContext) -> None:all_cleanup_ok = Falselogging.exception("structure_merge: build failed for kb=%s compile_kwd=%s", ctx.kb_id, compile_kwd)+ # Bulk inserts use refresh=False; make the complete merge visible once all+ # templates have been written instead of waiting on every bulk request.+ await _refresh_index(ctx.tenant_id, ctx.kb_id)+# Delete pending deletion markers only after every pair has been# processed successfully. If any pair had a store error the markers# survive so the next run retries the cleanup.