Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: handle disabled documents in dataset structures (#18033)

buua436c0582b8e1834
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index ffee50c209..a465bf2993 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -949,15 +949,19 @@ def delete_index(dataset_id: str, tenant_id: str, index_type: str, wipe: bool =
elif wipe and index_type in _STRUCTURE_INDEX_TYPES:
from rag.nlp import search
- # Wipe the merged KB-wide dataset_graph rows for the requested kind
- # (all kinds for the merge-all "structure" type). The per-document
- # entity/relation rows the merge reads from are left intact.
+ # Wipe the merged KB-wide metadata and entity/relation rows for the
+ # requested kind (all kinds for the merge-all "structure" type). The
+ # per-document entity/relation rows the merge reads from are left intact.
friendly = _STRUCTURE_INDEX_TYPE_TO_KIND.get(index_type)
resolved_kind = _resolve_dataset_structure_kind(friendly) if friendly else None
- condition: dict = {"knowledge_graph_kwd": ["dataset_graph"]}
- if resolved_kind:
- condition["compilation_template_kind_kwd"] = [resolved_kind]
- settings.docStoreConn.delete(condition, search.index_name(kb.tenant_id), dataset_id)
+ conditions = [
+ {"knowledge_graph_kwd": ["dataset_graph"]},
+ {"knowledge_graph_kwd": ["entity", "relation"], "scope_kwd": ["dataset"]},
+ ]
+ for condition in conditions:
+ if resolved_kind:
+ condition["compilation_template_kind_kwd"] = [resolved_kind]
+ settings.docStoreConn.delete(condition, search.index_name(kb.tenant_id), dataset_id)
KnowledgebaseService.update_by_id(kb.id, {task_id_field: "", task_finish_at_field: None})
return True, {}
@@ -1841,6 +1845,14 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
from api.apps.services import structure_graph_common as sgc
keywords = (keywords or "").strip()
+ _, active_doc_ids = await _current_dataset_docs(dataset_id)
+ disabled_doc_ids = await _disabled_dataset_doc_ids(dataset_id)
+ if not active_doc_ids:
+ return True, empty
+ # Dataset rows use the KB id as ``doc_id``. If a historical row has no
+ # source provenance, exclude it while documents are disabled and fall back
+ # to active document rows instead of returning potentially stale content.
+ dataset_excluded_doc_ids = disabled_doc_ids | ({dataset_id} if disabled_doc_ids else set())
def _row_template_id(row: dict) -> str | None:
raw = row.get("compilation_template_ids")
@@ -2011,6 +2023,7 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
keywords,
_scope_for_template,
log_ctx=f"kb={dataset_id}",
+ excluded_doc_ids=dataset_excluded_doc_ids,
)
if resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:
kw_entities = sgc.filter_entities_with_relations(kw_entities, kw_relations)
@@ -2028,13 +2041,26 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
for tid in kind_template_ids:
scope_kwd = template_scope_by_id.get(tid, "dataset")
try:
- entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]})
+ scope = {"compilation_template_ids": [tid], "scope_kwd": [scope_kwd]}
+ if scope_kwd == "doc":
+ scope["doc_id"] = sorted(active_doc_ids)
+ entities, relations = await sgc.build_bucket(
+ index_nm,
+ dataset_id,
+ scope,
+ excluded_doc_ids=dataset_excluded_doc_ids if scope_kwd == "dataset" else disabled_doc_ids,
+ )
except Exception:
logging.exception("get_dataset_structure: bucket build failed for kb=%s template=%s", dataset_id, tid)
continue
if scope_kwd == "dataset" and not entities and not relations:
try:
- entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid], "scope_kwd": ["doc"]})
+ entities, relations = await sgc.build_bucket(
+ index_nm,
+ dataset_id,
+ {"compilation_template_ids": [tid], "scope_kwd": ["doc"], "doc_id": sorted(active_doc_ids)},
+ excluded_doc_ids=disabled_doc_ids,
+ )
except Exception:
logging.exception("get_dataset_structure: doc fallback bucket build failed for kb=%s template=%s", dataset_id, tid)
continue
@@ -2054,7 +2080,7 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
try:
res_l = await thread_pool_exec(
settings.docStoreConn.search,
- ["content_with_weight", "compilation_template_kind_kwd"],
+ ["content_with_weight", "compilation_template_kind_kwd", "compile_kwd"],
[],
{"knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD], "must_not": {"exists": "compilation_template_ids"}},
[],
@@ -2064,25 +2090,43 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keyw
index_nm,
[dataset_id],
)
- legacy_rows = settings.docStoreConn.get_fields(res_l, ["content_with_weight", "compilation_template_kind_kwd"]) or {}
+ legacy_rows = settings.docStoreConn.get_fields(res_l, ["content_with_weight", "compilation_template_kind_kwd", "compile_kwd"]) or {}
except Exception:
logging.exception("get_dataset_structure: legacy blob fetch failed for kb=%s", dataset_id)
legacy_rows = {}
legacy_bucket = {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind, "entities": [], "relations": []}
+ reconstructed_compile_kwds: set[str] = set()
for row in legacy_rows.values():
stamped_kind = row.get("compilation_template_kind_kwd") or ""
if isinstance(stamped_kind, list):
stamped_kind = stamped_kind[0].strip()
if _resolve_dataset_structure_kind((stamped_kind or "").strip()) != resolved_kind:
continue
+ if disabled_doc_ids:
+ compile_kwd = _scalar(row.get("compile_kwd"))
+ if not compile_kwd or compile_kwd in reconstructed_compile_kwds:
+ continue
+ reconstructed_compile_kwds.add(compile_kwd)
+ try:
+ entities, relations = await sgc.build_bucket(
+ index_nm,
+ dataset_id,
+ {"compile_kwd": [compile_kwd], "doc_id": sorted(active_doc_ids)},
+ excluded_doc_ids=disabled_doc_ids,
+ )
+ except Exception:
+ continue
+ legacy_bucket["entities"].extend(entities)
+ legacy_bucket["relations"].extend(relations)
+ continue
try:
graph = json.loads(row.get("content_with_weight") or "{}")
except Exception:
continue
if not isinstance(graph, dict):
continue
- legacy_bucket["entities"].extend(graph.get("entities") or [])
- legacy_bucket["relations"].extend(graph.get("relations") or [])
+ legacy_bucket["entities"].extend(item for item in (graph.get("entities") or []) if isinstance(item, dict))
+ legacy_bucket["relations"].extend(item for item in (graph.get("relations") or []) if isinstance(item, dict))
if resolved_kind in {"knowledge_graph", "mind_map", "timeline"}:
legacy_bucket["entities"] = sgc.filter_entities_with_relations(legacy_bucket["entities"], legacy_bucket["relations"])
if legacy_bucket["entities"] or legacy_bucket["relations"]:
@@ -2109,8 +2153,9 @@ _ALTERATION_ELIGIBLE_TEMPLATE_KINDS = {
"tree": {"tree", "page_index"},
}
-# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` on
-# ``scope_kwd="dataset"`` rows, discovered by their stamped top-level kind.
+# Dataset-merged structure kinds carry provenance in ``source_doc_ids`` or
+# ``doc_ids_kwd`` on ``scope_kwd="dataset"`` rows, discovered by their stamped
+# top-level kind.
_ALTERATION_KIND_TO_MERGED_ROW_KIND = {
"graph": "knowledge_graph",
"mindmap": "mind_map",
@@ -2137,11 +2182,16 @@ async def _current_dataset_docs(dataset_id: str):
types=[],
suffix=[],
)
+ docs = [doc for doc in docs if str(doc.get("status", "1")) != "0"]
docs = docs or []
current_doc_ids = {str(doc.get("id")) for doc in docs if doc.get("id")}
return docs, current_doc_ids
+async def _disabled_dataset_doc_ids(dataset_id: str) -> set[str]:
+ return await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, dataset_id)
+
+
def _alteration_result(current_doc_ids: set, involved_doc_ids: set, eligible_doc_ids: set) -> dict:
"""Build the drift response dict shared by every ``kind``."""
removed_doc_ids = sorted(involved_doc_ids - current_doc_ids)
@@ -2184,6 +2234,8 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:
pipeline_cache: dict[str, bool] = {}
eligible: set[str] = set()
for doc in docs or []:
+ if str(doc.get("status", "1")) == "0":
+ continue
doc_id = str(doc.get("id") or "")
if not doc_id:
continue
@@ -2196,16 +2248,17 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:
return eligible
-async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str, from_list: bool) -> set:
- """Page a docStore search, folding each row's ``field`` into a doc-id set.
+async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str | list[str], from_list: bool) -> set:
+ """Page a docStore search, folding provenance fields into a doc-id set.
- ``from_list`` reads ``field`` as ``source_doc_ids`` (a list of provenance doc
- ids); otherwise ``field`` is the row's own ``doc_id`` scalar.
+ ``from_list`` reads the selected fields as lists of provenance document IDs;
+ otherwise the selected field is the row's own scalar document ID.
"""
from common.doc_store.doc_store_base import OrderByExpr
involved: set[str] = set()
- select_fields = ["id", field]
+ fields = [field] if isinstance(field, str) else field
+ select_fields = ["id", *fields]
offset = 0
page_size = 1000
while True:
@@ -2230,11 +2283,11 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fi
if not rows:
break
for row in rows.values():
- value = row.get(field)
if from_list:
- involved.update(_flatten_provenance_doc_ids(value))
- elif value:
- involved.add(str(value))
+ for field_name in fields:
+ involved.update(_flatten_provenance_doc_ids(row.get(field_name)))
+ else:
+ involved.update(_flatten_provenance_doc_ids(row.get(fields[0])))
offset += page_size
total = settings.docStoreConn.get_total(res)
@@ -2253,7 +2306,33 @@ async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str) -> se
"scope_kwd": ["dataset"],
"compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],
}
- return await _involved_doc_ids_paged(index_nm, dataset_id, condition, "source_doc_ids", from_list=True)
+ involved = await _involved_doc_ids_paged(index_nm, dataset_id, condition, ["source_doc_ids", "doc_ids_kwd"], from_list=True)
+ if involved:
+ return involved
+
+ # Historical dataset rows may predate provenance columns. Recover the
+ # source document set from their template/compile identity so alteration
+ # still reports disabled documents instead of treating every active
+ # document as newly uploaded.
+ template_ids = await _involved_doc_ids_paged(index_nm, dataset_id, condition, "compilation_template_ids", from_list=True)
+ compile_kwds = await _involved_doc_ids_paged(index_nm, dataset_id, condition, "compile_kwd", from_list=True)
+ if not template_ids and not compile_kwds:
+ legacy_condition = {
+ "knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD],
+ "compilation_template_kind_kwd": [_ALTERATION_KIND_TO_MERGED_ROW_KIND[kind]],
+ }
+ template_ids = await _involved_doc_ids_paged(index_nm, dataset_id, legacy_condition, "compilation_template_ids", from_list=True)
+ compile_kwds = await _involved_doc_ids_paged(index_nm, dataset_id, legacy_condition, "compile_kwd", from_list=True)
+ source_condition: dict = {"knowledge_graph_kwd": ["entity", "relation"]}
+ if template_ids:
+ source_condition["compilation_template_ids"] = sorted(template_ids)
+ elif compile_kwds:
+ source_condition["compile_kwd"] = sorted(compile_kwds)
+ else:
+ return set()
+ involved = await _involved_doc_ids_paged(index_nm, dataset_id, source_condition, "doc_id", from_list=False)
+ involved.discard(dataset_id)
+ return involved
if kind == "tree":
return await _involved_doc_ids_paged(index_nm, dataset_id, {"compile_kwd": list(_ALTERATION_TREE_COMPILE_KWDS)}, "doc_id", from_list=False)
return set()
diff --git a/api/apps/services/structure_graph_common.py b/api/apps/services/structure_graph_common.py
index fd862d2dbd..249ad5910a 100644
--- a/api/apps/services/structure_graph_common.py
+++ b/api/apps/services/structure_graph_common.py
@@ -47,12 +47,24 @@ GRAPH_TOP_ENTITIES = 256
# blow up the response.
GRAPH_EXPANSION_CAP = 4096
-GRAPH_ENTITY_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids"]
-GRAPH_RELATION_FIELDS = ["id", "content_with_weight", "from_entity_kwd", "to_entity_kwd"]
-GRAPH_ALL_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd"]
-
-
-async def graph_search(index_name, kb_id, select_fields, condition, order_by, limit, match_expressions=None):
+GRAPH_ENTITY_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "doc_id", "doc_ids_kwd", "source_doc_ids"]
+GRAPH_RELATION_FIELDS = ["id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"]
+GRAPH_ALL_FIELDS = [
+ "id",
+ "content_with_weight",
+ "name_kwd",
+ "mention_count_int",
+ "source_chunk_ids",
+ "from_entity_kwd",
+ "to_entity_kwd",
+ "knowledge_graph_kwd",
+ "doc_id",
+ "doc_ids_kwd",
+ "source_doc_ids",
+]
+
+
+async def graph_search(index_name, kb_id, select_fields, condition, order_by, limit, match_expressions=None, offset=0):
"""One raw-row search. Returns ``(field_map, total)`` where ``total`` is the
full match count (not the returned slice)."""
res = await thread_pool_exec(
@@ -62,7 +74,7 @@ async def graph_search(index_name, kb_id, select_fields, condition, order_by, li
condition,
match_expressions or [],
order_by,
- 0,
+ offset,
max(int(limit or 0), 1),
index_name,
[kb_id],
@@ -226,7 +238,38 @@ def filter_entities_with_relations(entities: list[dict], relations: list[dict])
return filtered
-async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list[dict]]:
+def _row_has_enabled_source(row: dict, excluded_doc_ids: set[str]) -> bool:
+ if not excluded_doc_ids:
+ return True
+
+ def _flatten_ids(value) -> set[str]:
+ if value is None:
+ return set()
+ if isinstance(value, str):
+ raw = value.strip()
+ if not raw:
+ return set()
+ try:
+ return _flatten_ids(json.loads(raw))
+ except (json.JSONDecodeError, TypeError):
+ return {raw}
+ if isinstance(value, (list, tuple, set)):
+ result: set[str] = set()
+ for item in value:
+ result.update(_flatten_ids(item))
+ return result
+ return {str(value)}
+
+ source_ids: set[str] = set()
+ for field in ("doc_ids_kwd", "source_doc_ids"):
+ source_ids.update(_flatten_ids(row.get(field)))
+ if source_ids:
+ return bool(source_ids - excluded_doc_ids)
+ doc_ids = _flatten_ids(row.get("doc_id"))
+ return not doc_ids or bool(doc_ids - excluded_doc_ids)
+
+
+async def build_bucket(index_name, kb_id, scope: dict, excluded_doc_ids: set[str] | None = None) -> tuple[list[dict], list[dict]]:
"""Build one bucket's ``(entities, relations)`` from raw rows.
``scope`` is the filter WITHOUT ``knowledge_graph_kwd`` — e.g.
@@ -236,6 +279,7 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list
by ``mention_count_int``, the relations sourced from them, and those
relations' target entities.
"""
+ excluded_doc_ids = excluded_doc_ids or set()
both_cond = dict(scope, knowledge_graph_kwd=["entity", "relation"])
_, total = await graph_search(index_name, kb_id, ["id"], both_cond, OrderByExpr(), 1)
@@ -244,6 +288,8 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list
entities: list[dict] = []
relations: list[dict] = []
for row in field_map.values():
+ if not _row_has_enabled_source(row, excluded_doc_ids):
+ continue
if row.get("knowledge_graph_kwd") == "relation":
edge = project_relation(row)
if edge:
@@ -261,8 +307,24 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list
order_by.desc("mention_count_int")
except Exception:
order_by = OrderByExpr()
- ent_a_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"]), order_by, GRAPH_TOP_ENTITIES)
- set_a = [n for n in (project_entity(r) for r in ent_a_map.values()) if n]
+ set_a: list[dict] = []
+ entity_offset = 0
+ entity_total = None
+ while len(set_a) < GRAPH_TOP_ENTITIES and (entity_total is None or entity_offset < entity_total):
+ ent_a_map, entity_total = await graph_search(
+ index_name,
+ kb_id,
+ GRAPH_ENTITY_FIELDS,
+ dict(scope, knowledge_graph_kwd=["entity"]),
+ order_by,
+ GRAPH_TOP_ENTITIES,
+ offset=entity_offset,
+ )
+ if not ent_a_map:
+ break
+ set_a.extend(n for n in (project_entity(r) for r in ent_a_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n)
+ entity_offset += len(ent_a_map)
+ set_a = set_a[:GRAPH_TOP_ENTITIES]
a_names = sorted({str(e.get("name") or "").strip() for e in set_a if str(e.get("name") or "").strip()})
a_name_terms = sorted({term for name in a_names for term in _endpoint_terms(name)})
@@ -272,6 +334,8 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list
if a_name_terms:
rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], from_entity_kwd=a_name_terms), OrderByExpr(), GRAPH_EXPANSION_CAP)
for row in rel_map.values():
+ if not _row_has_enabled_source(row, excluded_doc_ids):
+ continue
edge = project_relation(row)
if edge:
relations.append(edge)
@@ -283,13 +347,22 @@ async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list
set_t = []
if target_names_lower:
tgt_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(target_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)
- set_t = [n for n in (project_entity(r) for r in tgt_map.values()) if n]
+ set_t = [n for n in (project_entity(r) for r in tgt_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n]
entities = dedup_entities(set_a + set_t)
return entities, normalize_relation_endpoints(entities, relations)
-async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, keywords, scope_for_template, log_ctx="") -> tuple[dict | None, list[dict], list[dict]]:
+async def keyword_subgraph(
+ index_name,
+ kb_id,
+ embd_mdl,
+ base_entity_condition,
+ keywords,
+ scope_for_template,
+ log_ctx="",
+ excluded_doc_ids: set[str] | None = None,
+) -> tuple[dict | None, list[dict], list[dict]]:
"""KNN the entity rows matching ``base_entity_condition`` for ``keywords``;
return ``(top1_bucket_meta, entities, relations)`` for the top-1 entity's
1-hop subgraph (top-1 + neighbors + touching relations). ``(None, [], [])``
@@ -303,6 +376,8 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, k
"""
from common.doc_store.doc_store_base import MatchDenseExpr
+ excluded_doc_ids = excluded_doc_ids or set()
+
try:
qv, _ = await thread_pool_exec(embd_mdl.encode_queries, keywords)
vec = list(qv)
@@ -317,14 +392,16 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, k
embedding_data=vec,
embedding_data_type="float",
distance_type="cosine",
- topn=1,
+ topn=GRAPH_TOP_ENTITIES,
extra_options={"similarity": 0.0},
)
top_fields = GRAPH_ENTITY_FIELDS + ["compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd"]
- top_map, _ = await graph_search(index_name, kb_id, top_fields, base_entity_condition, OrderByExpr(), 1, match_expressions=[match_expr])
+ top_map, _ = await graph_search(index_name, kb_id, top_fields, base_entity_condition, OrderByExpr(), GRAPH_TOP_ENTITIES, match_expressions=[match_expr])
if not top_map:
return None, [], []
- top_row = next(iter(top_map.values()))
+ top_row = next((row for row in top_map.values() if _row_has_enabled_source(row, excluded_doc_ids)), None)
+ if top_row is None:
+ return None, [], []
top_node = project_entity(top_row)
if not top_node:
return None, [], []
@@ -341,6 +418,8 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, k
for field in ("from_entity_kwd", "to_entity_kwd"):
rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], **{field: top_name_terms}), OrderByExpr(), GRAPH_EXPANSION_CAP)
for row in rel_map.values():
+ if not _row_has_enabled_source(row, excluded_doc_ids):
+ continue
edge = project_relation(row)
if not edge:
continue
@@ -357,7 +436,7 @@ async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, k
entities = [top_node]
if neighbor_names_lower:
nb_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(neighbor_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)
- entities.extend(n for n in (project_entity(r) for r in nb_map.values()) if n)
+ entities.extend(n for n in (project_entity(r) for r in nb_map.values() if _row_has_enabled_source(r, excluded_doc_ids)) if n)
entities = dedup_entities(entities)
return bucket_meta, entities, normalize_relation_endpoints(entities, relations)
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index 0a88e85935..0d583cdf26 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -42,6 +42,11 @@ from rag.utils.redis_conn import REDIS_CONN
class DocumentService(CommonService):
model = Document
+ @classmethod
+ @DB.connection_context()
+ def get_disabled_doc_ids_by_kb_id(cls, kb_id) -> set[str]:
+ return {str(doc_id) for doc_id in cls.model.select(cls.model.id).where((cls.model.kb_id == kb_id) & (cls.model.status == "0")).tuples()}
+
@classmethod
def get_cls_model_fields(cls):
return [
diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py
index 6b07b68e53..558dc427fe 100644
--- a/rag/advanced_rag/knowlege_compile/structure.py
+++ b/rag/advanced_rag/knowlege_compile/structure.py
@@ -648,6 +648,38 @@ def _struct_merge_graph_entities(entities: list[dict]) -> list[dict]:
target.get("source_chunk_ids"),
entity.get("source_chunk_ids"),
)
+ doc_ids = _union_ordered(
+ target.get("doc_ids_kwd"),
+ entity.get("doc_ids_kwd"),
+ )
+ if doc_ids:
+ target["doc_ids_kwd"] = doc_ids
+ return [merged[key] for key in order]
+
+
+def _struct_merge_graph_relations(relations: list[dict]) -> list[dict]:
+ """Deduplicate relation payloads while preserving source provenance."""
+ merged: dict[tuple[str, str, str], dict] = {}
+ order: list[tuple[str, str, str]] = []
+ for relation in relations:
+ key = (
+ str(relation.get("from") or "").strip().casefold(),
+ str(relation.get("to") or "").strip().casefold(),
+ str(relation.get("type") or "related").strip().casefold(),
+ )
+ if not key[0] or not key[1]:
+ continue
+ if key not in merged:
+ merged[key] = relation
+ order.append(key)
+ continue
+ target = merged[key]
+ doc_ids = _union_ordered(
+ target.get("doc_ids_kwd"),
+ relation.get("doc_ids_kwd"),
+ )
+ if doc_ids:
+ target["doc_ids_kwd"] = doc_ids
return [merged[key] for key in order]
@@ -2200,7 +2232,7 @@ async def _struct_rebuild_graph_json(
from common.doc_store.doc_store_base import OrderByExpr
index = _rag_search.index_name(tenant_id)
- fields = ["content_with_weight", "knowledge_graph_kwd", "source_chunk_ids"]
+ fields = ["content_with_weight", "knowledge_graph_kwd", "source_chunk_ids", "doc_id"]
# ``doc_id is None`` collects every document's entities/relations in the KB
# for the dataset-level graph; a concrete id keeps it document-scoped.
condition: dict = {
@@ -2225,22 +2257,35 @@ async def _struct_rebuild_graph_json(
)
rows = settings.docStoreConn.get_fields(res, fields)
+ disabled_doc_ids: set[str] = set()
+ if doc_id is None:
+ from api.db.services.document_service import DocumentService
+
+ disabled_doc_ids = await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)
+
entities: list[dict] = []
relations: list[dict] = []
for row in rows.values():
+ if str(row.get("doc_id") or "") in disabled_doc_ids:
+ continue
+ source_doc_id = str(row.get("doc_id") or "").strip()
payload = _struct_load_payload(row)
if row.get("knowledge_graph_kwd") == "relation":
relation = _struct_graph_relation(payload)
if relation:
+ if doc_id is None and source_doc_id:
+ relation["doc_ids_kwd"] = [source_doc_id]
relations.append(relation)
else:
entity = _struct_graph_entity(payload, row.get("source_chunk_ids"))
if entity:
+ if doc_id is None and source_doc_id:
+ entity["doc_ids_kwd"] = [source_doc_id]
entities.append(entity)
return {
"entities": _struct_merge_graph_entities(entities),
- "relations": relations,
+ "relations": _struct_merge_graph_relations(relations) if doc_id is None else relations,
}
diff --git a/rag/svr/task_executor_refactor/dataset_structure_merger.py b/rag/svr/task_executor_refactor/dataset_structure_merger.py
index edc06c075a..c0769a9971 100644
--- a/rag/svr/task_executor_refactor/dataset_structure_merger.py
+++ b/rag/svr/task_executor_refactor/dataset_structure_merger.py
@@ -31,8 +31,11 @@ Key design:
from __future__ import annotations
+import asyncio
+import inspect
import json
import logging
+from functools import lru_cache
from typing import Optional
import xxhash
@@ -57,10 +60,22 @@ _PAGE_SIZE = 1000
_SCOPE_KWD_DOC = "doc"
_SCOPE_KWD_DATASET = "dataset"
_DELETION_META_KWD = "doc_deleted"
+_EMBED_BATCH_SIZE = 32
+_EMBED_MAX_IN_FLIGHT = 4
+_BUCKET_MERGE_MAX_IN_FLIGHT = 4
# Row id seeds for metadata
_META_ROW_KWD = "kg_build_meta"
+
+@lru_cache(maxsize=None)
+def _supports_bulk_refresh(connection_type: type) -> bool:
+ try:
+ return "refresh" in inspect.signature(connection_type.insert).parameters
+ except (TypeError, ValueError):
+ return False
+
+
STRUCTURE_MERGE_TASK_TYPES = frozenset(
{
"structure_graph",
@@ -152,11 +167,51 @@ async def _index_insert(tenant_id: str, kb_id: str, rows: list[dict]) -> None:
return
index = _index_name(tenant_id)
try:
- await thread_pool_exec(settings.docStoreConn.insert, rows, index, kb_id)
+ insert = settings.docStoreConn.insert
+ if _supports_bulk_refresh(type(settings.docStoreConn)):
+ await thread_pool_exec(insert, rows, index, kb_id, refresh=False)
+ else:
+ await thread_pool_exec(insert, rows, index, kb_id)
except Exception:
logging.exception("structure_merge: insert failed for kb=%s", kb_id)
+async def _refresh_index(tenant_id: str, kb_id: str) -> None:
+ """Refresh search visibility once after a merge, instead of per bulk write."""
+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)
+ if not callable(refresh_idx):
+ return
+ try:
+ await thread_pool_exec(refresh_idx, _index_name(tenant_id))
+ except Exception:
+ logging.exception("structure_merge: final index refresh failed for kb=%s", kb_id)
+
+
+async def _embed_rows(embd_mdl, rows: list[dict], texts: list[str]) -> None:
+ """Embed rows in bounded concurrent batches and attach vectors in order."""
+ if not embd_mdl or not rows:
+ return
+
+ semaphore = asyncio.Semaphore(_EMBED_MAX_IN_FLIGHT)
+
+ async def _embed_batch(start: int):
+ async with semaphore:
+ return start, await _encode(embd_mdl, texts[start : start + _EMBED_BATCH_SIZE])
+
+ tasks = [_embed_batch(start) for start in range(0, len(rows), _EMBED_BATCH_SIZE)]
+ results = await asyncio.gather(*tasks)
+ for start, vectors in results:
+ for offset, vector in enumerate(vectors):
+ if vector is not None and len(vector) > 0:
+ rows[start + offset][f"q_{len(vector)}_vec"] = list(vector)
+
+
+async def _disabled_doc_ids(kb_id: str) -> set[str]:
+ from api.db.services.document_service import DocumentService
+
+ return await thread_pool_exec(DocumentService.get_disabled_doc_ids_by_kb_id, kb_id)
+
+
# ---------------------------------------------------------------------------
# Document deletion tracking (sync, called from DocumentService.remove_document)
# ---------------------------------------------------------------------------
@@ -268,6 +323,7 @@ async def _merge_bucket(
groups.setdefault(key, []).append(row)
rows_out: list[dict] = []
+ embedding_texts: list[str] = []
for (name_lower, typ), rows in groups.items():
# Collect all source_chunk_ids, doc_ids, and original-cased names
all_chunks: list[str] = []
@@ -319,13 +375,10 @@ async def _merge_bucket(
row["compilation_template_ids"] = [template_id]
if structure_kind:
row["compilation_template_kind_kwd"] = str(structure_kind)
- # Re-embed
- if embd_mdl:
- vecs = await _encode(embd_mdl, [best_desc])
- if vecs and len(vecs[0]) > 0:
- dim = len(vecs[0])
- row[f"q_{dim}_vec"] = list(vecs[0])
rows_out.append(row)
+ embedding_texts.append(best_desc)
+
+ await _embed_rows(embd_mdl, rows_out, embedding_texts)
return rows_out
@@ -356,6 +409,7 @@ async def _merge_relations(
groups.setdefault(key, []).append(row)
rows_out: list[dict] = []
+ embedding_texts: list[str] = []
for (src, rel_type, tgt), rows in groups.items():
all_chunks: list[str] = []
all_docs: list[str] = []
@@ -397,13 +451,10 @@ async def _merge_relations(
row["compilation_template_ids"] = [template_id]
if structure_kind:
row["compilation_template_kind_kwd"] = str(structure_kind)
- # Re-embed (matching _merge_bucket)
- if embd_mdl:
- vecs = await _encode(embd_mdl, [best_desc])
- if vecs and len(vecs[0]) > 0:
- dim = len(vecs[0])
- row[f"q_{dim}_vec"] = list(vecs[0])
rows_out.append(row)
+ embedding_texts.append(best_desc)
+
+ await _embed_rows(embd_mdl, rows_out, embedding_texts)
return rows_out
@@ -614,6 +665,7 @@ async def _do_build(
structure_kind: str | None,
embd_mdl,
incremental: bool = True,
+ disabled_doc_ids: set[str] | None = None,
) -> bool:
"""Core build logic — read doc_graph rows, bucket, merge, write dataset rows.
@@ -629,6 +681,7 @@ async def _do_build(
markers existed), ``False`` if a store error prevented cleanup.
"""
cleanup_ok = True
+ disabled_doc_ids = disabled_doc_ids or set()
index = _index_name(tenant_id)
@@ -722,6 +775,8 @@ async def _do_build(
if not rows:
break
for row in rows:
+ if str(row.get("doc_id") or "") in disabled_doc_ids:
+ continue
# Time filter (incremental mode)
if ts_cond:
ts = row.get("create_timestamp_flt", 0)
@@ -749,37 +804,25 @@ async def _do_build(
else:
offset += limit
- # Merge entity buckets into dataset rows
- for bi in range(_BUCKET_COUNT):
- if buckets[bi]:
- out = await _merge_bucket(
- tenant_id,
- kb_id,
- compile_kwd,
- template_id,
- buckets[bi],
- embd_mdl,
- structure_kind,
- )
- if out:
- for start in range(0, len(out), _BUCKET_FLUSH_THRESHOLD):
- await _index_insert(tenant_id, kb_id, out[start : start + _BUCKET_FLUSH_THRESHOLD])
-
- # Merge relation buckets into dataset rows
- for bi in range(_BUCKET_COUNT):
- if rel_buckets[bi]:
- rel_out = await _merge_relations(
- tenant_id,
- kb_id,
- compile_kwd,
- template_id,
- rel_buckets[bi],
- embd_mdl,
- structure_kind,
- )
- if rel_out:
- for start in range(0, len(rel_out), _BUCKET_FLUSH_THRESHOLD):
- await _index_insert(tenant_id, kb_id, rel_out[start : start + _BUCKET_FLUSH_THRESHOLD])
+ async def _merge_buckets(bucket_rows, merge_fn):
+ non_empty = [rows for rows in bucket_rows if rows]
+ for start in range(0, len(non_empty), _BUCKET_MERGE_MAX_IN_FLIGHT):
+ merged = await asyncio.gather(*(merge_fn(rows) for rows in non_empty[start : start + _BUCKET_MERGE_MAX_IN_FLIGHT]))
+ for rows in merged:
+ for batch_start in range(0, len(rows), _BUCKET_FLUSH_THRESHOLD):
+ await _index_insert(tenant_id, kb_id, rows[batch_start : batch_start + _BUCKET_FLUSH_THRESHOLD])
+
+ # Merge entity and relation buckets concurrently in bounded groups. Each
+ # bucket embeds its rows in batches, so neither the bucket loop nor the
+ # individual row loop is a serial embedding bottleneck.
+ await _merge_buckets(
+ buckets,
+ lambda rows: _merge_bucket(tenant_id, kb_id, compile_kwd, template_id, rows, embd_mdl, structure_kind),
+ )
+ await _merge_buckets(
+ rel_buckets,
+ lambda rows: _merge_relations(tenant_id, kb_id, compile_kwd, template_id, rows, embd_mdl, structure_kind),
+ )
# ── Save build timestamp ─────────────────────────────────────────
import datetime
@@ -860,6 +903,11 @@ async def run_structure_merge(ctx: TaskContext) -> None:
progress(1.0, f"No eligible templates for {kind_label}.")
return
+ disabled_doc_ids = await _disabled_doc_ids(ctx.kb_id)
+ # A disabled document does not trigger an immediate rebuild. On the next
+ # explicit merge, rebuild from all active source rows so shared entities
+ # and relations are recalculated without the disabled document's data.
+ rebuild_all = bool(disabled_doc_ids)
total = len(eligible)
rebuilt = 0
all_cleanup_ok = True
@@ -869,7 +917,16 @@ async def run_structure_merge(ctx: TaskContext) -> None:
return
progress(0.05 + 0.9 * (i / total), f"Building dataset graph {i + 1}/{total} ...")
try:
- cleanup_ok = await _do_build(ctx.tenant_id, ctx.kb_id, compile_kwd, template_id, structure_kind, embd_mdl)
+ cleanup_ok = await _do_build(
+ ctx.tenant_id,
+ ctx.kb_id,
+ compile_kwd,
+ template_id,
+ structure_kind,
+ embd_mdl,
+ incremental=not rebuild_all,
+ disabled_doc_ids=disabled_doc_ids,
+ )
if not cleanup_ok:
all_cleanup_ok = False
rebuilt += 1
@@ -877,6 +934,10 @@ async def run_structure_merge(ctx: TaskContext) -> None:
all_cleanup_ok = False
logging.exception("structure_merge: build failed for kb=%s compile_kwd=%s", ctx.kb_id, compile_kwd)
+ # Bulk inserts use refresh=False; make the complete merge visible once all
+ # templates have been written instead of waiting on every bulk request.
+ await _refresh_index(ctx.tenant_id, ctx.kb_id)
+
# Delete pending deletion markers only after every pair has been
# processed successfully. If any pair had a store error the markers
# survive so the next run retries the cleanup.