Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(structure-graph): discover buckets from raw rows (#19523)

Sbaaoui Idrissbcf33c6238b6
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.py
index 3865cf942b..a147308238 100644
--- a/api/apps/restful_apis/chunk_api.py
+++ b/api/apps/restful_apis/chunk_api.py
@@ -911,31 +911,39 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):
return get_result(data=_response([bucket]))
# ── normal mode: per-template subgraph sampling from the raw rows ──
- # Metadata-only scan of the per-doc graph blob rows (one per
- # (compile_kwd, template_id)) purely to discover buckets and resolve their
- # display name/kind — WITHOUT loading the (potentially huge)
- # content_with_weight. Each bucket's entities/relations are then fetched
- # from the raw ``knowledge_graph_kwd`` rows with subgraph sampling.
- meta_fields = ["compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- meta_fields,
- [],
- {"doc_id": [document_id], "knowledge_graph_kwd": ["graph"]},
- [],
- OrderByExpr(),
- 0,
- 1000,
- index_name,
- [dataset_id],
- )
- meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}
- except Exception as e:
- return server_error_response(e)
-
- # Discover unique buckets. A template can own multiple compile_kwd blob
- # rows; scoping by template_id folds them together, matching prior behavior.
+ # Discover buckets from the authoritative entity/relation rows. The
+ # structure compiler no longer emits a synthetic ``graph`` row, and a
+ # paged metadata-only scan avoids loading payloads or truncating large docs.
+ meta_fields = ["id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]
+ meta_rows: dict = {}
+ page_size = 1000
+ offset = 0
+ while True:
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ meta_fields,
+ [],
+ {"doc_id": [document_id], "knowledge_graph_kwd": ["entity", "relation"]},
+ [],
+ OrderByExpr(),
+ offset,
+ page_size,
+ index_name,
+ [dataset_id],
+ )
+ page_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}
+ except Exception as e:
+ return server_error_response(e)
+ if not page_rows:
+ break
+ meta_rows.update(page_rows)
+ if len(page_rows) < page_size:
+ break
+ offset += page_size
+
+ # Discover unique buckets. A template can own multiple raw entity/relation
+ # rows; scoping by template_id folds them together.
bucket_metas: dict[str, dict] = {}
bucket_scopes: dict[str, dict] = {}
for row in meta_rows.values():
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.go
index d3af89ad92..db99d62af3 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go
@@ -177,9 +177,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// return common.Outputs{}, err
// }
- // Buffer every product (plus the graph) in one slice; the component merges
- // them into the upstream chunk stream (matching Python, which appends
- // compiled units onto the chunk list).
+ // Buffer the deduplicated row products in one slice; the component merges
+ // them into the upstream chunk stream.
products := append([]common.Product{}, prods...)
// products = append(products, graphProduct)
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go
index d735422cbb..633969b1f9 100644
--- a/internal/service/dataset_structure_graph.go
+++ b/internal/service/dataset_structure_graph.go
@@ -779,14 +779,27 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume
return resp, nil
}
- // normal mode: discover buckets from per-doc graph blob rows. "id" is
- // required for the same reason as dataset discovery (Infinity only projects
- // listed fields; graphRowSearch keys by id).
+ // normal mode: discover buckets from the authoritative entity/relation
+ // rows. The structure compiler no longer emits a synthetic graph row;
+ // page metadata only so large documents cannot hide later buckets.
metaFields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}
- metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,
- map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)
- if err != nil {
- return nil, err
+ metaRows := map[string]map[string]interface{}{}
+ const pageSize = 1000
+ for offset := 0; ; offset += pageSize {
+ page, _, searchErr := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,
+ map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"entity", "relation"}}, nil, offset, pageSize, nil)
+ if searchErr != nil {
+ return nil, searchErr
+ }
+ if len(page) == 0 {
+ break
+ }
+ for id, row := range page {
+ metaRows[id] = row
+ }
+ if len(page) < pageSize {
+ break
+ }
}
bucketMetas := map[string]map[string]interface{}{}
diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go
index 74a767b9e2..9772e44ca4 100644
--- a/internal/service/dataset_structure_graph_test.go
+++ b/internal/service/dataset_structure_graph_test.go
@@ -105,3 +105,24 @@ func TestRowTemplateID_FromList(t *testing.T) {
t.Errorf("rowTemplateID = %q, want tid1", got)
}
}
+
+// TestResolveGraphBucket_UsesRawStructureMetadata covers the post-#19474
+// storage shape: entity/relation rows carry the template stamp, with no
+// synthetic knowledge_graph_kwd="graph" row available for discovery.
+func TestResolveGraphBucket_UsesRawStructureMetadata(t *testing.T) {
+ row := map[string]interface{}{
+ "knowledge_graph_kwd": "entity",
+ "compilation_template_ids": []string{"tree-1"},
+ "compilation_template_kind_kwd": "tree",
+ "compile_kwd": "tree",
+ }
+ meta, scope := resolveGraphBucket(row, map[string]map[string]interface{}{
+ "tree-1": {"template_name": "Tree", "kind": "tree"},
+ }, "doc-1")
+ if meta["template_id"] != "tree-1" || meta["template_name"] != "Tree" {
+ t.Fatalf("meta = %#v, want resolved template metadata", meta)
+ }
+ if got := scope["compilation_template_ids"].([]string); len(got) != 1 || got[0] != "tree-1" {
+ t.Fatalf("scope = %#v, want template-scoped raw-row filter", scope)
+ }
+}