RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(structure-graph): discover buckets from raw rows (#19523)
Sbaaoui Idrissbcf33c6238b6
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex 3865cf942b..a147308238 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -911,31 +911,39 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):return get_result(data=_response([bucket]))# ── normal mode: per-template subgraph sampling from the raw rows ──- # Metadata-only scan of the per-doc graph blob rows (one per- # (compile_kwd, template_id)) purely to discover buckets and resolve their- # display name/kind — WITHOUT loading the (potentially huge)- # content_with_weight. Each bucket's entities/relations are then fetched- # from the raw ``knowledge_graph_kwd`` rows with subgraph sampling.- meta_fields = ["compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- meta_fields,- [],- {"doc_id": [document_id], "knowledge_graph_kwd": ["graph"]},- [],- OrderByExpr(),- 0,- 1000,- index_name,- [dataset_id],- )- meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}- except Exception as e:- return server_error_response(e)-- # Discover unique buckets. A template can own multiple compile_kwd blob- # rows; scoping by template_id folds them together, matching prior behavior.+ # Discover buckets from the authoritative entity/relation rows. The+ # structure compiler no longer emits a synthetic ``graph`` row, and a+ # paged metadata-only scan avoids loading payloads or truncating large docs.+ meta_fields = ["id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]+ meta_rows: dict = {}+ page_size = 1000+ offset = 0+ while True:+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ meta_fields,+ [],+ {"doc_id": [document_id], "knowledge_graph_kwd": ["entity", "relation"]},+ [],+ OrderByExpr(),+ offset,+ page_size,+ index_name,+ [dataset_id],+ )+ page_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}+ except Exception as e:+ return server_error_response(e)+ if not page_rows:+ break+ meta_rows.update(page_rows)+ if len(page_rows) < page_size:+ break+ offset += page_size++ # Discover unique buckets. A template can own multiple raw entity/relation+ # rows; scoping by template_id folds them together.bucket_metas: dict[str, dict] = {}bucket_scopes: dict[str, dict] = {}for row in meta_rows.values():diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.goindex d3af89ad92..db99d62af3 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go@@ -177,9 +177,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// return common.Outputs{}, err// }- // Buffer every product (plus the graph) in one slice; the component merges- // them into the upstream chunk stream (matching Python, which appends- // compiled units onto the chunk list).+ // Buffer the deduplicated row products in one slice; the component merges+ // them into the upstream chunk stream.products := append([]common.Product{}, prods...)// products = append(products, graphProduct)diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex d735422cbb..633969b1f9 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -779,14 +779,27 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Documereturn resp, nil}- // normal mode: discover buckets from per-doc graph blob rows. "id" is- // required for the same reason as dataset discovery (Infinity only projects- // listed fields; graphRowSearch keys by id).+ // normal mode: discover buckets from the authoritative entity/relation+ // rows. The structure compiler no longer emits a synthetic graph row;+ // page metadata only so large documents cannot hide later buckets.metaFields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}- metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,- map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)- if err != nil {- return nil, err+ metaRows := map[string]map[string]interface{}{}+ const pageSize = 1000+ for offset := 0; ; offset += pageSize {+ page, _, searchErr := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,+ map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"entity", "relation"}}, nil, offset, pageSize, nil)+ if searchErr != nil {+ return nil, searchErr+ }+ if len(page) == 0 {+ break+ }+ for id, row := range page {+ metaRows[id] = row+ }+ if len(page) < pageSize {+ break+ }}bucketMetas := map[string]map[string]interface{}{}diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.goindex 74a767b9e2..9772e44ca4 100644--- a/internal/service/dataset_structure_graph_test.go+++ b/internal/service/dataset_structure_graph_test.go@@ -105,3 +105,24 @@ func TestRowTemplateID_FromList(t *testing.T) {t.Errorf("rowTemplateID = %q, want tid1", got)}}++// TestResolveGraphBucket_UsesRawStructureMetadata covers the post-#19474+// storage shape: entity/relation rows carry the template stamp, with no+// synthetic knowledge_graph_kwd="graph" row available for discovery.+func TestResolveGraphBucket_UsesRawStructureMetadata(t *testing.T) {+ row := map[string]interface{}{+ "knowledge_graph_kwd": "entity",+ "compilation_template_ids": []string{"tree-1"},+ "compilation_template_kind_kwd": "tree",+ "compile_kwd": "tree",+ }+ meta, scope := resolveGraphBucket(row, map[string]map[string]interface{}{+ "tree-1": {"template_name": "Tree", "kind": "tree"},+ }, "doc-1")+ if meta["template_id"] != "tree-1" || meta["template_name"] != "Tree" {+ t.Fatalf("meta = %#v, want resolved template metadata", meta)+ }+ if got := scope["compilation_template_ids"].([]string); len(got) != 1 || got[0] != "tree-1" {+ t.Fatalf("scope = %#v, want template-scoped raw-row filter", scope)+ }+}