RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pageindex): stop sampling the structure graph and emit rows in source order (#20750)
qinling0210eb57a8bed143
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex ffdf08689b..2bc558e3e0 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -16,15 +16,14 @@ import ("sort""strings"+ "ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types")-// Structure-graph sampling constants (mirror structure_graph_common.py).+// Structure-graph query constants.const (- graphFullThreshold = 1024 // below this combined count, return all rows- graphTopEntities = 256 // seed set A size for large bucketsgraphKeywordCandidates = 16 // keyword candidate rowsgraphExpansionCap = 4096 // hub-node expansion cap)@@ -338,6 +337,9 @@ func sortedUnique(in []string) []string {}// normalizeRelationEndpoints aligns relation endpoints to returned entity ids.+// Matching ignores case and internal whitespace, so an endpoint that differs+// from the entity name only by the space character (e.g. U+0020 vs U+3000) is+// still aligned instead of being dropped by the exact-name lookup downstream.func normalizeRelationEndpoints(entities []StructureGraphNode, relations []StructureGraphRelation) []StructureGraphRelation {if len(entities) == 0 || len(relations) == 0 {return relations@@ -354,7 +356,7 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Strucif !ok || strings.TrimSpace(v) == "" {continue}- key := strings.ToLower(strings.TrimSpace(v))+ key := normalizeEndpointKey(v)if cur, exists := lookup[key]; exists && cur != respID {ambiguous[key] = truecontinue@@ -373,7 +375,7 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Struc}for _, f := range []string{"from", "to"} {if v, ok := item[f].(string); ok {- if mapped, found := lookup[strings.ToLower(strings.TrimSpace(v))]; found {+ if mapped, found := lookup[normalizeEndpointKey(v)]; found {item[f] = mapped}}@@ -383,6 +385,13 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Strucreturn normalized}+// normalizeEndpointKey is the case- and whitespace-insensitive match key for+// entity names / relation endpoints: any run of Unicode spaces (including+// U+3000) folds to one ASCII space. Mirrors the compiler's normalizedEntityName.+func normalizeEndpointKey(s string) string {+ return strings.ToLower(strings.Join(strings.Fields(s), " "))+}+// rowHasEnabledSource mirrors _row_has_enabled_source.func rowHasEnabledSource(row map[string]interface{}, excludedDocIDs map[string]bool) bool {if len(excludedDocIDs) == 0 {@@ -457,132 +466,142 @@ func graphStr(v interface{}) string {return fmt.Sprintf("%v", v)}-// buildBucket mirrors sgc.build_bucket: small buckets whole, large sampled.+// buildBucket returns every entity/relation row in the bucket, emitted in source+// (reading) order. The graph is returned whole — no sampling — so a page_index+// or tree keeps every title and include edge it needs to nest correctly.func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, datasetID string, scope map[string]interface{}, excludedDocIDs map[string]bool) ([]StructureGraphNode, []StructureGraphRelation, error) {excludedDocIDs = excludedDocIDsOrEmpty(excludedDocIDs)bothCond := copyFilter(scope)bothCond["knowledge_graph_kwd"] = []string{"entity", "relation"}++ // Count, then fetch the whole bucket in one request at the exact limit. That+ // lets the engine walk the result set in a single cursor pass (search_after+ // past index.max_result_window) instead of re-skipping from the start on+ // every offset page. Asc("id") is the stable cursor search_after needs._, total, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id"}, bothCond, nil, 0, 1, nil)if err != nil {return nil, nil, err}- if total < graphFullThreshold {- fieldMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, nil, 0, int(total), nil)+ fieldMap := map[string]map[string]interface{}{}+ if total > 0 {+ orderBy := (&types.OrderByExpr{}).Asc("id")+ fieldMap, _, err = graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, orderBy, 0, int(total), nil)if err != nil {return nil, nil, err}- // Initialize as empty (non-nil) slices so a bucket with entities but no- // relations serializes "relations": [] instead of null — the frontend- // adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on- // relations without a null guard (mirror Python, which always returns []).- entities := make([]StructureGraphNode, 0)- relations := make([]StructureGraphRelation, 0)- for _, row := range fieldMap {- if !rowHasEnabledSource(row, excludedDocIDs) {- continue- }- kg := firstStringValue(row["type_kwd"])- if kg == "" {- kg = firstStringValue(row["knowledge_graph_kwd"])- }- if kg == "relation" {- if edge := projectRelation(row); edge != nil {- relations = append(relations, edge)- }- } else {- if node := projectEntity(row); node != nil {- entities = append(entities, node)- }- }- }- entities = dedupEntities(entities)- relations = normalizeRelationEndpoints(entities, relations)- return entities, relations, nil}- // Large bucket: sample. A = top entities by mention_count_int desc.- orderBy := (&types.OrderByExpr{}).Desc("mention_count_int")- var setA []StructureGraphNode- entityOffset := 0- var entityTotal int64 = -1- for len(setA) < graphTopEntities && (entityTotal == -1 || int64(entityOffset) < entityTotal) {- cond := copyFilter(scope)- cond["knowledge_graph_kwd"] = []string{"entity"}- entAMap, entTotal, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, orderBy, entityOffset, graphTopEntities, nil)- if err != nil {- return nil, nil, err+ // fieldMap is keyed by id (the engine returns rows in a map), so ranging it+ // would discard the requested Asc("id") order. Walk the ids in sorted order+ // to fix a deterministic base: the stable source-position sort below then+ // orders by reading position, and equal/absent positions keep THIS id order+ // instead of map iteration order.+ ids := make([]string, 0, len(fieldMap))+ for id := range fieldMap {+ ids = append(ids, id)+ }+ sort.Strings(ids)+ rows := make([]map[string]interface{}, 0, len(fieldMap))+ for _, id := range ids {+ row := fieldMap[id]+ if !rowHasEnabledSource(row, excludedDocIDs) {+ continue}- entityTotal = entTotal- if len(entAMap) == 0 {- break+ rows = append(rows, row)+ }+ sourceOrder, err := s.structureSourceOrder(ctx, tenantID, datasetID, rows)+ if err != nil {+ // Ordering is cosmetic; never fail the graph read on the auxiliary lookup.+ common.Error("structure graph: source-order lookup failed", err)+ sourceOrder = nil+ }+ sort.SliceStable(rows, func(i, j int) bool {+ oi, oki := structureRowSourceOrder(rows[i], sourceOrder)+ oj, okj := structureRowSourceOrder(rows[j], sourceOrder)+ if oki != okj {+ return oki}- for _, row := range entAMap {- if !rowHasEnabledSource(row, excludedDocIDs) {- continue+ if oki && oi != oj {+ return oi < oj+ }+ return false+ })++ // Initialize as empty (non-nil) slices so a bucket with entities but no+ // relations serializes "relations": [] instead of null — the frontend+ // adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on+ // relations without a null guard (mirror Python, which always returns []).+ entities := make([]StructureGraphNode, 0, len(rows))+ relations := make([]StructureGraphRelation, 0, len(rows))+ for _, row := range rows {+ kg := firstStringValue(row["type_kwd"])+ if kg == "" {+ kg = firstStringValue(row["knowledge_graph_kwd"])+ }+ if kg == "relation" {+ if edge := projectRelation(row); edge != nil {+ relations = append(relations, edge)}- if n := projectEntity(row); n != nil {- setA = append(setA, n)+ } else {+ if node := projectEntity(row); node != nil {+ entities = append(entities, node)}}- entityOffset += len(entAMap)}- if len(setA) > graphTopEntities {- setA = setA[:graphTopEntities]- }- var aNames []string- for _, e := range setA {- if n := strings.TrimSpace(graphStr(e["name"])); n != "" {- aNames = append(aNames, n)+ entities = dedupEntities(entities)+ relations = normalizeRelationEndpoints(entities, relations)+ return entities, relations, nil+}++// structureSourceOrder maps each referenced source chunk id to its reading order+// (chunk_order_int), so buildBucket can emit rows in source order. Unresolvable+// ids are absent from the returned map (nil when the bucket references none).+func (s *DatasetArtifactService) structureSourceOrder(ctx context.Context, tenantID, datasetID string, rows []map[string]interface{}) (map[string]int, error) {+ ids := make([]string, 0)+ seen := map[string]bool{}+ for _, row := range rows {+ for _, id := range graphSourceChunkIDs(nil, row) {+ if id != "" && !seen[id] {+ seen[id] = true+ ids = append(ids, id)+ }}}- var aNameTerms []string- for _, name := range aNames {- aNameTerms = append(aNameTerms, endpointTerms(name)...)+ if len(ids) == 0 {+ return nil, nil}- aNameTerms = sortedUnique(aNameTerms)-- relations := make([]StructureGraphRelation, 0)- targetNamesLower := map[string]bool{}- if len(aNameTerms) > 0 {- cond := copyFilter(scope)- cond["knowledge_graph_kwd"] = []string{"relation"}- cond["from_entity_kwd"] = aNameTerms- relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)- if err != nil {- return nil, nil, err- }- for _, row := range relMap {- if !rowHasEnabledSource(row, excludedDocIDs) {- continue- }- if edge := projectRelation(row); edge != nil {- relations = append(relations, edge)- if tgt := strings.ToLower(strings.TrimSpace(graphStr(edge["to"]))); tgt != "" {- targetNamesLower[tgt] = true- }- }+ // One fetch at the exact id count; Asc("id") keeps the engine's cursor+ // pagination (search_after) available past index.max_result_window.+ page, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id", "chunk_order_int"},+ map[string]interface{}{"id": ids}, (&types.OrderByExpr{}).Asc("id"), 0, len(ids), nil)+ if err != nil {+ return nil, err+ }+ order := make(map[string]int, len(page))+ for id, row := range page {+ // intValue cannot distinguish an absent field from 0, so gate on+ // presence: chunk 0 is a valid reading order.+ if raw, ok := row["chunk_order_int"]; ok && raw != nil {+ order[id] = intValue(raw)}}- var setT []StructureGraphNode- if len(targetNamesLower) > 0 {- cond := copyFilter(scope)- cond["knowledge_graph_kwd"] = []string{"entity"}- cond["name_kwd"] = sortedKeys(targetNamesLower)- tgtMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)- if err != nil {- return nil, nil, err+ return order, nil+}++// structureRowSourceOrder is a row's source position: the smallest+// chunk_order_int among its source_chunk_ids (false when none resolve).+func structureRowSourceOrder(row map[string]interface{}, order map[string]int) (int, bool) {+ best, found := 0, false+ for _, id := range graphSourceChunkIDs(nil, row) {+ v, ok := order[id]+ if !ok {+ continue}- for _, row := range tgtMap {- if !rowHasEnabledSource(row, excludedDocIDs) {- continue- }- if n := projectEntity(row); n != nil {- setT = append(setT, n)- }+ if !found || v < best {+ best, found = v, true}}- entities := dedupEntities(append(setA, setT...))- return entities, normalizeRelationEndpoints(entities, relations), nil+ return best, found}func copyFilter(in map[string]interface{}) map[string]interface{} {diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.goindex 07a3379e55..3c6a58cf6e 100644--- a/internal/service/dataset_structure_graph_test.go+++ b/internal/service/dataset_structure_graph_test.go@@ -141,3 +141,34 @@ func TestBuildDocumentGraphTemplateShells_PreservesTemplateMetadata(t *testing.Tt.Fatal("empty template collections must be non-nil")}}++// TestStructureRowSourceOrder_MinChunkOrder verifies a row's source position is+// the smallest chunk_order_int among its source_chunk_ids, and that a row with+// no resolvable chunks reports ok=false (so it sorts to the end).+func TestStructureRowSourceOrder_MinChunkOrder(t *testing.T) {+ order := map[string]int{"c1": 5, "c2": 2, "c3": 9}+ row := map[string]interface{}{"source_chunk_ids": []interface{}{"c1", "c2", "c3"}}+ got, ok := structureRowSourceOrder(row, order)+ if !ok || got != 2 {+ t.Fatalf("structureRowSourceOrder = (%d,%v), want (2,true)", got, ok)+ }+ if _, ok := structureRowSourceOrder(map[string]interface{}{"source_chunk_ids": []string{"zz"}}, order); ok {+ t.Fatal("expected ok=false when no source chunk resolves")+ }+}++// TestNormalizeRelationEndpoints_WhitespaceInsensitive verifies an endpoint that+// differs from the entity name only by the space character (half-width U+0020+// vs ideographic U+3000) is still aligned to the entity, so the exact-name map+// lookup downstream (frontend tree building) matches it.+func TestNormalizeRelationEndpoints_WhitespaceInsensitive(t *testing.T) {+ entities := []StructureGraphNode{{"name": "第一章 文忠公", "type": "title"}}+ relations := []StructureGraphRelation{{"from": "第一章\u3000文忠公", "to": "某句", "type": "include"}}+ out := normalizeRelationEndpoints(entities, relations)+ if out[0]["from"] != "第一章 文忠公" {+ t.Errorf("from = %q, want the entity name 第一章 文忠公", out[0]["from"])+ }+ if out[0]["to"] != "某句" {+ t.Errorf("to = %q, want unchanged when no entity matches", out[0]["to"])+ }+}