Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pageindex): stop sampling the structure graph and emit rows in source order (#20750)

qinling0210eb57a8bed143
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go
index ffdf08689b..2bc558e3e0 100644
--- a/internal/service/dataset_structure_graph.go
+++ b/internal/service/dataset_structure_graph.go
@@ -16,15 +16,14 @@ import (
"sort"
"strings"
+ "ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
)
-// Structure-graph sampling constants (mirror structure_graph_common.py).
+// Structure-graph query constants.
const (
- graphFullThreshold = 1024 // below this combined count, return all rows
- graphTopEntities = 256 // seed set A size for large buckets
graphKeywordCandidates = 16 // keyword candidate rows
graphExpansionCap = 4096 // hub-node expansion cap
)
@@ -338,6 +337,9 @@ func sortedUnique(in []string) []string {
}
// normalizeRelationEndpoints aligns relation endpoints to returned entity ids.
+// Matching ignores case and internal whitespace, so an endpoint that differs
+// from the entity name only by the space character (e.g. U+0020 vs U+3000) is
+// still aligned instead of being dropped by the exact-name lookup downstream.
func normalizeRelationEndpoints(entities []StructureGraphNode, relations []StructureGraphRelation) []StructureGraphRelation {
if len(entities) == 0 || len(relations) == 0 {
return relations
@@ -354,7 +356,7 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Struc
if !ok || strings.TrimSpace(v) == "" {
continue
}
- key := strings.ToLower(strings.TrimSpace(v))
+ key := normalizeEndpointKey(v)
if cur, exists := lookup[key]; exists && cur != respID {
ambiguous[key] = true
continue
@@ -373,7 +375,7 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Struc
}
for _, f := range []string{"from", "to"} {
if v, ok := item[f].(string); ok {
- if mapped, found := lookup[strings.ToLower(strings.TrimSpace(v))]; found {
+ if mapped, found := lookup[normalizeEndpointKey(v)]; found {
item[f] = mapped
}
}
@@ -383,6 +385,13 @@ func normalizeRelationEndpoints(entities []StructureGraphNode, relations []Struc
return normalized
}
+// normalizeEndpointKey is the case- and whitespace-insensitive match key for
+// entity names / relation endpoints: any run of Unicode spaces (including
+// U+3000) folds to one ASCII space. Mirrors the compiler's normalizedEntityName.
+func normalizeEndpointKey(s string) string {
+ return strings.ToLower(strings.Join(strings.Fields(s), " "))
+}
+
// rowHasEnabledSource mirrors _row_has_enabled_source.
func rowHasEnabledSource(row map[string]interface{}, excludedDocIDs map[string]bool) bool {
if len(excludedDocIDs) == 0 {
@@ -457,132 +466,142 @@ func graphStr(v interface{}) string {
return fmt.Sprintf("%v", v)
}
-// buildBucket mirrors sgc.build_bucket: small buckets whole, large sampled.
+// buildBucket returns every entity/relation row in the bucket, emitted in source
+// (reading) order. The graph is returned whole — no sampling — so a page_index
+// or tree keeps every title and include edge it needs to nest correctly.
func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, datasetID string, scope map[string]interface{}, excludedDocIDs map[string]bool) ([]StructureGraphNode, []StructureGraphRelation, error) {
excludedDocIDs = excludedDocIDsOrEmpty(excludedDocIDs)
bothCond := copyFilter(scope)
bothCond["knowledge_graph_kwd"] = []string{"entity", "relation"}
+
+ // Count, then fetch the whole bucket in one request at the exact limit. That
+ // lets the engine walk the result set in a single cursor pass (search_after
+ // past index.max_result_window) instead of re-skipping from the start on
+ // every offset page. Asc("id") is the stable cursor search_after needs.
_, total, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id"}, bothCond, nil, 0, 1, nil)
if err != nil {
return nil, nil, err
}
- if total < graphFullThreshold {
- fieldMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, nil, 0, int(total), nil)
+ fieldMap := map[string]map[string]interface{}{}
+ if total > 0 {
+ orderBy := (&types.OrderByExpr{}).Asc("id")
+ fieldMap, _, err = graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, orderBy, 0, int(total), nil)
if err != nil {
return nil, nil, err
}
- // Initialize as empty (non-nil) slices so a bucket with entities but no
- // relations serializes "relations": [] instead of null — the frontend
- // adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on
- // relations without a null guard (mirror Python, which always returns []).
- entities := make([]StructureGraphNode, 0)
- relations := make([]StructureGraphRelation, 0)
- for _, row := range fieldMap {
- if !rowHasEnabledSource(row, excludedDocIDs) {
- continue
- }
- kg := firstStringValue(row["type_kwd"])
- if kg == "" {
- kg = firstStringValue(row["knowledge_graph_kwd"])
- }
- if kg == "relation" {
- if edge := projectRelation(row); edge != nil {
- relations = append(relations, edge)
- }
- } else {
- if node := projectEntity(row); node != nil {
- entities = append(entities, node)
- }
- }
- }
- entities = dedupEntities(entities)
- relations = normalizeRelationEndpoints(entities, relations)
- return entities, relations, nil
}
- // Large bucket: sample. A = top entities by mention_count_int desc.
- orderBy := (&types.OrderByExpr{}).Desc("mention_count_int")
- var setA []StructureGraphNode
- entityOffset := 0
- var entityTotal int64 = -1
- for len(setA) < graphTopEntities && (entityTotal == -1 || int64(entityOffset) < entityTotal) {
- cond := copyFilter(scope)
- cond["knowledge_graph_kwd"] = []string{"entity"}
- entAMap, entTotal, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, orderBy, entityOffset, graphTopEntities, nil)
- if err != nil {
- return nil, nil, err
+ // fieldMap is keyed by id (the engine returns rows in a map), so ranging it
+ // would discard the requested Asc("id") order. Walk the ids in sorted order
+ // to fix a deterministic base: the stable source-position sort below then
+ // orders by reading position, and equal/absent positions keep THIS id order
+ // instead of map iteration order.
+ ids := make([]string, 0, len(fieldMap))
+ for id := range fieldMap {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ rows := make([]map[string]interface{}, 0, len(fieldMap))
+ for _, id := range ids {
+ row := fieldMap[id]
+ if !rowHasEnabledSource(row, excludedDocIDs) {
+ continue
}
- entityTotal = entTotal
- if len(entAMap) == 0 {
- break
+ rows = append(rows, row)
+ }
+ sourceOrder, err := s.structureSourceOrder(ctx, tenantID, datasetID, rows)
+ if err != nil {
+ // Ordering is cosmetic; never fail the graph read on the auxiliary lookup.
+ common.Error("structure graph: source-order lookup failed", err)
+ sourceOrder = nil
+ }
+ sort.SliceStable(rows, func(i, j int) bool {
+ oi, oki := structureRowSourceOrder(rows[i], sourceOrder)
+ oj, okj := structureRowSourceOrder(rows[j], sourceOrder)
+ if oki != okj {
+ return oki
}
- for _, row := range entAMap {
- if !rowHasEnabledSource(row, excludedDocIDs) {
- continue
+ if oki && oi != oj {
+ return oi < oj
+ }
+ return false
+ })
+
+ // Initialize as empty (non-nil) slices so a bucket with entities but no
+ // relations serializes "relations": [] instead of null — the frontend
+ // adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on
+ // relations without a null guard (mirror Python, which always returns []).
+ entities := make([]StructureGraphNode, 0, len(rows))
+ relations := make([]StructureGraphRelation, 0, len(rows))
+ for _, row := range rows {
+ kg := firstStringValue(row["type_kwd"])
+ if kg == "" {
+ kg = firstStringValue(row["knowledge_graph_kwd"])
+ }
+ if kg == "relation" {
+ if edge := projectRelation(row); edge != nil {
+ relations = append(relations, edge)
}
- if n := projectEntity(row); n != nil {
- setA = append(setA, n)
+ } else {
+ if node := projectEntity(row); node != nil {
+ entities = append(entities, node)
}
}
- entityOffset += len(entAMap)
}
- if len(setA) > graphTopEntities {
- setA = setA[:graphTopEntities]
- }
- var aNames []string
- for _, e := range setA {
- if n := strings.TrimSpace(graphStr(e["name"])); n != "" {
- aNames = append(aNames, n)
+ entities = dedupEntities(entities)
+ relations = normalizeRelationEndpoints(entities, relations)
+ return entities, relations, nil
+}
+
+// structureSourceOrder maps each referenced source chunk id to its reading order
+// (chunk_order_int), so buildBucket can emit rows in source order. Unresolvable
+// ids are absent from the returned map (nil when the bucket references none).
+func (s *DatasetArtifactService) structureSourceOrder(ctx context.Context, tenantID, datasetID string, rows []map[string]interface{}) (map[string]int, error) {
+ ids := make([]string, 0)
+ seen := map[string]bool{}
+ for _, row := range rows {
+ for _, id := range graphSourceChunkIDs(nil, row) {
+ if id != "" && !seen[id] {
+ seen[id] = true
+ ids = append(ids, id)
+ }
}
}
- var aNameTerms []string
- for _, name := range aNames {
- aNameTerms = append(aNameTerms, endpointTerms(name)...)
+ if len(ids) == 0 {
+ return nil, nil
}
- aNameTerms = sortedUnique(aNameTerms)
-
- relations := make([]StructureGraphRelation, 0)
- targetNamesLower := map[string]bool{}
- if len(aNameTerms) > 0 {
- cond := copyFilter(scope)
- cond["knowledge_graph_kwd"] = []string{"relation"}
- cond["from_entity_kwd"] = aNameTerms
- relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)
- if err != nil {
- return nil, nil, err
- }
- for _, row := range relMap {
- if !rowHasEnabledSource(row, excludedDocIDs) {
- continue
- }
- if edge := projectRelation(row); edge != nil {
- relations = append(relations, edge)
- if tgt := strings.ToLower(strings.TrimSpace(graphStr(edge["to"]))); tgt != "" {
- targetNamesLower[tgt] = true
- }
- }
+ // One fetch at the exact id count; Asc("id") keeps the engine's cursor
+ // pagination (search_after) available past index.max_result_window.
+ page, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id", "chunk_order_int"},
+ map[string]interface{}{"id": ids}, (&types.OrderByExpr{}).Asc("id"), 0, len(ids), nil)
+ if err != nil {
+ return nil, err
+ }
+ order := make(map[string]int, len(page))
+ for id, row := range page {
+ // intValue cannot distinguish an absent field from 0, so gate on
+ // presence: chunk 0 is a valid reading order.
+ if raw, ok := row["chunk_order_int"]; ok && raw != nil {
+ order[id] = intValue(raw)
}
}
- var setT []StructureGraphNode
- if len(targetNamesLower) > 0 {
- cond := copyFilter(scope)
- cond["knowledge_graph_kwd"] = []string{"entity"}
- cond["name_kwd"] = sortedKeys(targetNamesLower)
- tgtMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)
- if err != nil {
- return nil, nil, err
+ return order, nil
+}
+
+// structureRowSourceOrder is a row's source position: the smallest
+// chunk_order_int among its source_chunk_ids (false when none resolve).
+func structureRowSourceOrder(row map[string]interface{}, order map[string]int) (int, bool) {
+ best, found := 0, false
+ for _, id := range graphSourceChunkIDs(nil, row) {
+ v, ok := order[id]
+ if !ok {
+ continue
}
- for _, row := range tgtMap {
- if !rowHasEnabledSource(row, excludedDocIDs) {
- continue
- }
- if n := projectEntity(row); n != nil {
- setT = append(setT, n)
- }
+ if !found || v < best {
+ best, found = v, true
}
}
- entities := dedupEntities(append(setA, setT...))
- return entities, normalizeRelationEndpoints(entities, relations), nil
+ return best, found
}
func copyFilter(in map[string]interface{}) map[string]interface{} {
diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go
index 07a3379e55..3c6a58cf6e 100644
--- a/internal/service/dataset_structure_graph_test.go
+++ b/internal/service/dataset_structure_graph_test.go
@@ -141,3 +141,34 @@ func TestBuildDocumentGraphTemplateShells_PreservesTemplateMetadata(t *testing.T
t.Fatal("empty template collections must be non-nil")
}
}
+
+// TestStructureRowSourceOrder_MinChunkOrder verifies a row's source position is
+// the smallest chunk_order_int among its source_chunk_ids, and that a row with
+// no resolvable chunks reports ok=false (so it sorts to the end).
+func TestStructureRowSourceOrder_MinChunkOrder(t *testing.T) {
+ order := map[string]int{"c1": 5, "c2": 2, "c3": 9}
+ row := map[string]interface{}{"source_chunk_ids": []interface{}{"c1", "c2", "c3"}}
+ got, ok := structureRowSourceOrder(row, order)
+ if !ok || got != 2 {
+ t.Fatalf("structureRowSourceOrder = (%d,%v), want (2,true)", got, ok)
+ }
+ if _, ok := structureRowSourceOrder(map[string]interface{}{"source_chunk_ids": []string{"zz"}}, order); ok {
+ t.Fatal("expected ok=false when no source chunk resolves")
+ }
+}
+
+// TestNormalizeRelationEndpoints_WhitespaceInsensitive verifies an endpoint that
+// differs from the entity name only by the space character (half-width U+0020
+// vs ideographic U+3000) is still aligned to the entity, so the exact-name map
+// lookup downstream (frontend tree building) matches it.
+func TestNormalizeRelationEndpoints_WhitespaceInsensitive(t *testing.T) {
+ entities := []StructureGraphNode{{"name": "第一章 文忠公", "type": "title"}}
+ relations := []StructureGraphRelation{{"from": "第一章\u3000文忠公", "to": "某句", "type": "include"}}
+ out := normalizeRelationEndpoints(entities, relations)
+ if out[0]["from"] != "第一章 文忠公" {
+ t.Errorf("from = %q, want the entity name 第一章 文忠公", out[0]["from"])
+ }
+ if out[0]["to"] != "某句" {
+ t.Errorf("to = %q, want unchanged when no entity matches", out[0]["to"])
+ }
+}