Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(knowledge_compile): preserve structure merge metadata (#20858)

lorenzozane738df90a546a
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 279c65371d..3d128bf0d7 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -1599,6 +1599,8 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
from := metaString(p.Meta, "from")
to := metaString(p.Meta, "to")
if kind == "relation" || (from != "" && to != "") {
+ from = strings.TrimSpace(from)
+ to = strings.TrimSpace(to)
// Skip a relation with an incomplete endpoint pair: writing a bucket
// like "A -> " would let multiple malformed relations collapse into
// the same dataset row.
@@ -1640,6 +1642,15 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
}
appendBucket(b, p)
}
+ for key, bucket := range relByKey {
+ if entity := entByKey[ekey{template: key.template, name: key.from, ckwd: key.ckwd}]; entity != nil {
+ bucket.FromEntity = entity.Name
+ }
+ if entity := entByKey[ekey{template: key.template, name: key.to, ckwd: key.ckwd}]; entity != nil {
+ bucket.ToEntity = entity.Name
+ }
+ bucket.Name = bucket.FromEntity + " -> " + bucket.ToEntity
+ }
buckets := make([]StructureBucket, 0, len(entByKey)+len(relByKey))
for _, b := range entByKey {
buckets = append(buckets, *b)
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index fcd401fce6..d1817ce150 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"strconv"
+ "strings"
"go.uber.org/zap"
@@ -356,6 +357,16 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
merged := isAvailable(c["available_int"])
meta := map[string]any{}
+ var payload map[string]any
+ if expect == kccommon.VariantStructure || expect == kccommon.VariantMindmap {
+ if err := json.Unmarshal([]byte(content), &payload); err != nil {
+ if strings.HasPrefix(strings.TrimSpace(content), "{") {
+ common.Warn("knowledge_compile: failed to decode structure payload",
+ zap.String("row_id", id), zap.String("compile_kwd", variant), zap.Error(err))
+ }
+ payload = nil
+ }
+ }
if raw := asString(c["extra"]); raw != "" {
var extra map[string]any
if err := json.Unmarshal([]byte(raw), &extra); err != nil {
@@ -372,7 +383,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
}
}
if v, ok := c["name_kwd"].(string); ok && v != "" {
- meta["name"] = v
+ name := v
+ for _, key := range []string{"name", "text", "term", "title"} {
+ if extracted, ok := payload[key].(string); ok && extracted != "" && normalizedStructureEntityName(extracted) == normalizedStructureEntityName(v) {
+ name = strings.TrimSpace(extracted)
+ break
+ }
+ }
+ meta["name"] = name
}
if v, ok := c["entity_type_kwd"].(string); ok && v != "" {
if expect == kccommon.VariantMindmap && v == "mindmap" {
@@ -388,6 +406,11 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
meta["to"] = v
meta["kind"] = "relation"
}
+ if metaString(meta, "kind") == "relation" {
+ if v, ok := payload["type"].(string); ok && v != "" {
+ meta["relation_type"] = v
+ }
+ }
if v, ok := metaInt(c, "mention_count_int"); ok {
meta["mention_count"] = v
}
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index 33e7c839e0..3e26e8276e 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -403,3 +403,65 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {
t.Errorf("tree Meta.kind = %q, want root (so nav dispatch can pick it)", kind)
}
}
+
+func TestProductFromChunkMap_PreservesStructurePayloadFieldsThroughMerge(t *testing.T) {
+ rows := []map[string]interface{}{
+ {
+ "id": "entity-acme", "doc_id": "d1", "compile_kwd": "knowledge_graph",
+ "content_with_weight": `{"name":" Acme Corp ","type":"organization","description":"Supplier"}`,
+ "knowledge_graph_kwd": "graph", "type_kwd": "entity", "name_kwd": "acme corp",
+ },
+ {
+ "id": "entity-beta", "doc_id": "d1", "compile_kwd": "knowledge_graph",
+ "content_with_weight": `{"text":"Beta Ltd","type":"organization","description":"Vendor"}`,
+ "knowledge_graph_kwd": "graph", "type_kwd": "entity", "name_kwd": "beta ltd",
+ },
+ {
+ "id": "relation-supplies", "doc_id": "d1", "compile_kwd": "knowledge_graph",
+ "content_with_weight": `{"from":"Acme Corp","to":"Beta Ltd","type":"supplies","description":"supply"}`,
+ "knowledge_graph_kwd": "graph", "type_kwd": "relation",
+ "from_entity_kwd": " ACME CORP ", "to_entity_kwd": "BETA LTD ",
+ },
+ {
+ "id": "relation-owns", "doc_id": "d1", "compile_kwd": "knowledge_graph",
+ "content_with_weight": `{"from":"Acme Corp","to":"Beta Ltd","type":"owns","description":"ownership"}`,
+ "knowledge_graph_kwd": "graph", "type_kwd": "relation",
+ "from_entity_kwd": " ACME CORP ", "to_entity_kwd": "BETA LTD ",
+ },
+ }
+ products := make([]kccommon.Product, 0, len(rows))
+ for _, row := range rows {
+ product, ok := productFromChunkMap(row, "t1", kccommon.VariantStructure)
+ if !ok {
+ t.Fatalf("row %q should reconstruct", row["id"])
+ }
+ products = append(products, product)
+ }
+
+ c := &Consumer{writer: &fakeWriter{}}
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ buckets := c.writer.(*fakeWriter).buckets
+ if len(buckets) != 4 {
+ t.Fatalf("want 2 entities and 2 typed relations, got %d: %+v", len(buckets), buckets)
+ }
+ entities := map[string]bool{}
+ relations := map[string]bool{}
+ for _, bucket := range buckets {
+ if bucket.Type == "relation" {
+ relations[bucket.RelationType] = true
+ if bucket.FromEntity != "Acme Corp" || bucket.ToEntity != "Beta Ltd" {
+ t.Errorf("relation endpoints = %q -> %q, want trimmed entity spellings", bucket.FromEntity, bucket.ToEntity)
+ }
+ continue
+ }
+ entities[bucket.Name] = true
+ }
+ if !entities["Acme Corp"] || !entities["Beta Ltd"] {
+ t.Errorf("entity display names = %v, want extracted spelling", entities)
+ }
+ if !relations["supplies"] || !relations["owns"] {
+ t.Errorf("relation types = %v, want supplies and owns", relations)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 9b96e5b057..60dabda8cc 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -561,7 +561,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
row["to_entity_kwd"] = b.ToEntity
} else {
row["type_kwd"] = "entity"
- row["name_kwd"] = b.Name
+ row["name_kwd"] = strings.ToLower(b.Name)
row["entity_type_kwd"] = b.Type
if b.MentionCount > 0 {
row["mention_count_int"] = b.MentionCount
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 47ac4bb258..49373befbe 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -237,6 +237,57 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
}
}
+func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ if err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{
+ Name: "Acme Corp", Type: "organization", Description: "company", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph",
+ }}); err != nil {
+ t.Fatalf("WriteMergedStructure: %v", err)
+ }
+ for _, row := range eng.insertedChunks {
+ if row["type_kwd"] != "entity" {
+ continue
+ }
+ if row["name_kwd"] != "acme corp" {
+ t.Errorf("name_kwd = %v, want normalized keyword acme corp", row["name_kwd"])
+ }
+ payload, _ := row["content_with_weight"].(string)
+ if !strings.Contains(payload, `"name":"Acme Corp"`) {
+ t.Errorf("entity payload = %s, want extracted spelling Acme Corp", payload)
+ }
+ return
+ }
+ t.Fatal("inserted rows contain no entity")
+}
+
+func TestWriteMergedStructurePreservesDistinctRelationTypes(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ buckets := []StructureBucket{
+ {Name: "Acme Corp -> Beta Ltd", Type: "relation", FromEntity: "Acme Corp", ToEntity: "Beta Ltd", RelationType: "supplies", Description: "supply", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph"},
+ {Name: "Acme Corp -> Beta Ltd", Type: "relation", FromEntity: "Acme Corp", ToEntity: "Beta Ltd", RelationType: "owns", Description: "ownership", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph"},
+ }
+ if err := w.WriteMergedStructure(context.Background(), "t1", "kb1", buckets); err != nil {
+ t.Fatalf("WriteMergedStructure: %v", err)
+ }
+ relations := map[string]bool{}
+ for _, row := range eng.insertedChunks {
+ if row["type_kwd"] != "relation" {
+ continue
+ }
+ payload, _ := row["content_with_weight"].(string)
+ for _, relationType := range []string{"supplies", "owns"} {
+ if strings.Contains(payload, `"type":"`+relationType+`"`) {
+ relations[relationType] = true
+ }
+ }
+ }
+ if len(relations) != 2 || !relations["supplies"] || !relations["owns"] {
+ t.Errorf("stored relation types = %v, want separate supplies and owns rows", relations)
+ }
+}
+
func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {
oldID := "dataset_structure_old"
eng := &fakeEngine{searchChunks: []map[string]interface{}{