RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(knowledge_compile): preserve structure merge metadata (#20858)
lorenzozane738df90a546a
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 279c65371d..3d128bf0d7 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -1599,6 +1599,8 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,from := metaString(p.Meta, "from")to := metaString(p.Meta, "to")if kind == "relation" || (from != "" && to != "") {+ from = strings.TrimSpace(from)+ to = strings.TrimSpace(to)// Skip a relation with an incomplete endpoint pair: writing a bucket// like "A -> " would let multiple malformed relations collapse into// the same dataset row.@@ -1640,6 +1642,15 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,}appendBucket(b, p)}+ for key, bucket := range relByKey {+ if entity := entByKey[ekey{template: key.template, name: key.from, ckwd: key.ckwd}]; entity != nil {+ bucket.FromEntity = entity.Name+ }+ if entity := entByKey[ekey{template: key.template, name: key.to, ckwd: key.ckwd}]; entity != nil {+ bucket.ToEntity = entity.Name+ }+ bucket.Name = bucket.FromEntity + " -> " + bucket.ToEntity+ }buckets := make([]StructureBucket, 0, len(entByKey)+len(relByKey))for _, b := range entByKey {buckets = append(buckets, *b)diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex fcd401fce6..d1817ce150 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -20,6 +20,7 @@ import ("encoding/json""fmt""strconv"+ "strings""go.uber.org/zap"@@ -356,6 +357,16 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommomerged := isAvailable(c["available_int"])meta := map[string]any{}+ var payload map[string]any+ if expect == kccommon.VariantStructure || expect == kccommon.VariantMindmap {+ if err := json.Unmarshal([]byte(content), &payload); err != nil {+ if strings.HasPrefix(strings.TrimSpace(content), "{") {+ common.Warn("knowledge_compile: failed to decode structure payload",+ zap.String("row_id", id), zap.String("compile_kwd", variant), zap.Error(err))+ }+ payload = nil+ }+ }if raw := asString(c["extra"]); raw != "" {var extra map[string]anyif err := json.Unmarshal([]byte(raw), &extra); err != nil {@@ -372,7 +383,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo}}if v, ok := c["name_kwd"].(string); ok && v != "" {- meta["name"] = v+ name := v+ for _, key := range []string{"name", "text", "term", "title"} {+ if extracted, ok := payload[key].(string); ok && extracted != "" && normalizedStructureEntityName(extracted) == normalizedStructureEntityName(v) {+ name = strings.TrimSpace(extracted)+ break+ }+ }+ meta["name"] = name}if v, ok := c["entity_type_kwd"].(string); ok && v != "" {if expect == kccommon.VariantMindmap && v == "mindmap" {@@ -388,6 +406,11 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommometa["to"] = vmeta["kind"] = "relation"}+ if metaString(meta, "kind") == "relation" {+ if v, ok := payload["type"].(string); ok && v != "" {+ meta["relation_type"] = v+ }+ }if v, ok := metaInt(c, "mention_count_int"); ok {meta["mention_count"] = v}diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex 33e7c839e0..3e26e8276e 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -403,3 +403,65 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {t.Errorf("tree Meta.kind = %q, want root (so nav dispatch can pick it)", kind)}}++func TestProductFromChunkMap_PreservesStructurePayloadFieldsThroughMerge(t *testing.T) {+ rows := []map[string]interface{}{+ {+ "id": "entity-acme", "doc_id": "d1", "compile_kwd": "knowledge_graph",+ "content_with_weight": `{"name":" Acme Corp ","type":"organization","description":"Supplier"}`,+ "knowledge_graph_kwd": "graph", "type_kwd": "entity", "name_kwd": "acme corp",+ },+ {+ "id": "entity-beta", "doc_id": "d1", "compile_kwd": "knowledge_graph",+ "content_with_weight": `{"text":"Beta Ltd","type":"organization","description":"Vendor"}`,+ "knowledge_graph_kwd": "graph", "type_kwd": "entity", "name_kwd": "beta ltd",+ },+ {+ "id": "relation-supplies", "doc_id": "d1", "compile_kwd": "knowledge_graph",+ "content_with_weight": `{"from":"Acme Corp","to":"Beta Ltd","type":"supplies","description":"supply"}`,+ "knowledge_graph_kwd": "graph", "type_kwd": "relation",+ "from_entity_kwd": " ACME CORP ", "to_entity_kwd": "BETA LTD ",+ },+ {+ "id": "relation-owns", "doc_id": "d1", "compile_kwd": "knowledge_graph",+ "content_with_weight": `{"from":"Acme Corp","to":"Beta Ltd","type":"owns","description":"ownership"}`,+ "knowledge_graph_kwd": "graph", "type_kwd": "relation",+ "from_entity_kwd": " ACME CORP ", "to_entity_kwd": "BETA LTD ",+ },+ }+ products := make([]kccommon.Product, 0, len(rows))+ for _, row := range rows {+ product, ok := productFromChunkMap(row, "t1", kccommon.VariantStructure)+ if !ok {+ t.Fatalf("row %q should reconstruct", row["id"])+ }+ products = append(products, product)+ }++ c := &Consumer{writer: &fakeWriter{}}+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ buckets := c.writer.(*fakeWriter).buckets+ if len(buckets) != 4 {+ t.Fatalf("want 2 entities and 2 typed relations, got %d: %+v", len(buckets), buckets)+ }+ entities := map[string]bool{}+ relations := map[string]bool{}+ for _, bucket := range buckets {+ if bucket.Type == "relation" {+ relations[bucket.RelationType] = true+ if bucket.FromEntity != "Acme Corp" || bucket.ToEntity != "Beta Ltd" {+ t.Errorf("relation endpoints = %q -> %q, want trimmed entity spellings", bucket.FromEntity, bucket.ToEntity)+ }+ continue+ }+ entities[bucket.Name] = true+ }+ if !entities["Acme Corp"] || !entities["Beta Ltd"] {+ t.Errorf("entity display names = %v, want extracted spelling", entities)+ }+ if !relations["supplies"] || !relations["owns"] {+ t.Errorf("relation types = %v, want supplies and owns", relations)+ }+}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 9b96e5b057..60dabda8cc 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -561,7 +561,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinrow["to_entity_kwd"] = b.ToEntity} else {row["type_kwd"] = "entity"- row["name_kwd"] = b.Name+ row["name_kwd"] = strings.ToLower(b.Name)row["entity_type_kwd"] = b.Typeif b.MentionCount > 0 {row["mention_count_int"] = b.MentionCountdiff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 47ac4bb258..49373befbe 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -237,6 +237,57 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {}}+func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{+ Name: "Acme Corp", Type: "organization", Description: "company", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph",+ }}); err != nil {+ t.Fatalf("WriteMergedStructure: %v", err)+ }+ for _, row := range eng.insertedChunks {+ if row["type_kwd"] != "entity" {+ continue+ }+ if row["name_kwd"] != "acme corp" {+ t.Errorf("name_kwd = %v, want normalized keyword acme corp", row["name_kwd"])+ }+ payload, _ := row["content_with_weight"].(string)+ if !strings.Contains(payload, `"name":"Acme Corp"`) {+ t.Errorf("entity payload = %s, want extracted spelling Acme Corp", payload)+ }+ return+ }+ t.Fatal("inserted rows contain no entity")+}++func TestWriteMergedStructurePreservesDistinctRelationTypes(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ buckets := []StructureBucket{+ {Name: "Acme Corp -> Beta Ltd", Type: "relation", FromEntity: "Acme Corp", ToEntity: "Beta Ltd", RelationType: "supplies", Description: "supply", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph"},+ {Name: "Acme Corp -> Beta Ltd", Type: "relation", FromEntity: "Acme Corp", ToEntity: "Beta Ltd", RelationType: "owns", Description: "ownership", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph"},+ }+ if err := w.WriteMergedStructure(context.Background(), "t1", "kb1", buckets); err != nil {+ t.Fatalf("WriteMergedStructure: %v", err)+ }+ relations := map[string]bool{}+ for _, row := range eng.insertedChunks {+ if row["type_kwd"] != "relation" {+ continue+ }+ payload, _ := row["content_with_weight"].(string)+ for _, relationType := range []string{"supplies", "owns"} {+ if strings.Contains(payload, `"type":"`+relationType+`"`) {+ relations[relationType] = true+ }+ }+ }+ if len(relations) != 2 || !relations["supplies"] || !relations["owns"] {+ t.Errorf("stored relation types = %v, want separate supplies and owns rows", relations)+ }+}+func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {oldID := "dataset_structure_old"eng := &fakeEngine{searchChunks: []map[string]interface{}{