RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: merge same-name knowledge graph entities (#19525)
buua4369562f0f3abf5
diff --git a/internal/ingestion/component/knowledge_compiler/common/memstore.go b/internal/ingestion/component/knowledge_compiler/common/memstore.goindex 983d33e55a..2a358e6ef2 100644--- a/internal/ingestion/component/knowledge_compiler/common/memstore.go+++ b/internal/ingestion/component/knowledge_compiler/common/memstore.go@@ -123,6 +123,46 @@ func (m *MemStore) DedupeAdd(row Product, threshold float64, cb DedupCallback) (}}+// DedupeAddByID applies a deduplication decision against a specific stored+// product. Callers use this when a domain identity (such as an entity name)+// must take precedence over vector similarity.+func (m *MemStore) DedupeAddByID(row Product, candidateID string, cb DedupCallback) (KeepAction, error) {+ m.mu.RLock()+ idx, ok := m.byID[candidateID]+ var best Product+ if ok && idx >= 0 && idx < len(m.items) {+ best = m.items[idx]+ }+ m.mu.RUnlock()+ if best.ID == "" {+ return m.DedupeAdd(row, 0, cb)+ }+ action, replacement, err := cb(best, 1)+ if err != nil {+ return 0, err+ }+ m.mu.Lock()+ defer m.mu.Unlock()+ resolvedIdx, ok := m.byID[best.ID]+ if !ok || resolvedIdx < 0 || resolvedIdx >= len(m.items) || m.items[resolvedIdx].ID != best.ID {+ m.addLocked(row)+ return KeepAdd, nil+ }+ switch action {+ case KeepDrop:+ return KeepDrop, nil+ case KeepMerge:+ replacement.ID = best.ID+ m.items[resolvedIdx] = replacement+ m.replaceMatLocked(resolvedIdx, replacement.Vector)+ m.norms[resolvedIdx] = l2Norm(replacement.Vector)+ return KeepMerge, nil+ default:+ m.addLocked(row)+ return KeepAdd, nil+ }+}+func (m *MemStore) addLocked(p Product) {m.items = append(m.items, p)m.appendMatLocked(p.Vector)diff --git a/internal/ingestion/component/knowledge_compiler/structure/graph.go b/internal/ingestion/component/knowledge_compiler/structure/graph.goindex c35ec9a251..fdb8751fbc 100644--- a/internal/ingestion/component/knowledge_compiler/structure/graph.go+++ b/internal/ingestion/component/knowledge_compiler/structure/graph.go@@ -90,16 +90,17 @@ func graphRelation(payload map[string]any) map[string]any {return map[string]any{"from": src, "to": tgt, "type": typ}}-// mergeGraphEntities mirrors _struct_merge_graph_entities: collapse entities-// sharing (name, type) — mention_count sums, aliases/description/+// mergeGraphEntities collapses entities sharing the same name — mention_count+// sums, aliases/description/// source_chunk_ids union, first-seen order preserved.func mergeGraphEntities(entities []map[string]any) []map[string]any {- type key struct{ name, typ string }+ type key struct{ name string }merged := map[key]int{}var order []keyvar out []map[string]anyfor _, entity := range entities {- k := key{name: entity["name"].(string), typ: entity["type"].(string)}+ name, _ := entity["name"].(string)+ k := key{name: normalizedEntityName(name)}idx, ok := merged[k]if !ok {merged[k] = len(out)@@ -109,6 +110,7 @@ func mergeGraphEntities(entities []map[string]any) []map[string]any {}target := out[idx]target["mention_count"] = mentionOf(target) + mentionOf(entity)+ target["type"] = preferredEntityType(stringOf(target["type"]), stringOf(entity["type"]))target["aliases"] = unionOrdered(toStrings(target["aliases"]), toStrings(entity["aliases"]))if target["description"] == "" && entity["description"] != "" {target["description"] = entity["description"]diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.goindex d36f800b69..19eea60e82 100644--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go@@ -180,6 +180,18 @@ func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, jobs []f// Decide implements MergeDecider.func (d *LLMMergeDecider) Decide(ctx context.Context, existing, incoming common.Product, bestScore float64) (MergeDecision, common.Product, error) {+ // Entity names are the graph's endpoint identity. A type is descriptive+ // metadata, not a second node identity; otherwise relations that only carry+ // endpoint names become ambiguous when the same name is emitted with two+ // different types.+ if sameEntityName(existing, incoming) {+ merged := mergeEntitiesByName(existing, incoming)+ replacement, err := d.BuildReplacement(ctx, existing, incoming, merged)+ if err != nil {+ return DecisionKeepBoth, common.Product{}, err+ }+ return DecisionMerge, replacement, nil+ }if bestScore < d.Threshold {return DecisionKeepBoth, common.Product{}, nil}@@ -197,6 +209,61 @@ func (d *LLMMergeDecider) Decide(ctx context.Context, existing, incoming common.return DecisionMerge, replacement, nil}+func sameEntityName(existing, incoming common.Product) bool {+ if kind, _ := existing.Meta["kind"].(string); kind != "entity" {+ return false+ }+ if kind, _ := incoming.Meta["kind"].(string); kind != "entity" {+ return false+ }+ name := normalizedEntityName(entityNameValue(existing))+ return name != "" && name == normalizedEntityName(entityNameValue(incoming))+}++func normalizedEntityName(name string) string {+ return strings.ToLower(strings.Join(strings.Fields(name), " "))+}++func preferredEntityType(existing, incoming string) string {+ existing = strings.TrimSpace(existing)+ incoming = strings.TrimSpace(incoming)+ if existing == "" || strings.EqualFold(existing, "other") {+ if incoming != "" && !strings.EqualFold(incoming, "other") {+ return incoming+ }+ }+ if existing != "" {+ return existing+ }+ return incoming+}++func mergeEntitiesByName(existing, incoming common.Product) map[string]any {+ merged := parsePayload(existing.Content)+ if merged == nil {+ merged = map[string]any{}+ }+ existingPayload := parsePayload(existing.Content)+ incomingPayload := parsePayload(incoming.Content)+ for key, value := range incomingPayload {+ if _, exists := merged[key]; !exists || strings.TrimSpace(stringOf(merged[key])) == "" {+ merged[key] = value+ }+ }+ merged["type"] = preferredEntityType(stringOf(existingPayload["type"]), stringOf(incomingPayload["type"]))+ left := strings.TrimSpace(stringOf(existingPayload["description"]))+ right := strings.TrimSpace(stringOf(incomingPayload["description"]))+ if left != "" && right != "" && !strings.EqualFold(left, right) {+ merged["description"] = left + "\n" + right+ } else if left != "" {+ merged["description"] = left+ } else if right != "" {+ merged["description"] = right+ }+ merged["aliases"] = unionOrdered(toStrings(merged["aliases"]), toStrings(incomingPayload["aliases"]))+ return merged+}+// buildReplacement folds an LLM-merged payload back into a replacement Product// for the existing row: aliases are recorded (entities), merge invariants are// applied (relations), provenance is unioned, and the payload is re-embedded.@@ -634,6 +701,7 @@ type MergeStats struct {Inserted intUpdated intDuplicatesDropped int+ survivorID string}// MergeIntoStore folds the incoming rows into store, using DedupeAdd so the@@ -649,7 +717,9 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDefor _, row := range rows {var droppedExisting common.Productvar dropped bool+ var survivorID stringaction, err := store.DedupeAdd(row, 0, func(existing common.Product, score float64) (common.KeepAction, common.Product, error) {+ survivorID = existing.IDd, replacement, e := decider.Decide(ctx, existing, row, score)if e != nil {return common.KeepAdd, common.Product{}, e@@ -674,6 +744,7 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDeswitch action {case common.KeepDrop:stats.DuplicatesDropped+++ stats.survivorID = droppedExisting.ID// Fold the incoming source_chunk_ids into the surviving entry.if dropped && droppedExisting.ID != "" {if ids := metaStrings(row.Meta, "source_chunk_ids"); len(ids) > 0 {@@ -683,39 +754,42 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDecase common.KeepMerge:stats.Updated++stats.DuplicatesDropped+++ stats.survivorID = survivorIDdefault:stats.Inserted+++ stats.survivorID = row.ID}}return stats, nil}-// GroupedDeduper mirrors _struct_local_dedup's group-by-filter-key behaviour:-// rows are bucketed by their relation endpoints (entities share one bucket,-// mirroring _struct_filter_key where entity rows carry no from/to), so an-// entity never merges with a relation and relations only merge with-// same-endpoint relations. One MemStore per bucket keeps cosine candidates-// group-scoped. doc/compile/template are constant per run and therefore not-// part of the key (they are in Python's key for the cross-document ES case).+// GroupedDeduper keeps entities in one semantic candidate pool while tracking+// exact names separately. Exact-name matches take precedence over vectors, so+// same-name entities always merge regardless of their emitted type; different+// names retain the existing alias/similarity dedupe behavior.type GroupedDeduper struct {decider MergeDecider- mu sync.Mutex- order []string- stores map[string]*common.MemStore- stats MergeStats+ mu sync.Mutex+ order []string+ stores map[string]*common.MemStore+ entityByName map[string]string+ stats MergeStats}// NewGroupedDeduper constructs a deduper around the given decider.func NewGroupedDeduper(decider MergeDecider) *GroupedDeduper {- return &GroupedDeduper{decider: decider, stores: map[string]*common.MemStore{}}+ return &GroupedDeduper{decider: decider, stores: map[string]*common.MemStore{}, entityByName: map[string]string{}}}-// groupKey mirrors _struct_filter_key minus the per-run constants.+// groupKey returns the graph identity used by the in-run deduper.func groupKey(row common.Product) string {+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {+ return "entity"+ }from, _ := row.Meta["from"].(string)to, _ := row.Meta["to"].(string)- return from + "\x00" + to+ return "relation\x00" + normalizedEntityName(from) + "\x00" + normalizedEntityName(to)}// Add folds one row into its endpoint bucket. Safe for concurrent use, but@@ -732,14 +806,77 @@ func (g *GroupedDeduper) Add(ctx context.Context, row common.Product) error {}g.mu.Unlock()- s, err := MergeIntoStore(ctx, store, g.decider, []common.Product{row})+ var action common.KeepAction+ var err error+ var candidateID string+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {+ name := normalizedEntityName(entityNameValue(row))+ g.mu.Lock()+ candidateID = g.entityByName[name]+ g.mu.Unlock()+ if candidateID != "" {+ action, err = store.DedupeAddByID(row, candidateID, func(existing common.Product, _ float64) (common.KeepAction, common.Product, error) {+ d, replacement, e := g.decider.Decide(ctx, existing, row, 1)+ if e != nil {+ return common.KeepAdd, common.Product{}, e+ }+ switch d {+ case DecisionDropIncoming:+ return common.KeepDrop, common.Product{}, nil+ case DecisionMerge:+ return common.KeepMerge, replacement, nil+ default:+ return common.KeepAdd, common.Product{}, nil+ }+ })+ if err == nil && action == common.KeepDrop {+ store.MergeSourceChunkIDs(candidateID, metaStrings(row.Meta, "source_chunk_ids"))+ }+ } else {+ var stats MergeStats+ stats, err = MergeIntoStore(ctx, store, g.decider, []common.Product{row})+ candidateID = stats.survivorID+ action = common.KeepAdd+ if stats.Updated > 0 {+ action = common.KeepMerge+ } else if stats.DuplicatesDropped > 0 {+ action = common.KeepDrop+ }+ }+ } else {+ var stats MergeStats+ stats, err = MergeIntoStore(ctx, store, g.decider, []common.Product{row})+ action = common.KeepAdd+ if stats.Updated > 0 {+ action = common.KeepMerge+ } else if stats.DuplicatesDropped > 0 {+ action = common.KeepDrop+ }+ }if err != nil {return err}+ if action == common.KeepAdd {+ candidateID = row.ID+ }g.mu.Lock()- g.stats.Inserted += s.Inserted- g.stats.Updated += s.Updated- g.stats.DuplicatesDropped += s.DuplicatesDropped+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {+ name := normalizedEntityName(entityNameValue(row))+ if candidateID != "" {+ g.entityByName[name] = candidateID+ } else if action == common.KeepAdd && row.ID != "" {+ g.entityByName[name] = row.ID+ }+ }+ switch action {+ case common.KeepMerge:+ g.stats.Updated+++ g.stats.DuplicatesDropped+++ case common.KeepDrop:+ g.stats.DuplicatesDropped+++ default:+ g.stats.Inserted+++ }g.mu.Unlock()return nil}diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.goindex cea2ee056f..5c26cc380c 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go@@ -772,3 +772,101 @@ func TestCosineDecider(t *testing.T) {t.Fatalf("expected keep at 0.5, got %v", got)}}++func TestMergeEntitiesByNameIgnoresType(t *testing.T) {+ existing := common.Product{+ Content: `{"name":"Engine","type":"component","description":"mechanical part"}`,+ Meta: map[string]any{"kind": "entity", "name": "Engine", "entity_type": "component"},+ }+ incoming := common.Product{+ Content: `{"name":"engine","type":"system","description":"controls the vehicle"}`,+ Meta: map[string]any{"kind": "entity", "name": "engine", "entity_type": "system"},+ }+ if !sameEntityName(existing, incoming) {+ t.Fatal("entities with the same name should share one identity regardless of type")+ }+ merged := mergeEntitiesByName(existing, incoming)+ if merged["name"] != "Engine" || merged["type"] != "component" {+ t.Fatalf("merge should preserve the first entity identity: %v", merged)+ }+ if !strings.Contains(stringOf(merged["description"]), "controls the vehicle") {+ t.Fatalf("merge should retain both descriptions: %v", merged)+ }+}++func TestGroupedDeduperMergesSameNameBeforeVectorSimilarity(t *testing.T) {+ chat := &graphChat{}+ deduper := NewGroupedDeduper(NewLLMMergeDecider(chat, "llm", constEmbedder{dim: 2}, 0.99))+ rows := []common.Product{+ {ID: "first", Content: `{"name":"Engine","type":"other","description":"first"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Engine", "entity_type": "other", "source_chunk_ids": []string{"c1"}}},+ {ID: "second", Content: `{"name":" engine ","type":"component","description":"second"}`, Vector: []float32{0, 1}, Meta: map[string]any{"kind": "entity", "name": " engine ", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},+ }+ for _, row := range rows {+ if err := deduper.Add(context.Background(), row); err != nil {+ t.Fatal(err)+ }+ }+ got := deduper.Rows()+ if len(got) != 1 {+ t.Fatalf("same-name entities must merge despite dissimilar vectors: %+v", got)+ }+ payload := parsePayload(got[0].Content)+ if payload["type"] != "component" {+ t.Fatalf("specific type should replace other: %v", payload)+ }+ if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {+ t.Fatalf("source chunks were not merged: %v", ids)+ }+ if chat.mergeCalls != 0 {+ t.Fatalf("exact-name merge should not call the LLM, got %d calls", chat.mergeCalls)+ }+}++func TestMergeGraphEntitiesNormalizesName(t *testing.T) {+ got := mergeGraphEntities([]map[string]any{+ {"name": "Engine", "type": "other", "description": "first", "mention_count": 1},+ {"name": " engine ", "type": "component", "description": "second", "mention_count": 1},+ })+ if len(got) != 1 {+ t.Fatalf("case and surrounding whitespace must not split graph nodes: %+v", got)+ }+ if got[0]["type"] != "component" || mentionOf(got[0]) != 2 {+ t.Fatalf("merged graph entity = %+v", got[0])+ }+}++type similarityMergeDecider struct{}++func (similarityMergeDecider) Decide(_ context.Context, existing, incoming common.Product, score float64) (MergeDecision, common.Product, error) {+ if existing.ID == "" || score < 0.9 {+ return DecisionKeepBoth, common.Product{}, nil+ }+ replacement := existing+ replacement.Meta = make(map[string]any, len(existing.Meta))+ for key, value := range existing.Meta {+ replacement.Meta[key] = value+ }+ replacement.Meta["source_chunk_ids"] = unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))+ return DecisionMerge, replacement, nil+}++func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {+ deduper := NewGroupedDeduper(similarityMergeDecider{})+ rows := []common.Product{+ {ID: "canonical", Content: `{"name":"Alpha","type":"letter"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Alpha", "source_chunk_ids": []string{"c1"}}},+ {ID: "alias-1", Content: `{"name":"Al","type":"letter"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Al", "source_chunk_ids": []string{"c2"}}},+ {ID: "alias-2", Content: `{"name":"Al","type":"letter"}`, Vector: []float32{0, 1}, Meta: map[string]any{"kind": "entity", "name": "Al", "source_chunk_ids": []string{"c3"}}},+ }+ for _, row := range rows {+ if err := deduper.Add(context.Background(), row); err != nil {+ t.Fatal(err)+ }+ }+ got := deduper.Rows()+ if len(got) != 1 {+ t.Fatalf("semantic alias must remain addressable by its own name: %+v", got)+ }+ if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {+ t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)+ }+}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 6624a8d273..c2c7622a42 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -1518,22 +1518,22 @@ func structureEntityNavLine(payloadJSON string) string {}// mergeStructureDataset performs the dataset-level structure merge (G1/G4): it-// groups the structure products of a batch by (name, type), folds their+// groups the structure products of a batch by name, folds their// descriptions and source doc/chunk sets into one StructureBucket per group, and// writes scope_kwd="dataset" rows via WriteMergedStructure. This mirrors Python// dataset_structure_merger._merge_bucket but is a self-contained from-scratch-// path (B3: the legacy unified merge never aggregated structure by name/type).+// path (B3: the legacy unified merge never aggregated structure by name).func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {common.Info("knowledge_compile: mergeStructureDataset entry",zap.String("kb_id", kb),zap.Int("products", len(products)))- // Bucket by (lower(name), type, compile kind) for entities and- // (lower(from), lower(type), lower(to), compile kind) for relations —+ // Bucket by (template, lower(name), compile kind) for entities and+ // (template, lower(from), lower(type), lower(to), compile kind) for relations —// relations have no "name" and must not be silently dropped (review issue 3),// and the relation type + compile kind keep distinct structure kinds and// distinct relation types from collapsing into one dataset row.- type ekey struct{ name, typ, ckwd string }- type rkey struct{ from, typ, to, ckwd string }+ type ekey struct{ template, name, ckwd string }+ type rkey struct{ template, from, typ, to, ckwd string }entByKey := make(map[ekey]*StructureBucket, 16)relByKey := make(map[rkey]*StructureBucket, 16)// The authoritative Python (dataset_structure_merger._do_build) merges EVERY@@ -1559,6 +1559,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,ckwd = compileKwdForVariant(p.Variant)}kind := metaString(p.Meta, "kind")+ template := structureTemplateIdentity(p.TemplateID, p.Kind)from := metaString(p.Meta, "from")to := metaString(p.Meta, "to")if kind == "relation" || (from != "" && to != "") {@@ -1575,7 +1576,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if relType == "" {relType = "related"}- k := rkey{from: strings.ToLower(from), typ: strings.ToLower(relType), to: strings.ToLower(to), ckwd: ckwd}+ k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}b := relByKey[k]if b == nil {b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}@@ -1593,11 +1594,13 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if name == "" {continue}- k := ekey{name: strings.ToLower(name), typ: typ, ckwd: ckwd}+ k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}b := entByKey[k]if b == nil {b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}entByKey[k] = b+ } else {+ b.Type = preferredStructureEntityType(b.Type, typ)}appendBucket(b, p)}@@ -1625,6 +1628,31 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,return c.writer.WriteMergedStructure(ctx, tenant, kb, buckets)}+func normalizedStructureEntityName(name string) string {+ return strings.ToLower(strings.Join(strings.Fields(name), " "))+}++func preferredStructureEntityType(existing, incoming string) string {+ existing = strings.TrimSpace(existing)+ incoming = strings.TrimSpace(incoming)+ if existing == "" || strings.EqualFold(existing, "other") {+ if incoming != "" && !strings.EqualFold(incoming, "other") {+ return incoming+ }+ }+ if existing != "" {+ return existing+ }+ return incoming+}++func structureTemplateIdentity(templateID, templateKind string) string {+ if templateID = strings.TrimSpace(templateID); templateID != "" {+ return templateID+ }+ return strings.TrimSpace(templateKind)+}+// appendBucket folds a structure product into a bucket: concatenates its// description, unions its source doc/chunk ids, and folds its vector into a true// element-wise running mean (review issue 13). VecCount tracks how many vectorsdiff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.goindex 9e86549c09..e0fae24d58 100644--- a/internal/ingestion/knowledge_compile/fake_engine_test.go+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go@@ -38,7 +38,8 @@ type fakeEngine struct {// sweeps compile_kwd buckets and the scope_kwd="dataset" bucket separately).deleteConds []map[string]interface{}// deleteCount is the number returned by DeleteChunks.- deleteCount int64+ deleteCount int64+ insertedChunks []map[string]interface{}}func (f *fakeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {@@ -57,7 +58,8 @@ func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interffunc (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string) error {return nil}-func (f *fakeEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _ string) ([]string, error) {+ f.insertedChunks = append(f.insertedChunks, chunks...)return nil, nil}func (f *fakeEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex ce1f5ada00..4e10a1c922 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -37,10 +37,10 @@ func (f *fakeWriter) StripMergedSources(context.Context, string, string, []strinfunc (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }-// TestMergeStructureDataset_GroupsByNameType covers G1: structure products are-// bucketed by (name, type), descriptions folded, source ids unioned, and only+// TestMergeStructureDataset_GroupsByName covers G1: structure products are+// bucketed by name, descriptions folded, source ids unioned, and only// scope_kwd="dataset" structure rows written.-func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {+func TestMergeStructureDataset_GroupsByName(t *testing.T) {c := &Consumer{writer: &fakeWriter{}}products := []kccommon.Product{{Variant: kccommon.VariantStructure, DocID: "d1",@@ -49,10 +49,10 @@ func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {{Variant: kccommon.VariantStructure, DocID: "d2",Content: "Engine desc B",Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},- // different type -> separate bucket+ // same name with a different type still belongs to the same graph node{Variant: kccommon.VariantStructure, DocID: "d3",- Content: "Fuel desc",- Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "source_chunk_ids": []string{"c3"}}},+ Content: "Engine desc C",+ Meta: map[string]any{"name": "Engine", "entity_type": "other", "source_chunk_ids": []string{"c3"}}},// non-structure product ignored{Variant: kccommon.VariantWiki, DocID: "d4", Content: "wiki page",Meta: map[string]any{"name": "Ignored", "kind": "page"}},@@ -61,29 +61,49 @@ func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {t.Fatalf("mergeStructureDataset: %v", err)}fw := c.writer.(*fakeWriter)- if len(fw.buckets) != 2 {- t.Fatalf("want 2 buckets (Engine + Fuel), got %d", len(fw.buckets))+ if len(fw.buckets) != 1 {+ t.Fatalf("want 1 bucket (Engine), got %d", len(fw.buckets))}- var engineBucket, fuelBucket *StructureBucket+ var engineBucket *StructureBucketfor i := range fw.buckets {switch fw.buckets[i].Name {case "Engine":engineBucket = &fw.buckets[i]- case "Fuel":- fuelBucket = &fw.buckets[i]}}- if engineBucket == nil || fuelBucket == nil {+ if engineBucket == nil {t.Fatalf("missing expected buckets: %+v", fw.buckets)}if engineBucket.Type != "component" {t.Errorf("Engine type = %q, want component", engineBucket.Type)}- if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") {+ if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") || !strings.Contains(engineBucket.Description, "Engine desc C") {t.Errorf("Engine descriptions not folded: %q", engineBucket.Description)}- if len(engineBucket.SourceDocIDs) != 2 {- t.Errorf("Engine source doc union = %v, want [d1 d2]", engineBucket.SourceDocIDs)+ if len(engineBucket.SourceDocIDs) != 3 {+ t.Errorf("Engine source doc union = %v, want [d1 d2 d3]", engineBucket.SourceDocIDs)+ }+}++func TestMergeStructureDataset_IsolatesTemplates(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1", TemplateID: "tpl1", Kind: "knowledge_graph", Content: "first", Meta: map[string]any{"name": "Engine", "entity_type": "component", "compile_kwd": "hypergraph"}},+ {Variant: kccommon.VariantStructure, DocID: "d2", TemplateID: "tpl2", Kind: "knowledge_graph", Content: "second", Meta: map[string]any{"name": "engine", "entity_type": "system", "compile_kwd": "hypergraph"}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ buckets := c.writer.(*fakeWriter).buckets+ if len(buckets) != 2 {+ t.Fatalf("same-name entities from different templates must remain isolated: %+v", buckets)+ }+ seen := map[string]bool{}+ for _, bucket := range buckets {+ seen[bucket.TemplateID] = true+ }+ if !seen["tpl1"] || !seen["tpl2"] {+ t.Fatalf("template identities were lost: %+v", buckets)}}@@ -114,6 +134,26 @@ func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {}}+func TestMergeStructureDataset_NormalizesRelationIdentity(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1", Content: "first",+ Meta: map[string]any{"kind": "relation", "from": " A ", "to": " B ", "relation_type": " Related ", "source_chunk_ids": []string{"c1"}}},+ {Variant: kccommon.VariantStructure, DocID: "d2", Content: "second",+ Meta: map[string]any{"kind": "relation", "from": "a", "to": "b", "relation_type": "related", "source_chunk_ids": []string{"c2"}}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ buckets := c.writer.(*fakeWriter).buckets+ if len(buckets) != 1 {+ t.Fatalf("normalized relation identity should produce one bucket, got %d: %+v", len(buckets), buckets)+ }+ if got := len(buckets[0].SourceDocIDs); got != 2 {+ t.Fatalf("relation source docs = %v, want two docs", buckets[0].SourceDocIDs)+ }+}+// TestMergeStructureDataset_DescriptionIsPlainText covers the description bug:// the product Content is the doc row's content_with_weight JSON, so the dataset// row's folded description must be the plain-text "description" field, NOT the raw@@ -195,19 +235,25 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is// deterministic and case-insensitive on name so merges hit the same row.func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {- a := datasetLevelStructureID("t1", "kb1", "Engine", "component", "timeline", "")- b := datasetLevelStructureID("t1", "kb1", "engine", "component", "timeline", "")+ a := datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "component", "timeline", "")+ b := datasetLevelStructureID("t1", "kb1", "tpl1", "engine", "component", "timeline", "")if a != b {t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)}- if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component", "timeline", "") {+ if a == datasetLevelStructureID("t1", "kb1", "tpl1", "Fuel", "component", "timeline", "") {t.Errorf("different names must yield different ids")}- if a == datasetLevelStructureID("t1", "kb1", "Engine", "component", "mindmap", "") {+ if a == datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "component", "mindmap", "") {t.Errorf("different compile kinds must yield different ids")}- if datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "causes") ==- datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "contradicts") {+ if a != datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "other", "timeline", "") {+ t.Errorf("entity type must not change the dataset-level identity")+ }+ if a == datasetLevelStructureID("t1", "kb1", "tpl2", "Engine", "component", "timeline", "") {+ t.Errorf("different templates must yield different ids")+ }+ if datasetLevelStructureID("t1", "kb1", "tpl1", "A -> B", "relation", "graph", "causes") ==+ datasetLevelStructureID("t1", "kb1", "tpl1", "A -> B", "relation", "graph", "contradicts") {t.Errorf("different relation types between the same endpoints must yield different ids")}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 0c35a2437e..818a6a3f93 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -269,7 +269,7 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc// WriteMergedStructure writes the dataset-level structure merged rows for a KB// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable-// dataset-level id keyed on (name, type, raw compile kind), the folded+// dataset-level id keyed on (template, name, raw compile kind), the folded// description, the union of source docs/chunks, and the bucket vector. Rows are// available_int=1 so the dataset-level structure index is searchable; the raw// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the@@ -290,57 +290,18 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin}baseName := fmt.Sprintf("ragflow_%s", tenant)now := time.Now()- // Read-modify-write: an incremental batch must not drop the source docs/chunks- // an earlier batch already accumulated for a (name,type) bucket, so load the- // existing dataset rows by their stable id and union their sources (review- // issue 3 / #3 Major).- existing := map[string]StructureBucket{}- {- ids := make([]string, 0, len(buckets))- for _, b := range buckets {- if b.Name == "" {- continue- }- ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType))- }- if len(ids) > 0 {- res, err := eng.Search(ctx, &types.SearchRequest{- IndexNames: []string{baseName},- KbIDs: []string{kb},- SelectFields: []string{"id", "source_doc_ids", "source_chunk_ids"},- Filter: map[string]interface{}{"kb_id": kb, "id": ids},- Limit: len(ids),- })- if err != nil {- return fmt.Errorf("structure merge read-modify-write load: %w", err)- }- for _, c := range res.Chunks {- id, _ := c["id"].(string)- if id == "" {- continue- }- existing[id] = StructureBucket{- SourceDocIDs: firstStringSlice(c["source_doc_ids"]),- SourceChunkIDs: firstStringSlice(c["source_chunk_ids"]),- }- }- }+ staleIDs, err := mergeExistingStructureBuckets(ctx, eng, baseName, tenant, kb, buckets)+ if err != nil {+ return err}rows := make([]map[string]interface{}, 0, len(buckets))for _, b := range buckets {- desc := strings.TrimSpace(b.Description)- if desc == "" {- continue- }- if b.Name == "" {+ if !structureBucketWritable(b) {continue}- bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType)- // Union the current batch's sources with any already-accumulated ones.- if prev, ok := existing[bid]; ok {- b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)- b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)- }+ desc := strings.TrimSpace(b.Description)+ template := structureTemplateIdentity(b.TemplateID, b.TemplateKind)+ bid := datasetLevelStructureID(tenant, kb, template, b.Name, b.Type, b.CompileKwd, b.RelationType)ckwd := b.CompileKwdif ckwd == "" {ckwd = compileKwdStructure@@ -380,8 +341,8 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin}payload := string(payloadBytes)row := map[string]interface{}{- // Stable dataset-level id keyed on the (name, type) or (from, to)- // bucket plus the raw compile kind.+ // Stable dataset-level id keyed on the template-scoped entity name or+ // relation identity plus the raw compile kind."id": bid,"doc_id": kb,"tenant_id": tenant,@@ -473,7 +434,173 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinreturn err})}- return runCompilerJobs(ctx, jobs)+ if err := runCompilerJobs(ctx, jobs); err != nil {+ return err+ }+ if len(staleIDs) > 0 {+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": staleIDs, "kb_id": kb}, baseName, kb); err != nil {+ return fmt.Errorf("structure merge remove superseded rows: %w", err)+ }+ }+ return nil+}++func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, baseName, tenant, kb string, buckets []StructureBucket) ([]string, error) {+ byIdentity := make(map[string]int, len(buckets))+ compileKinds := make([]string, 0, len(buckets))+ for i := range buckets {+ byIdentity[structureBucketIdentity(buckets[i])] = i+ compileKinds = appendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})+ }+ if len(byIdentity) == 0 {+ return nil, nil+ }++ const pageSize = 500+ stale := map[string]bool{}+ for offset := 0; ; offset += pageSize {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{kb},+ SelectFields: []string{+ "id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",+ "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",+ "content_with_weight", "kc_payload", "source_doc_ids", "source_chunk_ids",+ },+ Filter: map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ "compile_kwd": compileKinds,+ "knowledge_graph_kwd": []string{"entity", "relation"},+ },+ Offset: offset,+ Limit: pageSize,+ })+ if err != nil {+ return nil, fmt.Errorf("structure merge load existing rows: %w", err)+ }+ for _, row := range res.Chunks {+ identity := structureRowIdentity(row)+ idx, ok := byIdentity[identity]+ if !ok {+ continue+ }+ bucket := &buckets[idx]+ if !structureBucketWritable(*bucket) {+ continue+ }+ bucket.SourceDocIDs = appendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))+ bucket.SourceChunkIDs = appendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))+ payload := structureRowPayload(row)+ if bucket.FromEntity == "" && bucket.ToEntity == "" {+ existingType := structureString(row["entity_type_kwd"])+ if existingType == "" {+ existingType = structureString(payload["type"])+ }+ bucket.Type = preferredStructureEntityType(existingType, bucket.Type)+ }+ id := structureString(row["id"])+ newID := datasetLevelStructureID(tenant, kb, structureTemplateIdentity(bucket.TemplateID, bucket.TemplateKind), bucket.Name, bucket.Type, bucket.CompileKwd, bucket.RelationType)+ if id != "" && id != newID {+ stale[id] = true+ }+ }+ if len(res.Chunks) < pageSize {+ break+ }+ }+ ids := make([]string, 0, len(stale))+ for id := range stale {+ ids = append(ids, id)+ }+ sort.Strings(ids)+ return ids, nil+}++func structureCompileKind(compileKwd string) string {+ if compileKwd = strings.TrimSpace(compileKwd); compileKwd != "" {+ return compileKwd+ }+ return compileKwdStructure+}++func structureRelationType(value string) string {+ value = strings.ToLower(strings.Join(strings.Fields(value), " "))+ if value == "" {+ return "related"+ }+ return value+}++func structureBucketWritable(bucket StructureBucket) bool {+ return strings.TrimSpace(bucket.Name) != "" && strings.TrimSpace(bucket.Description) != ""+}++func structureBucketIdentity(bucket StructureBucket) string {+ template := structureTemplateIdentity(bucket.TemplateID, bucket.TemplateKind)+ compileKwd := structureCompileKind(bucket.CompileKwd)+ if bucket.FromEntity != "" || bucket.ToEntity != "" {+ return "relation\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(bucket.FromEntity) + "\x00" + structureRelationType(bucket.RelationType) + "\x00" + normalizedStructureEntityName(bucket.ToEntity)+ }+ return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(bucket.Name)+}++func structureRowIdentity(row map[string]interface{}) string {+ templateID := ""+ if ids := firstStringSlice(row["compilation_template_ids"]); len(ids) > 0 {+ templateID = ids[0]+ }+ template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))+ compileKwd := structureCompileKind(structureString(row["compile_kwd"]))+ payload := structureRowPayload(row)+ if strings.EqualFold(structureString(row["knowledge_graph_kwd"]), "relation") {+ from := structureString(row["from_entity_kwd"])+ to := structureString(row["to_entity_kwd"])+ if from == "" {+ from = structureString(payload["from"])+ }+ if to == "" {+ to = structureString(payload["to"])+ }+ return "relation\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(from) + "\x00" + structureRelationType(structureString(payload["type"])) + "\x00" + normalizedStructureEntityName(to)+ }+ name := structureString(row["name_kwd"])+ if name == "" {+ name = structureString(payload["name"])+ }+ return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(name)+}++func structureRowPayload(row map[string]interface{}) map[string]interface{} {+ for _, field := range []string{"kc_payload", "content_with_weight"} {+ value := structureString(row[field])+ if value == "" {+ continue+ }+ var payload map[string]interface{}+ if json.Unmarshal([]byte(value), &payload) == nil {+ return payload+ }+ }+ return map[string]interface{}{}+}++func structureString(value interface{}) string {+ switch typed := value.(type) {+ case string:+ return strings.TrimSpace(typed)+ case []string:+ if len(typed) > 0 {+ return strings.TrimSpace(typed[0])+ }+ case []interface{}:+ if len(typed) > 0 {+ if text, ok := typed[0].(string); ok {+ return strings.TrimSpace(text)+ }+ }+ }+ return ""}// DeleteStructureForDocs removes dataset-level structure rows whose source docs@@ -599,14 +726,18 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str}// datasetLevelStructureID builds the stable dataset-level id for a structure-// bucket, keyed on (name, type, compile kind, relation type). It must be+// bucket, keyed on template, name, compile kind, and relation type. It must be// deterministic so an incremental merge read-modify-writes the same row (and a// rebuild clean removes it). Including the raw compile kind keeps// timeline/graph/mindmap buckets from colliding in the same dataset namespace;// including the relation type keeps two relation types between the same endpoints// (e.g. "causes" vs "contradicts") from colliding into one id (review fix).-func datasetLevelStructureID(tenant, kb, name, typ, compileKwd, relationType string) string {- return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ+"\x00"+compileKwd+"\x00"+strings.ToLower(relationType))+func datasetLevelStructureID(tenant, kb, template, name, typ, compileKwd, relationType string) string {+ identity := template + "\x00" + normalizedStructureEntityName(name) + "\x00" + structureCompileKind(compileKwd)+ if relationType != "" {+ identity += "\x00" + typ + "\x00" + structureRelationType(relationType)+ }+ return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+identity)}// f32ToF64Slice converts a float32 vector to float64 for the engine's densediff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 32a284c7fe..1c6a4e4ba3 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -112,6 +112,84 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {}}+func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {+ oldID := "dataset_structure_old_type_identity"+ eng := &fakeEngine{searchChunks: []map[string]interface{}{+ {+ "id": oldID,+ "compile_kwd": "hypergraph",+ "compilation_template_ids": []string{"tpl1"},+ "compilation_template_kind_kwd": "knowledge_graph",+ "knowledge_graph_kwd": "entity",+ "name_kwd": "Engine",+ "entity_type_kwd": "other",+ "content_with_weight": `{"name":"Engine","type":"other","description":"existing"}`,+ "source_doc_ids": []string{"d1"},+ "source_chunk_ids": []string{"c1"},+ },+ }}+ w := engineWriter{eng: eng}+ err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{+ Name: "engine", Type: "component", Description: "incoming", CompileKwd: "hypergraph",+ TemplateID: "tpl1", TemplateKind: "knowledge_graph", SourceDocIDs: []string{"d2"}, SourceChunkIDs: []string{"c2"},+ }})+ if err != nil {+ t.Fatal(err)+ }+ if len(eng.insertedChunks) == 0 {+ t.Fatal("new structure row was not inserted")+ }+ var entity map[string]interface{}+ for _, row := range eng.insertedChunks {+ if row["knowledge_graph_kwd"] == "entity" {+ entity = row+ break+ }+ }+ if entity == nil {+ t.Fatalf("inserted rows contain no entity: %+v", eng.insertedChunks)+ }+ if entity["id"] == oldID {+ t.Fatalf("old type-scoped id was reused: %v", entity["id"])+ }+ if entity["entity_type_kwd"] != "component" {+ t.Fatalf("specific incoming type should replace existing other: %v", entity["entity_type_kwd"])+ }+ if got := firstStringSlice(entity["source_doc_ids"]); len(got) != 2 {+ t.Fatalf("source documents were not migrated: %v", got)+ }+ ids, ok := eng.lastDeleteCond["id"].([]string)+ if !ok || len(ids) != 1 || ids[0] != oldID {+ t.Fatalf("superseded id was not deleted: %v", eng.lastDeleteCond)+ }+}++func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {+ oldID := "dataset_structure_old"+ eng := &fakeEngine{searchChunks: []map[string]interface{}{+ {+ "id": oldID,+ "compile_kwd": "hypergraph",+ "compilation_template_ids": []string{"tpl1"},+ "compilation_template_kind_kwd": "knowledge_graph",+ "knowledge_graph_kwd": "entity",+ "name_kwd": "Engine",+ "entity_type_kwd": "component",+ "content_with_weight": `{"name":"Engine","type":"component","description":"existing"}`,+ },+ }}+ w := engineWriter{eng: eng}+ err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{+ Name: "Engine", Type: "component", CompileKwd: "hypergraph", TemplateID: "tpl1", TemplateKind: "knowledge_graph",+ }})+ if err != nil {+ t.Fatal(err)+ }+ if eng.lastDeleteCond != nil {+ t.Fatalf("invalid bucket must not delete the existing row: %v", eng.lastDeleteCond)+ }+}+// TestDeleteMergedScopesToMergedWikiRows locks the W1 contract: DeleteMerged// must only target dataset-level (available_int=1) wiki merged rows. The// structural filter (kb_id + available_int + wiki page/section compile_kwd) isdiff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex 4e1b1f0a7c..53dc7ce69d 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -288,14 +288,13 @@ func projectRelation(row map[string]interface{}) StructureGraphRelation {return StructureGraphRelation{"from": src, "to": tgt, "type": typ}}-// dedupEntities order-preserving by (lowercased name, type).+// dedupEntities preserves the first entity for each lowercased name.func dedupEntities(entities []StructureGraphNode) []StructureGraphNode {var out []StructureGraphNodeseen := map[string]bool{}for _, e := range entities {name := strings.ToLower(strings.TrimSpace(graphStr(e["name"])))- typ := strings.ToLower(strings.TrimSpace(graphStr(e["type"])))- key := name + "\x00" + typ+ key := nameif name == "" || seen[key] {continue}@@ -1218,9 +1217,17 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,row map[string]interface{}node StructureGraphNode}) {- key := firstStringValue(candidate.row["id"])- if key == "" {- key = strings.ToLower(strings.TrimSpace(graphStr(candidate.node["name"]))) + "\x00" + strings.ToLower(strings.TrimSpace(graphStr(candidate.node["type"])))+ name := strings.ToLower(strings.Join(strings.Fields(graphStr(candidate.node["name"])), " "))+ template := rowTemplateID(candidate.row)+ if template == "" {+ template = firstStringValue(candidate.row["compilation_template_kind_kwd"])+ }+ if template == "" {+ template = firstStringValue(candidate.row["compile_kwd"])+ }+ key := template + "\x00" + name+ if name == "" {+ key = firstStringValue(candidate.row["id"])}if key != "" {if candidateSeen[key] {diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.goindex 9772e44ca4..0ed32ccbe3 100644--- a/internal/service/dataset_structure_graph_test.go+++ b/internal/service/dataset_structure_graph_test.go@@ -64,11 +64,11 @@ func TestProjectRelation_FallsBackToKwdColumns(t *testing.T) {}}-// TestDedupEntities_OrderPreserving verifies dedup by (lowercased name, type).+// TestDedupEntities_OrderPreserving verifies dedup by lowercased name.func TestDedupEntities_OrderPreserving(t *testing.T) {in := []StructureGraphNode{{"name": "A", "type": "x"},- {"name": "a", "type": "x"}, // dup (case-insensitive)+ {"name": "a", "type": "other"}, // dup despite case and type differences{"name": "B", "type": "y"},{"name": ""}, // dropped (empty name)}