Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: merge same-name knowledge graph entities (#19525)

buua4369562f0f3abf5
diff --git a/internal/ingestion/component/knowledge_compiler/common/memstore.go b/internal/ingestion/component/knowledge_compiler/common/memstore.go
index 983d33e55a..2a358e6ef2 100644
--- a/internal/ingestion/component/knowledge_compiler/common/memstore.go
+++ b/internal/ingestion/component/knowledge_compiler/common/memstore.go
@@ -123,6 +123,46 @@ func (m *MemStore) DedupeAdd(row Product, threshold float64, cb DedupCallback) (
}
}
+// DedupeAddByID applies a deduplication decision against a specific stored
+// product. Callers use this when a domain identity (such as an entity name)
+// must take precedence over vector similarity.
+func (m *MemStore) DedupeAddByID(row Product, candidateID string, cb DedupCallback) (KeepAction, error) {
+ m.mu.RLock()
+ idx, ok := m.byID[candidateID]
+ var best Product
+ if ok && idx >= 0 && idx < len(m.items) {
+ best = m.items[idx]
+ }
+ m.mu.RUnlock()
+ if best.ID == "" {
+ return m.DedupeAdd(row, 0, cb)
+ }
+ action, replacement, err := cb(best, 1)
+ if err != nil {
+ return 0, err
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ resolvedIdx, ok := m.byID[best.ID]
+ if !ok || resolvedIdx < 0 || resolvedIdx >= len(m.items) || m.items[resolvedIdx].ID != best.ID {
+ m.addLocked(row)
+ return KeepAdd, nil
+ }
+ switch action {
+ case KeepDrop:
+ return KeepDrop, nil
+ case KeepMerge:
+ replacement.ID = best.ID
+ m.items[resolvedIdx] = replacement
+ m.replaceMatLocked(resolvedIdx, replacement.Vector)
+ m.norms[resolvedIdx] = l2Norm(replacement.Vector)
+ return KeepMerge, nil
+ default:
+ m.addLocked(row)
+ return KeepAdd, nil
+ }
+}
+
func (m *MemStore) addLocked(p Product) {
m.items = append(m.items, p)
m.appendMatLocked(p.Vector)
diff --git a/internal/ingestion/component/knowledge_compiler/structure/graph.go b/internal/ingestion/component/knowledge_compiler/structure/graph.go
index c35ec9a251..fdb8751fbc 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/graph.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/graph.go
@@ -90,16 +90,17 @@ func graphRelation(payload map[string]any) map[string]any {
return map[string]any{"from": src, "to": tgt, "type": typ}
}
-// mergeGraphEntities mirrors _struct_merge_graph_entities: collapse entities
-// sharing (name, type) — mention_count sums, aliases/description/
+// mergeGraphEntities collapses entities sharing the same name — mention_count
+// sums, aliases/description/
// source_chunk_ids union, first-seen order preserved.
func mergeGraphEntities(entities []map[string]any) []map[string]any {
- type key struct{ name, typ string }
+ type key struct{ name string }
merged := map[key]int{}
var order []key
var out []map[string]any
for _, entity := range entities {
- k := key{name: entity["name"].(string), typ: entity["type"].(string)}
+ name, _ := entity["name"].(string)
+ k := key{name: normalizedEntityName(name)}
idx, ok := merged[k]
if !ok {
merged[k] = len(out)
@@ -109,6 +110,7 @@ func mergeGraphEntities(entities []map[string]any) []map[string]any {
}
target := out[idx]
target["mention_count"] = mentionOf(target) + mentionOf(entity)
+ target["type"] = preferredEntityType(stringOf(target["type"]), stringOf(entity["type"]))
target["aliases"] = unionOrdered(toStrings(target["aliases"]), toStrings(entity["aliases"]))
if target["description"] == "" && entity["description"] != "" {
target["description"] = entity["description"]
diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.go
index d36f800b69..19eea60e82 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go
@@ -180,6 +180,18 @@ func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, jobs []f
// Decide implements MergeDecider.
func (d *LLMMergeDecider) Decide(ctx context.Context, existing, incoming common.Product, bestScore float64) (MergeDecision, common.Product, error) {
+ // Entity names are the graph's endpoint identity. A type is descriptive
+ // metadata, not a second node identity; otherwise relations that only carry
+ // endpoint names become ambiguous when the same name is emitted with two
+ // different types.
+ if sameEntityName(existing, incoming) {
+ merged := mergeEntitiesByName(existing, incoming)
+ replacement, err := d.BuildReplacement(ctx, existing, incoming, merged)
+ if err != nil {
+ return DecisionKeepBoth, common.Product{}, err
+ }
+ return DecisionMerge, replacement, nil
+ }
if bestScore < d.Threshold {
return DecisionKeepBoth, common.Product{}, nil
}
@@ -197,6 +209,61 @@ func (d *LLMMergeDecider) Decide(ctx context.Context, existing, incoming common.
return DecisionMerge, replacement, nil
}
+func sameEntityName(existing, incoming common.Product) bool {
+ if kind, _ := existing.Meta["kind"].(string); kind != "entity" {
+ return false
+ }
+ if kind, _ := incoming.Meta["kind"].(string); kind != "entity" {
+ return false
+ }
+ name := normalizedEntityName(entityNameValue(existing))
+ return name != "" && name == normalizedEntityName(entityNameValue(incoming))
+}
+
+func normalizedEntityName(name string) string {
+ return strings.ToLower(strings.Join(strings.Fields(name), " "))
+}
+
+func preferredEntityType(existing, incoming string) string {
+ existing = strings.TrimSpace(existing)
+ incoming = strings.TrimSpace(incoming)
+ if existing == "" || strings.EqualFold(existing, "other") {
+ if incoming != "" && !strings.EqualFold(incoming, "other") {
+ return incoming
+ }
+ }
+ if existing != "" {
+ return existing
+ }
+ return incoming
+}
+
+func mergeEntitiesByName(existing, incoming common.Product) map[string]any {
+ merged := parsePayload(existing.Content)
+ if merged == nil {
+ merged = map[string]any{}
+ }
+ existingPayload := parsePayload(existing.Content)
+ incomingPayload := parsePayload(incoming.Content)
+ for key, value := range incomingPayload {
+ if _, exists := merged[key]; !exists || strings.TrimSpace(stringOf(merged[key])) == "" {
+ merged[key] = value
+ }
+ }
+ merged["type"] = preferredEntityType(stringOf(existingPayload["type"]), stringOf(incomingPayload["type"]))
+ left := strings.TrimSpace(stringOf(existingPayload["description"]))
+ right := strings.TrimSpace(stringOf(incomingPayload["description"]))
+ if left != "" && right != "" && !strings.EqualFold(left, right) {
+ merged["description"] = left + "\n" + right
+ } else if left != "" {
+ merged["description"] = left
+ } else if right != "" {
+ merged["description"] = right
+ }
+ merged["aliases"] = unionOrdered(toStrings(merged["aliases"]), toStrings(incomingPayload["aliases"]))
+ return merged
+}
+
// buildReplacement folds an LLM-merged payload back into a replacement Product
// for the existing row: aliases are recorded (entities), merge invariants are
// applied (relations), provenance is unioned, and the payload is re-embedded.
@@ -634,6 +701,7 @@ type MergeStats struct {
Inserted int
Updated int
DuplicatesDropped int
+ survivorID string
}
// MergeIntoStore folds the incoming rows into store, using DedupeAdd so the
@@ -649,7 +717,9 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDe
for _, row := range rows {
var droppedExisting common.Product
var dropped bool
+ var survivorID string
action, err := store.DedupeAdd(row, 0, func(existing common.Product, score float64) (common.KeepAction, common.Product, error) {
+ survivorID = existing.ID
d, replacement, e := decider.Decide(ctx, existing, row, score)
if e != nil {
return common.KeepAdd, common.Product{}, e
@@ -674,6 +744,7 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDe
switch action {
case common.KeepDrop:
stats.DuplicatesDropped++
+ stats.survivorID = droppedExisting.ID
// Fold the incoming source_chunk_ids into the surviving entry.
if dropped && droppedExisting.ID != "" {
if ids := metaStrings(row.Meta, "source_chunk_ids"); len(ids) > 0 {
@@ -683,39 +754,42 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDe
case common.KeepMerge:
stats.Updated++
stats.DuplicatesDropped++
+ stats.survivorID = survivorID
default:
stats.Inserted++
+ stats.survivorID = row.ID
}
}
return stats, nil
}
-// GroupedDeduper mirrors _struct_local_dedup's group-by-filter-key behaviour:
-// rows are bucketed by their relation endpoints (entities share one bucket,
-// mirroring _struct_filter_key where entity rows carry no from/to), so an
-// entity never merges with a relation and relations only merge with
-// same-endpoint relations. One MemStore per bucket keeps cosine candidates
-// group-scoped. doc/compile/template are constant per run and therefore not
-// part of the key (they are in Python's key for the cross-document ES case).
+// GroupedDeduper keeps entities in one semantic candidate pool while tracking
+// exact names separately. Exact-name matches take precedence over vectors, so
+// same-name entities always merge regardless of their emitted type; different
+// names retain the existing alias/similarity dedupe behavior.
type GroupedDeduper struct {
decider MergeDecider
- mu sync.Mutex
- order []string
- stores map[string]*common.MemStore
- stats MergeStats
+ mu sync.Mutex
+ order []string
+ stores map[string]*common.MemStore
+ entityByName map[string]string
+ stats MergeStats
}
// NewGroupedDeduper constructs a deduper around the given decider.
func NewGroupedDeduper(decider MergeDecider) *GroupedDeduper {
- return &GroupedDeduper{decider: decider, stores: map[string]*common.MemStore{}}
+ return &GroupedDeduper{decider: decider, stores: map[string]*common.MemStore{}, entityByName: map[string]string{}}
}
-// groupKey mirrors _struct_filter_key minus the per-run constants.
+// groupKey returns the graph identity used by the in-run deduper.
func groupKey(row common.Product) string {
+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {
+ return "entity"
+ }
from, _ := row.Meta["from"].(string)
to, _ := row.Meta["to"].(string)
- return from + "\x00" + to
+ return "relation\x00" + normalizedEntityName(from) + "\x00" + normalizedEntityName(to)
}
// Add folds one row into its endpoint bucket. Safe for concurrent use, but
@@ -732,14 +806,77 @@ func (g *GroupedDeduper) Add(ctx context.Context, row common.Product) error {
}
g.mu.Unlock()
- s, err := MergeIntoStore(ctx, store, g.decider, []common.Product{row})
+ var action common.KeepAction
+ var err error
+ var candidateID string
+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {
+ name := normalizedEntityName(entityNameValue(row))
+ g.mu.Lock()
+ candidateID = g.entityByName[name]
+ g.mu.Unlock()
+ if candidateID != "" {
+ action, err = store.DedupeAddByID(row, candidateID, func(existing common.Product, _ float64) (common.KeepAction, common.Product, error) {
+ d, replacement, e := g.decider.Decide(ctx, existing, row, 1)
+ if e != nil {
+ return common.KeepAdd, common.Product{}, e
+ }
+ switch d {
+ case DecisionDropIncoming:
+ return common.KeepDrop, common.Product{}, nil
+ case DecisionMerge:
+ return common.KeepMerge, replacement, nil
+ default:
+ return common.KeepAdd, common.Product{}, nil
+ }
+ })
+ if err == nil && action == common.KeepDrop {
+ store.MergeSourceChunkIDs(candidateID, metaStrings(row.Meta, "source_chunk_ids"))
+ }
+ } else {
+ var stats MergeStats
+ stats, err = MergeIntoStore(ctx, store, g.decider, []common.Product{row})
+ candidateID = stats.survivorID
+ action = common.KeepAdd
+ if stats.Updated > 0 {
+ action = common.KeepMerge
+ } else if stats.DuplicatesDropped > 0 {
+ action = common.KeepDrop
+ }
+ }
+ } else {
+ var stats MergeStats
+ stats, err = MergeIntoStore(ctx, store, g.decider, []common.Product{row})
+ action = common.KeepAdd
+ if stats.Updated > 0 {
+ action = common.KeepMerge
+ } else if stats.DuplicatesDropped > 0 {
+ action = common.KeepDrop
+ }
+ }
if err != nil {
return err
}
+ if action == common.KeepAdd {
+ candidateID = row.ID
+ }
g.mu.Lock()
- g.stats.Inserted += s.Inserted
- g.stats.Updated += s.Updated
- g.stats.DuplicatesDropped += s.DuplicatesDropped
+ if kind, _ := row.Meta["kind"].(string); kind == "entity" {
+ name := normalizedEntityName(entityNameValue(row))
+ if candidateID != "" {
+ g.entityByName[name] = candidateID
+ } else if action == common.KeepAdd && row.ID != "" {
+ g.entityByName[name] = row.ID
+ }
+ }
+ switch action {
+ case common.KeepMerge:
+ g.stats.Updated++
+ g.stats.DuplicatesDropped++
+ case common.KeepDrop:
+ g.stats.DuplicatesDropped++
+ default:
+ g.stats.Inserted++
+ }
g.mu.Unlock()
return nil
}
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
index cea2ee056f..5c26cc380c 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
@@ -772,3 +772,101 @@ func TestCosineDecider(t *testing.T) {
t.Fatalf("expected keep at 0.5, got %v", got)
}
}
+
+func TestMergeEntitiesByNameIgnoresType(t *testing.T) {
+ existing := common.Product{
+ Content: `{"name":"Engine","type":"component","description":"mechanical part"}`,
+ Meta: map[string]any{"kind": "entity", "name": "Engine", "entity_type": "component"},
+ }
+ incoming := common.Product{
+ Content: `{"name":"engine","type":"system","description":"controls the vehicle"}`,
+ Meta: map[string]any{"kind": "entity", "name": "engine", "entity_type": "system"},
+ }
+ if !sameEntityName(existing, incoming) {
+ t.Fatal("entities with the same name should share one identity regardless of type")
+ }
+ merged := mergeEntitiesByName(existing, incoming)
+ if merged["name"] != "Engine" || merged["type"] != "component" {
+ t.Fatalf("merge should preserve the first entity identity: %v", merged)
+ }
+ if !strings.Contains(stringOf(merged["description"]), "controls the vehicle") {
+ t.Fatalf("merge should retain both descriptions: %v", merged)
+ }
+}
+
+func TestGroupedDeduperMergesSameNameBeforeVectorSimilarity(t *testing.T) {
+ chat := &graphChat{}
+ deduper := NewGroupedDeduper(NewLLMMergeDecider(chat, "llm", constEmbedder{dim: 2}, 0.99))
+ rows := []common.Product{
+ {ID: "first", Content: `{"name":"Engine","type":"other","description":"first"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Engine", "entity_type": "other", "source_chunk_ids": []string{"c1"}}},
+ {ID: "second", Content: `{"name":" engine ","type":"component","description":"second"}`, Vector: []float32{0, 1}, Meta: map[string]any{"kind": "entity", "name": " engine ", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},
+ }
+ for _, row := range rows {
+ if err := deduper.Add(context.Background(), row); err != nil {
+ t.Fatal(err)
+ }
+ }
+ got := deduper.Rows()
+ if len(got) != 1 {
+ t.Fatalf("same-name entities must merge despite dissimilar vectors: %+v", got)
+ }
+ payload := parsePayload(got[0].Content)
+ if payload["type"] != "component" {
+ t.Fatalf("specific type should replace other: %v", payload)
+ }
+ if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {
+ t.Fatalf("source chunks were not merged: %v", ids)
+ }
+ if chat.mergeCalls != 0 {
+ t.Fatalf("exact-name merge should not call the LLM, got %d calls", chat.mergeCalls)
+ }
+}
+
+func TestMergeGraphEntitiesNormalizesName(t *testing.T) {
+ got := mergeGraphEntities([]map[string]any{
+ {"name": "Engine", "type": "other", "description": "first", "mention_count": 1},
+ {"name": " engine ", "type": "component", "description": "second", "mention_count": 1},
+ })
+ if len(got) != 1 {
+ t.Fatalf("case and surrounding whitespace must not split graph nodes: %+v", got)
+ }
+ if got[0]["type"] != "component" || mentionOf(got[0]) != 2 {
+ t.Fatalf("merged graph entity = %+v", got[0])
+ }
+}
+
+type similarityMergeDecider struct{}
+
+func (similarityMergeDecider) Decide(_ context.Context, existing, incoming common.Product, score float64) (MergeDecision, common.Product, error) {
+ if existing.ID == "" || score < 0.9 {
+ return DecisionKeepBoth, common.Product{}, nil
+ }
+ replacement := existing
+ replacement.Meta = make(map[string]any, len(existing.Meta))
+ for key, value := range existing.Meta {
+ replacement.Meta[key] = value
+ }
+ replacement.Meta["source_chunk_ids"] = unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))
+ return DecisionMerge, replacement, nil
+}
+
+func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {
+ deduper := NewGroupedDeduper(similarityMergeDecider{})
+ rows := []common.Product{
+ {ID: "canonical", Content: `{"name":"Alpha","type":"letter"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Alpha", "source_chunk_ids": []string{"c1"}}},
+ {ID: "alias-1", Content: `{"name":"Al","type":"letter"}`, Vector: []float32{1, 0}, Meta: map[string]any{"kind": "entity", "name": "Al", "source_chunk_ids": []string{"c2"}}},
+ {ID: "alias-2", Content: `{"name":"Al","type":"letter"}`, Vector: []float32{0, 1}, Meta: map[string]any{"kind": "entity", "name": "Al", "source_chunk_ids": []string{"c3"}}},
+ }
+ for _, row := range rows {
+ if err := deduper.Add(context.Background(), row); err != nil {
+ t.Fatal(err)
+ }
+ }
+ got := deduper.Rows()
+ if len(got) != 1 {
+ t.Fatalf("semantic alias must remain addressable by its own name: %+v", got)
+ }
+ if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {
+ t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 6624a8d273..c2c7622a42 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -1518,22 +1518,22 @@ func structureEntityNavLine(payloadJSON string) string {
}
// mergeStructureDataset performs the dataset-level structure merge (G1/G4): it
-// groups the structure products of a batch by (name, type), folds their
+// groups the structure products of a batch by name, folds their
// descriptions and source doc/chunk sets into one StructureBucket per group, and
// writes scope_kwd="dataset" rows via WriteMergedStructure. This mirrors Python
// dataset_structure_merger._merge_bucket but is a self-contained from-scratch
-// path (B3: the legacy unified merge never aggregated structure by name/type).
+// path (B3: the legacy unified merge never aggregated structure by name).
func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {
common.Info("knowledge_compile: mergeStructureDataset entry",
zap.String("kb_id", kb),
zap.Int("products", len(products)))
- // Bucket by (lower(name), type, compile kind) for entities and
- // (lower(from), lower(type), lower(to), compile kind) for relations —
+ // Bucket by (template, lower(name), compile kind) for entities and
+ // (template, lower(from), lower(type), lower(to), compile kind) for relations —
// relations have no "name" and must not be silently dropped (review issue 3),
// and the relation type + compile kind keep distinct structure kinds and
// distinct relation types from collapsing into one dataset row.
- type ekey struct{ name, typ, ckwd string }
- type rkey struct{ from, typ, to, ckwd string }
+ type ekey struct{ template, name, ckwd string }
+ type rkey struct{ template, from, typ, to, ckwd string }
entByKey := make(map[ekey]*StructureBucket, 16)
relByKey := make(map[rkey]*StructureBucket, 16)
// The authoritative Python (dataset_structure_merger._do_build) merges EVERY
@@ -1559,6 +1559,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
ckwd = compileKwdForVariant(p.Variant)
}
kind := metaString(p.Meta, "kind")
+ template := structureTemplateIdentity(p.TemplateID, p.Kind)
from := metaString(p.Meta, "from")
to := metaString(p.Meta, "to")
if kind == "relation" || (from != "" && to != "") {
@@ -1575,7 +1576,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if relType == "" {
relType = "related"
}
- k := rkey{from: strings.ToLower(from), typ: strings.ToLower(relType), to: strings.ToLower(to), ckwd: ckwd}
+ k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}
b := relByKey[k]
if b == nil {
b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}
@@ -1593,11 +1594,13 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if name == "" {
continue
}
- k := ekey{name: strings.ToLower(name), typ: typ, ckwd: ckwd}
+ k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}
b := entByKey[k]
if b == nil {
b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}
entByKey[k] = b
+ } else {
+ b.Type = preferredStructureEntityType(b.Type, typ)
}
appendBucket(b, p)
}
@@ -1625,6 +1628,31 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
return c.writer.WriteMergedStructure(ctx, tenant, kb, buckets)
}
+func normalizedStructureEntityName(name string) string {
+ return strings.ToLower(strings.Join(strings.Fields(name), " "))
+}
+
+func preferredStructureEntityType(existing, incoming string) string {
+ existing = strings.TrimSpace(existing)
+ incoming = strings.TrimSpace(incoming)
+ if existing == "" || strings.EqualFold(existing, "other") {
+ if incoming != "" && !strings.EqualFold(incoming, "other") {
+ return incoming
+ }
+ }
+ if existing != "" {
+ return existing
+ }
+ return incoming
+}
+
+func structureTemplateIdentity(templateID, templateKind string) string {
+ if templateID = strings.TrimSpace(templateID); templateID != "" {
+ return templateID
+ }
+ return strings.TrimSpace(templateKind)
+}
+
// appendBucket folds a structure product into a bucket: concatenates its
// description, unions its source doc/chunk ids, and folds its vector into a true
// element-wise running mean (review issue 13). VecCount tracks how many vectors
diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.go
index 9e86549c09..e0fae24d58 100644
--- a/internal/ingestion/knowledge_compile/fake_engine_test.go
+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go
@@ -38,7 +38,8 @@ type fakeEngine struct {
// sweeps compile_kwd buckets and the scope_kwd="dataset" bucket separately).
deleteConds []map[string]interface{}
// deleteCount is the number returned by DeleteChunks.
- deleteCount int64
+ deleteCount int64
+ insertedChunks []map[string]interface{}
}
func (f *fakeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
@@ -57,7 +58,8 @@ func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interf
func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string) error {
return nil
}
-func (f *fakeEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _ string) ([]string, error) {
+ f.insertedChunks = append(f.insertedChunks, chunks...)
return nil, nil
}
func (f *fakeEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index ce1f5ada00..4e10a1c922 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -37,10 +37,10 @@ func (f *fakeWriter) StripMergedSources(context.Context, string, string, []strin
func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }
func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }
-// TestMergeStructureDataset_GroupsByNameType covers G1: structure products are
-// bucketed by (name, type), descriptions folded, source ids unioned, and only
+// TestMergeStructureDataset_GroupsByName covers G1: structure products are
+// bucketed by name, descriptions folded, source ids unioned, and only
// scope_kwd="dataset" structure rows written.
-func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {
+func TestMergeStructureDataset_GroupsByName(t *testing.T) {
c := &Consumer{writer: &fakeWriter{}}
products := []kccommon.Product{
{Variant: kccommon.VariantStructure, DocID: "d1",
@@ -49,10 +49,10 @@ func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {
{Variant: kccommon.VariantStructure, DocID: "d2",
Content: "Engine desc B",
Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},
- // different type -> separate bucket
+ // same name with a different type still belongs to the same graph node
{Variant: kccommon.VariantStructure, DocID: "d3",
- Content: "Fuel desc",
- Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "source_chunk_ids": []string{"c3"}}},
+ Content: "Engine desc C",
+ Meta: map[string]any{"name": "Engine", "entity_type": "other", "source_chunk_ids": []string{"c3"}}},
// non-structure product ignored
{Variant: kccommon.VariantWiki, DocID: "d4", Content: "wiki page",
Meta: map[string]any{"name": "Ignored", "kind": "page"}},
@@ -61,29 +61,49 @@ func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {
t.Fatalf("mergeStructureDataset: %v", err)
}
fw := c.writer.(*fakeWriter)
- if len(fw.buckets) != 2 {
- t.Fatalf("want 2 buckets (Engine + Fuel), got %d", len(fw.buckets))
+ if len(fw.buckets) != 1 {
+ t.Fatalf("want 1 bucket (Engine), got %d", len(fw.buckets))
}
- var engineBucket, fuelBucket *StructureBucket
+ var engineBucket *StructureBucket
for i := range fw.buckets {
switch fw.buckets[i].Name {
case "Engine":
engineBucket = &fw.buckets[i]
- case "Fuel":
- fuelBucket = &fw.buckets[i]
}
}
- if engineBucket == nil || fuelBucket == nil {
+ if engineBucket == nil {
t.Fatalf("missing expected buckets: %+v", fw.buckets)
}
if engineBucket.Type != "component" {
t.Errorf("Engine type = %q, want component", engineBucket.Type)
}
- if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") {
+ if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") || !strings.Contains(engineBucket.Description, "Engine desc C") {
t.Errorf("Engine descriptions not folded: %q", engineBucket.Description)
}
- if len(engineBucket.SourceDocIDs) != 2 {
- t.Errorf("Engine source doc union = %v, want [d1 d2]", engineBucket.SourceDocIDs)
+ if len(engineBucket.SourceDocIDs) != 3 {
+ t.Errorf("Engine source doc union = %v, want [d1 d2 d3]", engineBucket.SourceDocIDs)
+ }
+}
+
+func TestMergeStructureDataset_IsolatesTemplates(t *testing.T) {
+ c := &Consumer{writer: &fakeWriter{}}
+ products := []kccommon.Product{
+ {Variant: kccommon.VariantStructure, DocID: "d1", TemplateID: "tpl1", Kind: "knowledge_graph", Content: "first", Meta: map[string]any{"name": "Engine", "entity_type": "component", "compile_kwd": "hypergraph"}},
+ {Variant: kccommon.VariantStructure, DocID: "d2", TemplateID: "tpl2", Kind: "knowledge_graph", Content: "second", Meta: map[string]any{"name": "engine", "entity_type": "system", "compile_kwd": "hypergraph"}},
+ }
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ buckets := c.writer.(*fakeWriter).buckets
+ if len(buckets) != 2 {
+ t.Fatalf("same-name entities from different templates must remain isolated: %+v", buckets)
+ }
+ seen := map[string]bool{}
+ for _, bucket := range buckets {
+ seen[bucket.TemplateID] = true
+ }
+ if !seen["tpl1"] || !seen["tpl2"] {
+ t.Fatalf("template identities were lost: %+v", buckets)
}
}
@@ -114,6 +134,26 @@ func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {
}
}
+func TestMergeStructureDataset_NormalizesRelationIdentity(t *testing.T) {
+ c := &Consumer{writer: &fakeWriter{}}
+ products := []kccommon.Product{
+ {Variant: kccommon.VariantStructure, DocID: "d1", Content: "first",
+ Meta: map[string]any{"kind": "relation", "from": " A ", "to": " B ", "relation_type": " Related ", "source_chunk_ids": []string{"c1"}}},
+ {Variant: kccommon.VariantStructure, DocID: "d2", Content: "second",
+ Meta: map[string]any{"kind": "relation", "from": "a", "to": "b", "relation_type": "related", "source_chunk_ids": []string{"c2"}}},
+ }
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ buckets := c.writer.(*fakeWriter).buckets
+ if len(buckets) != 1 {
+ t.Fatalf("normalized relation identity should produce one bucket, got %d: %+v", len(buckets), buckets)
+ }
+ if got := len(buckets[0].SourceDocIDs); got != 2 {
+ t.Fatalf("relation source docs = %v, want two docs", buckets[0].SourceDocIDs)
+ }
+}
+
// TestMergeStructureDataset_DescriptionIsPlainText covers the description bug:
// the product Content is the doc row's content_with_weight JSON, so the dataset
// row's folded description must be the plain-text "description" field, NOT the raw
@@ -195,19 +235,25 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)
// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is
// deterministic and case-insensitive on name so merges hit the same row.
func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {
- a := datasetLevelStructureID("t1", "kb1", "Engine", "component", "timeline", "")
- b := datasetLevelStructureID("t1", "kb1", "engine", "component", "timeline", "")
+ a := datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "component", "timeline", "")
+ b := datasetLevelStructureID("t1", "kb1", "tpl1", "engine", "component", "timeline", "")
if a != b {
t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)
}
- if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component", "timeline", "") {
+ if a == datasetLevelStructureID("t1", "kb1", "tpl1", "Fuel", "component", "timeline", "") {
t.Errorf("different names must yield different ids")
}
- if a == datasetLevelStructureID("t1", "kb1", "Engine", "component", "mindmap", "") {
+ if a == datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "component", "mindmap", "") {
t.Errorf("different compile kinds must yield different ids")
}
- if datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "causes") ==
- datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "contradicts") {
+ if a != datasetLevelStructureID("t1", "kb1", "tpl1", "Engine", "other", "timeline", "") {
+ t.Errorf("entity type must not change the dataset-level identity")
+ }
+ if a == datasetLevelStructureID("t1", "kb1", "tpl2", "Engine", "component", "timeline", "") {
+ t.Errorf("different templates must yield different ids")
+ }
+ if datasetLevelStructureID("t1", "kb1", "tpl1", "A -> B", "relation", "graph", "causes") ==
+ datasetLevelStructureID("t1", "kb1", "tpl1", "A -> B", "relation", "graph", "contradicts") {
t.Errorf("different relation types between the same endpoints must yield different ids")
}
}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 0c35a2437e..818a6a3f93 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -269,7 +269,7 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
// WriteMergedStructure writes the dataset-level structure merged rows for a KB
// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable
-// dataset-level id keyed on (name, type, raw compile kind), the folded
+// dataset-level id keyed on (template, name, raw compile kind), the folded
// description, the union of source docs/chunks, and the bucket vector. Rows are
// available_int=1 so the dataset-level structure index is searchable; the raw
// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the
@@ -290,57 +290,18 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
}
baseName := fmt.Sprintf("ragflow_%s", tenant)
now := time.Now()
- // Read-modify-write: an incremental batch must not drop the source docs/chunks
- // an earlier batch already accumulated for a (name,type) bucket, so load the
- // existing dataset rows by their stable id and union their sources (review
- // issue 3 / #3 Major).
- existing := map[string]StructureBucket{}
- {
- ids := make([]string, 0, len(buckets))
- for _, b := range buckets {
- if b.Name == "" {
- continue
- }
- ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType))
- }
- if len(ids) > 0 {
- res, err := eng.Search(ctx, &types.SearchRequest{
- IndexNames: []string{baseName},
- KbIDs: []string{kb},
- SelectFields: []string{"id", "source_doc_ids", "source_chunk_ids"},
- Filter: map[string]interface{}{"kb_id": kb, "id": ids},
- Limit: len(ids),
- })
- if err != nil {
- return fmt.Errorf("structure merge read-modify-write load: %w", err)
- }
- for _, c := range res.Chunks {
- id, _ := c["id"].(string)
- if id == "" {
- continue
- }
- existing[id] = StructureBucket{
- SourceDocIDs: firstStringSlice(c["source_doc_ids"]),
- SourceChunkIDs: firstStringSlice(c["source_chunk_ids"]),
- }
- }
- }
+ staleIDs, err := mergeExistingStructureBuckets(ctx, eng, baseName, tenant, kb, buckets)
+ if err != nil {
+ return err
}
rows := make([]map[string]interface{}, 0, len(buckets))
for _, b := range buckets {
- desc := strings.TrimSpace(b.Description)
- if desc == "" {
- continue
- }
- if b.Name == "" {
+ if !structureBucketWritable(b) {
continue
}
- bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType)
- // Union the current batch's sources with any already-accumulated ones.
- if prev, ok := existing[bid]; ok {
- b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)
- b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)
- }
+ desc := strings.TrimSpace(b.Description)
+ template := structureTemplateIdentity(b.TemplateID, b.TemplateKind)
+ bid := datasetLevelStructureID(tenant, kb, template, b.Name, b.Type, b.CompileKwd, b.RelationType)
ckwd := b.CompileKwd
if ckwd == "" {
ckwd = compileKwdStructure
@@ -380,8 +341,8 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
}
payload := string(payloadBytes)
row := map[string]interface{}{
- // Stable dataset-level id keyed on the (name, type) or (from, to)
- // bucket plus the raw compile kind.
+ // Stable dataset-level id keyed on the template-scoped entity name or
+ // relation identity plus the raw compile kind.
"id": bid,
"doc_id": kb,
"tenant_id": tenant,
@@ -473,7 +434,173 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
return err
})
}
- return runCompilerJobs(ctx, jobs)
+ if err := runCompilerJobs(ctx, jobs); err != nil {
+ return err
+ }
+ if len(staleIDs) > 0 {
+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": staleIDs, "kb_id": kb}, baseName, kb); err != nil {
+ return fmt.Errorf("structure merge remove superseded rows: %w", err)
+ }
+ }
+ return nil
+}
+
+func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, baseName, tenant, kb string, buckets []StructureBucket) ([]string, error) {
+ byIdentity := make(map[string]int, len(buckets))
+ compileKinds := make([]string, 0, len(buckets))
+ for i := range buckets {
+ byIdentity[structureBucketIdentity(buckets[i])] = i
+ compileKinds = appendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})
+ }
+ if len(byIdentity) == 0 {
+ return nil, nil
+ }
+
+ const pageSize = 500
+ stale := map[string]bool{}
+ for offset := 0; ; offset += pageSize {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{kb},
+ SelectFields: []string{
+ "id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
+ "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
+ "content_with_weight", "kc_payload", "source_doc_ids", "source_chunk_ids",
+ },
+ Filter: map[string]interface{}{
+ "kb_id": kb,
+ "scope_kwd": "dataset",
+ "compile_kwd": compileKinds,
+ "knowledge_graph_kwd": []string{"entity", "relation"},
+ },
+ Offset: offset,
+ Limit: pageSize,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("structure merge load existing rows: %w", err)
+ }
+ for _, row := range res.Chunks {
+ identity := structureRowIdentity(row)
+ idx, ok := byIdentity[identity]
+ if !ok {
+ continue
+ }
+ bucket := &buckets[idx]
+ if !structureBucketWritable(*bucket) {
+ continue
+ }
+ bucket.SourceDocIDs = appendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))
+ bucket.SourceChunkIDs = appendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))
+ payload := structureRowPayload(row)
+ if bucket.FromEntity == "" && bucket.ToEntity == "" {
+ existingType := structureString(row["entity_type_kwd"])
+ if existingType == "" {
+ existingType = structureString(payload["type"])
+ }
+ bucket.Type = preferredStructureEntityType(existingType, bucket.Type)
+ }
+ id := structureString(row["id"])
+ newID := datasetLevelStructureID(tenant, kb, structureTemplateIdentity(bucket.TemplateID, bucket.TemplateKind), bucket.Name, bucket.Type, bucket.CompileKwd, bucket.RelationType)
+ if id != "" && id != newID {
+ stale[id] = true
+ }
+ }
+ if len(res.Chunks) < pageSize {
+ break
+ }
+ }
+ ids := make([]string, 0, len(stale))
+ for id := range stale {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ return ids, nil
+}
+
+func structureCompileKind(compileKwd string) string {
+ if compileKwd = strings.TrimSpace(compileKwd); compileKwd != "" {
+ return compileKwd
+ }
+ return compileKwdStructure
+}
+
+func structureRelationType(value string) string {
+ value = strings.ToLower(strings.Join(strings.Fields(value), " "))
+ if value == "" {
+ return "related"
+ }
+ return value
+}
+
+func structureBucketWritable(bucket StructureBucket) bool {
+ return strings.TrimSpace(bucket.Name) != "" && strings.TrimSpace(bucket.Description) != ""
+}
+
+func structureBucketIdentity(bucket StructureBucket) string {
+ template := structureTemplateIdentity(bucket.TemplateID, bucket.TemplateKind)
+ compileKwd := structureCompileKind(bucket.CompileKwd)
+ if bucket.FromEntity != "" || bucket.ToEntity != "" {
+ return "relation\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(bucket.FromEntity) + "\x00" + structureRelationType(bucket.RelationType) + "\x00" + normalizedStructureEntityName(bucket.ToEntity)
+ }
+ return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(bucket.Name)
+}
+
+func structureRowIdentity(row map[string]interface{}) string {
+ templateID := ""
+ if ids := firstStringSlice(row["compilation_template_ids"]); len(ids) > 0 {
+ templateID = ids[0]
+ }
+ template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))
+ compileKwd := structureCompileKind(structureString(row["compile_kwd"]))
+ payload := structureRowPayload(row)
+ if strings.EqualFold(structureString(row["knowledge_graph_kwd"]), "relation") {
+ from := structureString(row["from_entity_kwd"])
+ to := structureString(row["to_entity_kwd"])
+ if from == "" {
+ from = structureString(payload["from"])
+ }
+ if to == "" {
+ to = structureString(payload["to"])
+ }
+ return "relation\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(from) + "\x00" + structureRelationType(structureString(payload["type"])) + "\x00" + normalizedStructureEntityName(to)
+ }
+ name := structureString(row["name_kwd"])
+ if name == "" {
+ name = structureString(payload["name"])
+ }
+ return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(name)
+}
+
+func structureRowPayload(row map[string]interface{}) map[string]interface{} {
+ for _, field := range []string{"kc_payload", "content_with_weight"} {
+ value := structureString(row[field])
+ if value == "" {
+ continue
+ }
+ var payload map[string]interface{}
+ if json.Unmarshal([]byte(value), &payload) == nil {
+ return payload
+ }
+ }
+ return map[string]interface{}{}
+}
+
+func structureString(value interface{}) string {
+ switch typed := value.(type) {
+ case string:
+ return strings.TrimSpace(typed)
+ case []string:
+ if len(typed) > 0 {
+ return strings.TrimSpace(typed[0])
+ }
+ case []interface{}:
+ if len(typed) > 0 {
+ if text, ok := typed[0].(string); ok {
+ return strings.TrimSpace(text)
+ }
+ }
+ }
+ return ""
}
// DeleteStructureForDocs removes dataset-level structure rows whose source docs
@@ -599,14 +726,18 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
}
// datasetLevelStructureID builds the stable dataset-level id for a structure
-// bucket, keyed on (name, type, compile kind, relation type). It must be
+// bucket, keyed on template, name, compile kind, and relation type. It must be
// deterministic so an incremental merge read-modify-writes the same row (and a
// rebuild clean removes it). Including the raw compile kind keeps
// timeline/graph/mindmap buckets from colliding in the same dataset namespace;
// including the relation type keeps two relation types between the same endpoints
// (e.g. "causes" vs "contradicts") from colliding into one id (review fix).
-func datasetLevelStructureID(tenant, kb, name, typ, compileKwd, relationType string) string {
- return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ+"\x00"+compileKwd+"\x00"+strings.ToLower(relationType))
+func datasetLevelStructureID(tenant, kb, template, name, typ, compileKwd, relationType string) string {
+ identity := template + "\x00" + normalizedStructureEntityName(name) + "\x00" + structureCompileKind(compileKwd)
+ if relationType != "" {
+ identity += "\x00" + typ + "\x00" + structureRelationType(relationType)
+ }
+ return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+identity)
}
// f32ToF64Slice converts a float32 vector to float64 for the engine's dense
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 32a284c7fe..1c6a4e4ba3 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -112,6 +112,84 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {
}
}
+func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
+ oldID := "dataset_structure_old_type_identity"
+ eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ {
+ "id": oldID,
+ "compile_kwd": "hypergraph",
+ "compilation_template_ids": []string{"tpl1"},
+ "compilation_template_kind_kwd": "knowledge_graph",
+ "knowledge_graph_kwd": "entity",
+ "name_kwd": "Engine",
+ "entity_type_kwd": "other",
+ "content_with_weight": `{"name":"Engine","type":"other","description":"existing"}`,
+ "source_doc_ids": []string{"d1"},
+ "source_chunk_ids": []string{"c1"},
+ },
+ }}
+ w := engineWriter{eng: eng}
+ err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{
+ Name: "engine", Type: "component", Description: "incoming", CompileKwd: "hypergraph",
+ TemplateID: "tpl1", TemplateKind: "knowledge_graph", SourceDocIDs: []string{"d2"}, SourceChunkIDs: []string{"c2"},
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.insertedChunks) == 0 {
+ t.Fatal("new structure row was not inserted")
+ }
+ var entity map[string]interface{}
+ for _, row := range eng.insertedChunks {
+ if row["knowledge_graph_kwd"] == "entity" {
+ entity = row
+ break
+ }
+ }
+ if entity == nil {
+ t.Fatalf("inserted rows contain no entity: %+v", eng.insertedChunks)
+ }
+ if entity["id"] == oldID {
+ t.Fatalf("old type-scoped id was reused: %v", entity["id"])
+ }
+ if entity["entity_type_kwd"] != "component" {
+ t.Fatalf("specific incoming type should replace existing other: %v", entity["entity_type_kwd"])
+ }
+ if got := firstStringSlice(entity["source_doc_ids"]); len(got) != 2 {
+ t.Fatalf("source documents were not migrated: %v", got)
+ }
+ ids, ok := eng.lastDeleteCond["id"].([]string)
+ if !ok || len(ids) != 1 || ids[0] != oldID {
+ t.Fatalf("superseded id was not deleted: %v", eng.lastDeleteCond)
+ }
+}
+
+func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {
+ oldID := "dataset_structure_old"
+ eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ {
+ "id": oldID,
+ "compile_kwd": "hypergraph",
+ "compilation_template_ids": []string{"tpl1"},
+ "compilation_template_kind_kwd": "knowledge_graph",
+ "knowledge_graph_kwd": "entity",
+ "name_kwd": "Engine",
+ "entity_type_kwd": "component",
+ "content_with_weight": `{"name":"Engine","type":"component","description":"existing"}`,
+ },
+ }}
+ w := engineWriter{eng: eng}
+ err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{
+ Name: "Engine", Type: "component", CompileKwd: "hypergraph", TemplateID: "tpl1", TemplateKind: "knowledge_graph",
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if eng.lastDeleteCond != nil {
+ t.Fatalf("invalid bucket must not delete the existing row: %v", eng.lastDeleteCond)
+ }
+}
+
// TestDeleteMergedScopesToMergedWikiRows locks the W1 contract: DeleteMerged
// must only target dataset-level (available_int=1) wiki merged rows. The
// structural filter (kb_id + available_int + wiki page/section compile_kwd) is
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go
index 4e1b1f0a7c..53dc7ce69d 100644
--- a/internal/service/dataset_structure_graph.go
+++ b/internal/service/dataset_structure_graph.go
@@ -288,14 +288,13 @@ func projectRelation(row map[string]interface{}) StructureGraphRelation {
return StructureGraphRelation{"from": src, "to": tgt, "type": typ}
}
-// dedupEntities order-preserving by (lowercased name, type).
+// dedupEntities preserves the first entity for each lowercased name.
func dedupEntities(entities []StructureGraphNode) []StructureGraphNode {
var out []StructureGraphNode
seen := map[string]bool{}
for _, e := range entities {
name := strings.ToLower(strings.TrimSpace(graphStr(e["name"])))
- typ := strings.ToLower(strings.TrimSpace(graphStr(e["type"])))
- key := name + "\x00" + typ
+ key := name
if name == "" || seen[key] {
continue
}
@@ -1218,9 +1217,17 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,
row map[string]interface{}
node StructureGraphNode
}) {
- key := firstStringValue(candidate.row["id"])
- if key == "" {
- key = strings.ToLower(strings.TrimSpace(graphStr(candidate.node["name"]))) + "\x00" + strings.ToLower(strings.TrimSpace(graphStr(candidate.node["type"])))
+ name := strings.ToLower(strings.Join(strings.Fields(graphStr(candidate.node["name"])), " "))
+ template := rowTemplateID(candidate.row)
+ if template == "" {
+ template = firstStringValue(candidate.row["compilation_template_kind_kwd"])
+ }
+ if template == "" {
+ template = firstStringValue(candidate.row["compile_kwd"])
+ }
+ key := template + "\x00" + name
+ if name == "" {
+ key = firstStringValue(candidate.row["id"])
}
if key != "" {
if candidateSeen[key] {
diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go
index 9772e44ca4..0ed32ccbe3 100644
--- a/internal/service/dataset_structure_graph_test.go
+++ b/internal/service/dataset_structure_graph_test.go
@@ -64,11 +64,11 @@ func TestProjectRelation_FallsBackToKwdColumns(t *testing.T) {
}
}
-// TestDedupEntities_OrderPreserving verifies dedup by (lowercased name, type).
+// TestDedupEntities_OrderPreserving verifies dedup by lowercased name.
func TestDedupEntities_OrderPreserving(t *testing.T) {
in := []StructureGraphNode{
{"name": "A", "type": "x"},
- {"name": "a", "type": "x"}, // dup (case-insensitive)
+ {"name": "a", "type": "other"}, // dup despite case and type differences
{"name": "B", "type": "y"},
{"name": ""}, // dropped (empty name)
}