Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): consolidate unique* string dedup helpers into common (#21961)

xugangqiang8b8ccbf4780c
diff --git a/internal/common/dedup_test.go b/internal/common/dedup_test.go
new file mode 100644
index 0000000000..32fd3b660a
--- /dev/null
+++ b/internal/common/dedup_test.go
@@ -0,0 +1,99 @@
+package common
+
+import (
+ "reflect"
+ "testing"
+)
+
+// TestDeduplicateTrimmed verifies trim + drop-empty + dedup + order-preserving,
+// and the "empty input (or all dropped) returns nil" contract.
+func TestDeduplicateTrimmed(t *testing.T) {
+ cases := []struct {
+ name string
+ in []string
+ want []string
+ }{
+ {"nil input returns nil", nil, nil},
+ {"empty slice returns nil", []string{}, nil},
+ {"single element", []string{"a"}, []string{"a"}},
+ {"dedup preserves order", []string{"b", "a", "b", "c", "a"}, []string{"b", "a", "c"}},
+ {"trims surrounding space", []string{" a ", "a", " b"}, []string{"a", "b"}},
+ {"drops empty string", []string{"", "a", ""}, []string{"a"}},
+ {"drops whitespace-only", []string{" ", " ", "a"}, []string{"a"}},
+ // Adversarial: all entries collapse to empty -> nil, never []string{}.
+ {"all empty collapses to nil", []string{"", " ", " "}, nil},
+ // Adversarial: trim makes two distinct-looking entries identical.
+ {"trim collapses duplicates", []string{"x ", " x", "x"}, []string{"x"}},
+ // Adversarial: unicode whitespace must be trimmed.
+ {"unicode whitespace trimmed", []string{"\ta\n", "a"}, []string{"a"}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got := DeduplicateTrimmed(c.in)
+ if !reflect.DeepEqual(got, c.want) {
+ t.Fatalf("DeduplicateTrimmed(%v) = %v, want %v", c.in, got, c.want)
+ }
+ if c.want == nil && got != nil {
+ t.Fatalf("expected nil result for %v, got %v", c.in, got)
+ }
+ })
+ }
+}
+
+// TestDeduplicateSorted verifies drop-empty + dedup + sort, that it does NOT
+// mutate the caller's slice, and the empty->nil contract.
+func TestDeduplicateSorted(t *testing.T) {
+ // Sort correctness + drop empty (no trim: this variant does NOT trim).
+ in := []string{"c", "", "a", "b", "a"}
+ want := []string{"a", "b", "c"}
+ got := DeduplicateSorted(in)
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("DeduplicateSorted(%v) = %v, want %v", in, got, want)
+ }
+ // Adversarial: does NOT trim, so " a" and "a" stay distinct.
+ inNoTrim := []string{" a", "a"}
+ if got := DeduplicateSorted(inNoTrim); !reflect.DeepEqual(got, []string{" a", "a"}) {
+ t.Fatalf("DeduplicateSorted(%v) = %v, want [ a a] (no trim)", inNoTrim, got)
+ }
+ // Does not mutate the input slice.
+ if in[0] != "c" || in[1] != "" {
+ t.Fatalf("DeduplicateSorted mutated input: %v", in)
+ }
+ // Empty contract.
+ if got := DeduplicateSorted(nil); got != nil {
+ t.Fatalf("DeduplicateSorted(nil) = %v, want nil", got)
+ }
+ if got := DeduplicateSorted([]string{}); got != nil {
+ t.Fatalf("DeduplicateSorted([]string{}) = %v, want nil", got)
+ }
+ if got := DeduplicateSorted([]string{"", ""}); got != nil {
+ t.Fatalf("DeduplicateSorted(all empty) = %v, want nil", got)
+ }
+}
+
+// TestAppendUnique verifies append-into-dst, drop-empty, dedup against existing
+// dst entries, and order preservation.
+func TestAppendUnique(t *testing.T) {
+ // Appends new, skips empties and already-present.
+ dst := []string{"a", "b"}
+ got := AppendUnique(dst, []string{"b", "", "c", "a"})
+ want := []string{"a", "b", "c"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("AppendUnique(%v, ...) = %v, want %v", dst, got, want)
+ }
+ // Appending to a nil dst yields the new, deduped, non-empty items.
+ gotNil := AppendUnique(nil, []string{"x", "x", "y"})
+ if !reflect.DeepEqual(gotNil, []string{"x", "y"}) {
+ t.Fatalf("AppendUnique(nil, ...) = %v, want [x y]", gotNil)
+ }
+ // Appending only empties/dupes leaves dst unchanged (and nil stays nil).
+ gotEmpty := AppendUnique(nil, []string{"", ""})
+ if gotEmpty != nil {
+ t.Fatalf("AppendUnique(nil, empties) = %v, want nil", gotEmpty)
+ }
+ // Adversarial: AppendUnique does NOT trim, so " x" and "x" both stay.
+ gotNoTrim := AppendUnique(nil, []string{" x", "x"})
+ if !reflect.DeepEqual(gotNoTrim, []string{" x", "x"}) {
+ t.Fatalf("AppendUnique(nil, [ x x]) = %v, want [ x x] (no trim)", gotNoTrim)
+ }
+}
diff --git a/internal/common/idenfication.go b/internal/common/idenfication.go
index 7306bbccb2..fcd68c7252 100644
--- a/internal/common/idenfication.go
+++ b/internal/common/idenfication.go
@@ -16,6 +16,11 @@
package common
+import (
+ "sort"
+ "strings"
+)
+
func Deduplicate(names []string) []string {
if names == nil {
return nil
@@ -33,3 +38,60 @@ func Deduplicate(names []string) []string {
return result
}
+
+// DeduplicateTrimmed removes surrounding whitespace, drops empty strings, and
+// removes duplicates while preserving order. Returns nil when the result is
+// empty.
+func DeduplicateTrimmed(values []string) []string {
+ seen := make(map[string]bool)
+ result := make([]string, 0, len(values))
+ for _, v := range values {
+ v = strings.TrimSpace(v)
+ if v == "" || seen[v] {
+ continue
+ }
+ seen[v] = true
+ result = append(result, v)
+ }
+ if len(result) == 0 {
+ return nil
+ }
+ return result
+}
+
+// DeduplicateSorted drops empty strings, removes duplicates, and sorts the
+// result. It does not mutate the input slice. Returns nil when empty.
+func DeduplicateSorted(values []string) []string {
+ seen := make(map[string]bool)
+ result := make([]string, 0, len(values))
+ for _, v := range values {
+ if v == "" || seen[v] {
+ continue
+ }
+ seen[v] = true
+ result = append(result, v)
+ }
+ if len(result) == 0 {
+ return nil
+ }
+ out := append([]string(nil), result...)
+ sort.Strings(out)
+ return out
+}
+
+// AppendUnique appends values not already present in dst (empty strings
+// dropped), preserving order, and returns the extended dst.
+func AppendUnique(dst, values []string) []string {
+ seen := make(map[string]bool, len(dst)+len(values))
+ for _, v := range dst {
+ seen[v] = true
+ }
+ for _, v := range values {
+ if v == "" || seen[v] {
+ continue
+ }
+ seen[v] = true
+ dst = append(dst, v)
+ }
+ return dst
+}
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index a6117b0dbb..57a451df21 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -269,10 +269,14 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
)
result := mergeChunks(inputs, compiled)
if len(out.AffectedPageSlugs) > 0 {
- result["wiki_affected_slugs"] = uniqueSorted(out.AffectedPageSlugs)
+ affected := clog.DeduplicateTrimmed(out.AffectedPageSlugs)
+ sort.Strings(affected)
+ result["wiki_affected_slugs"] = affected
}
if len(out.RemovedPageSlugs) > 0 {
- result["wiki_removed_slugs"] = uniqueSorted(out.RemovedPageSlugs)
+ removed := clog.DeduplicateTrimmed(out.RemovedPageSlugs)
+ sort.Strings(removed)
+ result["wiki_removed_slugs"] = removed
}
if len(out.WikiActiveStates) > 0 {
result["wiki_active_map_states"] = wikiActiveStateValues(out.WikiActiveStates)
@@ -301,24 +305,6 @@ func wikiActiveStateValues(states []common.WikiMapActiveState) []map[string]any
return values
}
-func uniqueSorted(values []string) []string {
- seen := make(map[string]struct{}, len(values))
- result := make([]string, 0, len(values))
- for _, value := range values {
- value = strings.TrimSpace(value)
- if value == "" {
- continue
- }
- if _, ok := seen[value]; ok {
- continue
- }
- seen[value] = struct{}{}
- result = append(result, value)
- }
- sort.Strings(result)
- return result
-}
-
// resolveTemplateSpecs resolves the configured compilation template spec(s) to
// their TemplateInfo rows. Priority: compilation_template_id >
// compilation_template_group_id. The group path resolves the group to its
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.go
index 946807382b..3868718732 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go
@@ -166,8 +166,8 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com
content = page.ContentRaw
}
summary := summarizeMarkdown(content, title)
- sourceChunkIDs := uniqueStrings(page.SourceChunkIDs)
- sourceDocIDs := uniqueStrings(page.SourceDocIDs)
+ sourceChunkIDs := appcommon.DeduplicateTrimmed(page.SourceChunkIDs)
+ sourceDocIDs := appcommon.DeduplicateTrimmed(page.SourceDocIDs)
if len(sourceDocIDs) == 0 && docID != "" {
sourceDocIDs = []string{docID}
}
@@ -186,9 +186,9 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com
"page_type": pageType,
"topic": common.NormalizeWikiTopicPath(page.Topic),
"summary": summary,
- "entity_names": uniqueStrings(page.EntityNames),
- "related_kb_pages": uniqueStrings(page.RelatedKBPages),
- "outlinks": uniqueStrings(page.Outlinks),
+ "entity_names": appcommon.DeduplicateTrimmed(page.EntityNames),
+ "related_kb_pages": appcommon.DeduplicateTrimmed(page.RelatedKBPages),
+ "outlinks": appcommon.DeduplicateTrimmed(page.Outlinks),
"source_chunk_ids": sourceChunkIDs,
"source_doc_ids": sourceDocIDs,
"content_md_raw": page.ContentRaw,
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index f7fb750a89..64cc98ab15 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -768,7 +768,7 @@ func buildWikiEntityPlanPages(reduced wikiExtract) []wikiPlanPage {
Title: name,
PageType: "entity",
Topic: selectWikiIdentityTopic(name, "entity", sourceChunkIDsForWikiIdentity(name, entity.SourceChunkIDs, reduced.Claims), reduced.Topics),
- EntityNames: uniqueStrings(append([]string{name}, entity.Aliases...)),
+ EntityNames: appcommon.DeduplicateTrimmed(append([]string{name}, entity.Aliases...)),
Priority: len(pages) + 1,
})
}
@@ -797,7 +797,7 @@ func sourceChunkIDsForWikiIdentity(name string, sourceChunkIDs []string, claims
ids = append(ids, claim.SourceChunkIDs...)
}
}
- return uniqueStrings(ids)
+ return appcommon.DeduplicateTrimmed(ids)
}
func selectWikiIdentityTopic(name, pageType string, sourceChunkIDs []string, topics []wikiTopic) string {
@@ -1065,8 +1065,8 @@ func (p *wikiPipeline) runRefinePage(
PageType: firstNonEmpty(planItem.PageType, "concept"),
Topic: topic,
Action: firstNonEmpty(planItem.Action, "CREATE"),
- EntityNames: uniqueStrings(planItem.EntityNames),
- RelatedKBPages: uniqueStrings(planItem.RelatedKB),
+ EntityNames: appcommon.DeduplicateTrimmed(planItem.EntityNames),
+ RelatedKBPages: appcommon.DeduplicateTrimmed(planItem.RelatedKB),
ContentRaw: contentRaw,
Content: contentRendered,
Outlinks: outlinks,
@@ -1177,7 +1177,7 @@ func assembleWikiPlanRelatedPages(pages []wikiPlanPage, relations []wikiRelation
pageByName := make(map[string][]int)
for i, page := range pages {
- names := uniqueStrings(append([]string{}, page.EntityNames...))
+ names := appcommon.DeduplicateTrimmed(append([]string{}, page.EntityNames...))
if len(names) == 0 && strings.TrimSpace(page.Title) != "" {
names = []string{page.Title}
}
@@ -1271,7 +1271,7 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {
if len(existing.RoutedEntityNames) > 0 {
// An explicit route membership set is authoritative: this is a
// migration, so do not union members that the LLM removed.
- page.EntityNames = uniqueStrings(existing.RoutedEntityNames)
+ page.EntityNames = appcommon.DeduplicateTrimmed(existing.RoutedEntityNames)
} else {
page.EntityNames = mergeStrings(page.EntityNames, existing.EntityNames)
}
@@ -1426,7 +1426,7 @@ func (p *wikiPipeline) sourceChunksForPlanPage(page wikiPlanPage) []string {
chunks = append(chunks, relation.SourceChunkIDs...)
}
}
- chunks = uniqueStrings(chunks)
+ chunks = appcommon.DeduplicateTrimmed(chunks)
if len(chunks) > wikiTopicSourceChunkMax {
chunks = chunks[:wikiTopicSourceChunkMax]
}
@@ -1685,8 +1685,8 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
continue
}
item := e
- item.Aliases = uniqueStrings(item.Aliases)
- item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)
+ item.Aliases = appcommon.DeduplicateTrimmed(item.Aliases)
+ item.SourceChunkIDs = appcommon.DeduplicateTrimmed(item.SourceChunkIDs)
entities[key] = &item
}
for _, c := range ex.Concepts {
@@ -1701,11 +1701,11 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
item := c
item.Term = strings.TrimSpace(item.Term)
item.Definition = strings.TrimSpace(item.Definition)
- item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)
+ item.SourceChunkIDs = appcommon.DeduplicateTrimmed(item.SourceChunkIDs)
concepts[key] = &item
}
for _, c := range ex.Claims {
- c.SourceChunkIDs = uniqueStrings(c.SourceChunkIDs)
+ c.SourceChunkIDs = appcommon.DeduplicateTrimmed(c.SourceChunkIDs)
out.Claims = append(out.Claims, c)
}
for _, r := range ex.Relations {
@@ -1714,7 +1714,7 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
out.Relations[cur].SourceChunkIDs = mergeStrings(out.Relations[cur].SourceChunkIDs, r.SourceChunkIDs)
continue
}
- r.SourceChunkIDs = uniqueStrings(r.SourceChunkIDs)
+ r.SourceChunkIDs = appcommon.DeduplicateTrimmed(r.SourceChunkIDs)
relationIndexes[key] = len(out.Relations)
out.Relations = append(out.Relations, r)
}
@@ -1732,7 +1732,7 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
continue
}
t.Description = strings.TrimSpace(t.Description)
- t.SourceChunkIDs = uniqueStrings(t.SourceChunkIDs)
+ t.SourceChunkIDs = appcommon.DeduplicateTrimmed(t.SourceChunkIDs)
topic := t
topics[key] = &topic
}
@@ -1876,7 +1876,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPla
continue
}
if idx, ok := existing[page.Slug]; ok {
- out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))
+ out[idx].RelatedKB = appcommon.DeduplicateTrimmed(append(out[idx].RelatedKB, page.RelatedKB...))
continue
}
key := wikiTitleKey(page.PageType, page.Title)
@@ -1884,7 +1884,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPla
key = page.PageType + "\x00" + page.Slug
}
if idx, ok := byTitle[key]; ok {
- out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))
+ out[idx].RelatedKB = appcommon.DeduplicateTrimmed(append(out[idx].RelatedKB, page.RelatedKB...))
continue
}
idx := len(out)
@@ -2041,8 +2041,8 @@ func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {
if page.Topic == "" {
page.Topic = common.GeneralWikiTopic
}
- page.EntityNames = uniqueStrings(page.EntityNames)
- page.RelatedKB = uniqueStrings(page.RelatedKB)
+ page.EntityNames = appcommon.DeduplicateTrimmed(page.EntityNames)
+ page.RelatedKB = appcommon.DeduplicateTrimmed(page.RelatedKB)
page.Sections = normalizeWikiPlanSections(page.Sections)
if page.Priority <= 0 {
page.Priority = 99
@@ -2057,7 +2057,7 @@ func normalizeWikiPlanSections(sections []wikiPlanSection) []wikiPlanSection {
out := make([]wikiPlanSection, 0, len(sections))
for _, sec := range sections {
sec.Heading = strings.TrimSpace(sec.Heading)
- sec.Points = uniqueStrings(sec.Points)
+ sec.Points = appcommon.DeduplicateTrimmed(sec.Points)
if sec.Heading == "" {
continue
}
@@ -2176,8 +2176,8 @@ func (p wikiPlan) firstPageCandidate() wikiPlanPage {
Title: p.Title,
PageType: p.PageType,
Topic: p.Topic,
- EntityNames: uniqueStrings(p.Entities),
- RelatedKB: uniqueStrings(p.Related),
+ EntityNames: appcommon.DeduplicateTrimmed(p.Entities),
+ RelatedKB: appcommon.DeduplicateTrimmed(p.Related),
Priority: 1,
Lead: p.Lead,
Sections: p.Sections,
@@ -2414,7 +2414,7 @@ func normalizeWikiTopics(topics []wikiTopic) []wikiTopic {
continue
}
topic.Description = strings.TrimSpace(topic.Description)
- topic.SourceChunkIDs = uniqueStrings(topic.SourceChunkIDs)
+ topic.SourceChunkIDs = appcommon.DeduplicateTrimmed(topic.SourceChunkIDs)
if index, exists := indexes[key]; exists {
if out[index].Description == "" {
out[index].Description = topic.Description
@@ -2468,7 +2468,7 @@ func firstNumber(v any) float64 {
func parseWikiStrings(raw any) []string {
switch v := raw.(type) {
case []string:
- return uniqueStrings(v)
+ return appcommon.DeduplicateTrimmed(v)
case []any:
out := make([]string, 0, len(v))
for _, item := range v {
@@ -2476,7 +2476,7 @@ func parseWikiStrings(raw any) []string {
out = append(out, s)
}
}
- return uniqueStrings(out)
+ return appcommon.DeduplicateTrimmed(out)
case string:
if s := strings.TrimSpace(v); s != "" {
return []string{s}
@@ -2551,7 +2551,7 @@ type wikiEvidenceItem struct {
}
func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityByName, conceptByTerm map[string]wikiExtractItem) []wikiEvidenceItem {
- rawNames := uniqueStrings(append([]string{}, planItem.EntityNames...))
+ rawNames := appcommon.DeduplicateTrimmed(append([]string{}, planItem.EntityNames...))
if len(rawNames) == 0 {
if t := strings.TrimSpace(planItem.Title); t != "" {
rawNames = []string{t}
@@ -2599,7 +2599,7 @@ func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityB
Statement: claim.Statement,
Subject: claim.Subject,
Confidence: firstNonEmpty(claim.Confidence, "explicit"),
- ChunkIDs: uniqueStrings(claim.SourceChunkIDs),
+ ChunkIDs: appcommon.DeduplicateTrimmed(claim.SourceChunkIDs),
})
}
if len(evidence) > 0 {
@@ -2647,7 +2647,7 @@ type wikiExtractItem struct {
func buildWikiEntityLookup(items []wikiEntity) map[string]wikiExtractItem {
out := map[string]wikiExtractItem{}
for _, item := range items {
- hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}
+ hit := wikiExtractItem{SourceChunkIDs: appcommon.DeduplicateTrimmed(item.SourceChunkIDs)}
name := strings.TrimSpace(item.Name)
if name != "" {
out[strings.ToLower(name)] = hit
@@ -2664,7 +2664,7 @@ func buildWikiEntityLookup(items []wikiEntity) map[string]wikiExtractItem {
func buildWikiConceptLookup(items []wikiConcept) map[string]wikiExtractItem {
out := map[string]wikiExtractItem{}
for _, item := range items {
- hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}
+ hit := wikiExtractItem{SourceChunkIDs: appcommon.DeduplicateTrimmed(item.SourceChunkIDs)}
term := strings.TrimSpace(item.Term)
if term != "" {
out[strings.ToLower(term)] = hit
@@ -3073,25 +3073,8 @@ func firstNonEmpty(vals ...string) string {
return ""
}
-func uniqueStrings(in []string) []string {
- if len(in) == 0 {
- return nil
- }
- seen := map[string]bool{}
- out := make([]string, 0, len(in))
- for _, s := range in {
- s = strings.TrimSpace(s)
- if s == "" || seen[s] {
- continue
- }
- seen[s] = true
- out = append(out, s)
- }
- return out
-}
-
func mergeStrings(a, b []string) []string {
- return uniqueStrings(append(append([]string{}, a...), b...))
+ return appcommon.DeduplicateTrimmed(append(append([]string{}, a...), b...))
}
func normalizedStringSet(items []string) map[string]struct{} {
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 2347404c4a..4f4797caa6 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -672,7 +672,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
for _, entry := range disabled {
retractedDocIDs = append(retractedDocIDs, entry.DocID)
}
- retractedDocIDs = uniqueSortedStrings(retractedDocIDs)
+ retractedDocIDs = common.DeduplicateSorted(retractedDocIDs)
wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, retractedDocIDs)
if err != nil {
return err
@@ -1237,7 +1237,7 @@ func completedDocsWithoutNavInput(completed []BacklogEntry, inputs []nav.UpsertD
withoutNav = append(withoutNav, entry.DocID)
}
}
- return uniqueSortedStrings(withoutNav)
+ return common.DeduplicateSorted(withoutNav)
}
func copyMeta(input map[string]any) map[string]any {
@@ -1794,8 +1794,8 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {
}
b.Description += desc
}
- b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})
- b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))
+ b.SourceDocIDs = common.AppendUnique(b.SourceDocIDs, []string{p.DocID})
+ b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))
// mention_count_int: sum the per-entity mention counts (Python
// _struct_merge_graph_entities sums mention_count across the merged entities).
// Each entity product carries mention_count ≥ 1 (structure compile.go L231);
@@ -1848,22 +1848,6 @@ func structureProductDescription(content string) string {
return content
}
-// appendUnique appends only values not already present, preserving order.
-func appendUnique(dst, values []string) []string {
- seen := make(map[string]bool, len(dst)+len(values))
- for _, v := range dst {
- seen[v] = true
- }
- for _, v := range values {
- if v == "" || seen[v] {
- continue
- }
- seen[v] = true
- dst = append(dst, v)
- }
- return dst
-}
-
// upsertNavLocked places the batch's Tree/PageIndex summaries into the dataset
// navigation tree under the per-dataset write/rebuild lock, with the claim-token
// check inside the lock (B7: nav writes are claim-fenced like any other
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index d0f897a574..a65104ea48 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -25,6 +25,7 @@ import (
"strings"
"sync"
+ "ragflow/internal/common"
"ragflow/internal/engine"
enginetypes "ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -272,29 +273,16 @@ func wikiContributionSignature(product kccommon.Product) string {
Title: strings.TrimSpace(metaString(product.Meta, "title")),
Topic: kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic")),
Summary: strings.TrimSpace(metaString(product.Meta, "summary")),
- EntityNames: sortedUniqueCopy(metaStringSlice(product.Meta, "entity_names")),
- RelatedKBPages: sortedUniqueCopy(metaStringSlice(product.Meta, "related_kb_pages")),
- Outlinks: sortedUniqueCopy(metaStringSlice(product.Meta, "outlinks")),
- SourceChunkIDs: sortedUniqueCopy(metaStringSlice(product.Meta, "source_chunk_ids")),
+ EntityNames: common.DeduplicateSorted(metaStringSlice(product.Meta, "entity_names")),
+ RelatedKBPages: common.DeduplicateSorted(metaStringSlice(product.Meta, "related_kb_pages")),
+ Outlinks: common.DeduplicateSorted(metaStringSlice(product.Meta, "outlinks")),
+ SourceChunkIDs: common.DeduplicateSorted(metaStringSlice(product.Meta, "source_chunk_ids")),
}
payload, _ := json.Marshal(normalized)
sum := sha256.Sum256(payload)
return hex.EncodeToString(sum[:])
}
-func sortedUniqueCopy(values []string) []string {
- copyValues := append([]string(nil), values...)
- sort.Strings(copyValues)
- out := copyValues[:0]
- for _, value := range copyValues {
- if value == "" || len(out) > 0 && out[len(out)-1] == value {
- continue
- }
- out = append(out, value)
- }
- return out
-}
-
type wikiContributionDiffResult struct {
productsByDoc map[string][]kccommon.Product
currentByDoc map[string]wikiDocumentContribution
diff --git a/internal/ingestion/knowledge_compile/wiki_dirty.go b/internal/ingestion/knowledge_compile/wiki_dirty.go
index b09ebdca83..b186e4d6f3 100644
--- a/internal/ingestion/knowledge_compile/wiki_dirty.go
+++ b/internal/ingestion/knowledge_compile/wiki_dirty.go
@@ -19,7 +19,6 @@ import (
"context"
"encoding/json"
"fmt"
- "sort"
"sync"
"time"
@@ -196,27 +195,10 @@ func processWikiDirty(ctx context.Context, db *gorm.DB, owner string, row entity
func parseDirtyChunkIDs(raw string) []string {
var ids []string
_ = json.Unmarshal([]byte(raw), &ids)
- return uniqueSortedStrings(ids)
+ return common.DeduplicateSorted(ids)
}
func marshalDirtyChunkIDs(ids []string) string {
- payload, _ := json.Marshal(uniqueSortedStrings(ids))
+ payload, _ := json.Marshal(common.DeduplicateSorted(ids))
return string(payload)
}
-
-func uniqueSortedStrings(values []string) []string {
- seen := make(map[string]struct{}, len(values))
- out := make([]string, 0, len(values))
- for _, value := range values {
- if value == "" {
- continue
- }
- if _, ok := seen[value]; ok {
- continue
- }
- seen[value] = struct{}{}
- out = append(out, value)
- }
- sort.Strings(out)
- return out
-}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 135000d768..f093e317f0 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -652,7 +652,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
compileKinds := make([]string, 0, len(buckets))
for i := range buckets {
byIdentity[structureBucketIdentity(buckets[i])] = i
- compileKinds = appendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})
+ compileKinds = common.AppendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})
}
if len(byIdentity) == 0 {
return nil, nil
@@ -691,8 +691,8 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
if !structureBucketWritable(*bucket) {
continue
}
- bucket.SourceDocIDs = appendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))
- bucket.SourceChunkIDs = appendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))
+ bucket.SourceDocIDs = common.AppendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))
+ bucket.SourceChunkIDs = common.AppendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))
payload := structureRowPayload(row)
if bucket.FromEntity == "" && bucket.ToEntity == "" {
existingType := structureString(row["entity_type_kwd"])
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 0cec413340..df69504b0b 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -146,7 +146,7 @@ func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.Wik
if len(affectedSlugs) == 0 && len(removedSlugs) == 0 {
return persistDirtyWikiActiveStates(ctx, request, activeStates)
}
- return replaceDirtyWikiProducts(ctx, request, compiled, uniqueStrings(affectedSlugs), uniqueStrings(removedSlugs), activeStates, false)
+ return replaceDirtyWikiProducts(ctx, request, compiled, appcommon.DeduplicateTrimmed(affectedSlugs), appcommon.DeduplicateTrimmed(removedSlugs), activeStates, false)
}
func loadWikiCompilerParams(ctx context.Context, doc *entity.Document) (map[string]any, error) {
@@ -206,13 +206,13 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[str
groupIDs := stringValues(params["compilation_template_group_id"])
groupIDs = append(groupIDs, stringValues(params["compilation_template_group_ids"])...)
if len(groupIDs) > 0 {
- resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, kcWiringDeps.DB, tenantID, uniqueStrings(groupIDs))
+ resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, kcWiringDeps.DB, tenantID, appcommon.DeduplicateTrimmed(groupIDs))
if err != nil {
return nil, err
}
ids = append(ids, resolved...)
}
- ids = uniqueStrings(ids)
+ ids = appcommon.DeduplicateTrimmed(ids)
if len(ids) == 0 {
return nil, nil
}
@@ -231,7 +231,7 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[str
wikiIDs = append(wikiIDs, template.ID)
}
}
- return uniqueStrings(wikiIDs), nil
+ return appcommon.DeduplicateTrimmed(wikiIDs), nil
}
func loadActiveSourceChunks(ctx context.Context, request knowledge_compile.WikiDirtyRequest) ([]map[string]any, error) {
@@ -353,7 +353,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
removedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))
}
}
- removedSlugs = uniqueStrings(removedSlugs)
+ removedSlugs = appcommon.DeduplicateTrimmed(removedSlugs)
}
staleIDs := make([]string, 0)
if existing != nil {
@@ -516,23 +516,6 @@ func stringValues(value any) []string {
return nil
}
-func uniqueStrings(values []string) []string {
- seen := make(map[string]struct{}, len(values))
- result := make([]string, 0, len(values))
- for _, value := range values {
- value = strings.TrimSpace(value)
- if value == "" {
- continue
- }
- if _, ok := seen[value]; ok {
- continue
- }
- seen[value] = struct{}{}
- result = append(result, value)
- }
- return result
-}
-
// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed by
// the compilation_template DAO. Without it, any config carrying
// compilation_template_group_id would fail loud at runtime (the component