RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): consolidate unique* string dedup helpers into common (#21961)
xugangqiang8b8ccbf4780c
diff --git a/internal/common/dedup_test.go b/internal/common/dedup_test.gonew file mode 100644index 0000000000..32fd3b660a--- /dev/null+++ b/internal/common/dedup_test.go@@ -0,0 +1,99 @@+package common++import (+ "reflect"+ "testing"+)++// TestDeduplicateTrimmed verifies trim + drop-empty + dedup + order-preserving,+// and the "empty input (or all dropped) returns nil" contract.+func TestDeduplicateTrimmed(t *testing.T) {+ cases := []struct {+ name string+ in []string+ want []string+ }{+ {"nil input returns nil", nil, nil},+ {"empty slice returns nil", []string{}, nil},+ {"single element", []string{"a"}, []string{"a"}},+ {"dedup preserves order", []string{"b", "a", "b", "c", "a"}, []string{"b", "a", "c"}},+ {"trims surrounding space", []string{" a ", "a", " b"}, []string{"a", "b"}},+ {"drops empty string", []string{"", "a", ""}, []string{"a"}},+ {"drops whitespace-only", []string{" ", " ", "a"}, []string{"a"}},+ // Adversarial: all entries collapse to empty -> nil, never []string{}.+ {"all empty collapses to nil", []string{"", " ", " "}, nil},+ // Adversarial: trim makes two distinct-looking entries identical.+ {"trim collapses duplicates", []string{"x ", " x", "x"}, []string{"x"}},+ // Adversarial: unicode whitespace must be trimmed.+ {"unicode whitespace trimmed", []string{"\ta\n", "a"}, []string{"a"}},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ got := DeduplicateTrimmed(c.in)+ if !reflect.DeepEqual(got, c.want) {+ t.Fatalf("DeduplicateTrimmed(%v) = %v, want %v", c.in, got, c.want)+ }+ if c.want == nil && got != nil {+ t.Fatalf("expected nil result for %v, got %v", c.in, got)+ }+ })+ }+}++// TestDeduplicateSorted verifies drop-empty + dedup + sort, that it does NOT+// mutate the caller's slice, and the empty->nil contract.+func TestDeduplicateSorted(t *testing.T) {+ // Sort correctness + drop empty (no trim: this variant does NOT trim).+ in := []string{"c", "", "a", "b", "a"}+ want := []string{"a", "b", "c"}+ got := DeduplicateSorted(in)+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("DeduplicateSorted(%v) = %v, want %v", in, got, want)+ }+ // Adversarial: does NOT trim, so " a" and "a" stay distinct.+ inNoTrim := []string{" a", "a"}+ if got := DeduplicateSorted(inNoTrim); !reflect.DeepEqual(got, []string{" a", "a"}) {+ t.Fatalf("DeduplicateSorted(%v) = %v, want [ a a] (no trim)", inNoTrim, got)+ }+ // Does not mutate the input slice.+ if in[0] != "c" || in[1] != "" {+ t.Fatalf("DeduplicateSorted mutated input: %v", in)+ }+ // Empty contract.+ if got := DeduplicateSorted(nil); got != nil {+ t.Fatalf("DeduplicateSorted(nil) = %v, want nil", got)+ }+ if got := DeduplicateSorted([]string{}); got != nil {+ t.Fatalf("DeduplicateSorted([]string{}) = %v, want nil", got)+ }+ if got := DeduplicateSorted([]string{"", ""}); got != nil {+ t.Fatalf("DeduplicateSorted(all empty) = %v, want nil", got)+ }+}++// TestAppendUnique verifies append-into-dst, drop-empty, dedup against existing+// dst entries, and order preservation.+func TestAppendUnique(t *testing.T) {+ // Appends new, skips empties and already-present.+ dst := []string{"a", "b"}+ got := AppendUnique(dst, []string{"b", "", "c", "a"})+ want := []string{"a", "b", "c"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("AppendUnique(%v, ...) = %v, want %v", dst, got, want)+ }+ // Appending to a nil dst yields the new, deduped, non-empty items.+ gotNil := AppendUnique(nil, []string{"x", "x", "y"})+ if !reflect.DeepEqual(gotNil, []string{"x", "y"}) {+ t.Fatalf("AppendUnique(nil, ...) = %v, want [x y]", gotNil)+ }+ // Appending only empties/dupes leaves dst unchanged (and nil stays nil).+ gotEmpty := AppendUnique(nil, []string{"", ""})+ if gotEmpty != nil {+ t.Fatalf("AppendUnique(nil, empties) = %v, want nil", gotEmpty)+ }+ // Adversarial: AppendUnique does NOT trim, so " x" and "x" both stay.+ gotNoTrim := AppendUnique(nil, []string{" x", "x"})+ if !reflect.DeepEqual(gotNoTrim, []string{" x", "x"}) {+ t.Fatalf("AppendUnique(nil, [ x x]) = %v, want [ x x] (no trim)", gotNoTrim)+ }+}diff --git a/internal/common/idenfication.go b/internal/common/idenfication.goindex 7306bbccb2..fcd68c7252 100644--- a/internal/common/idenfication.go+++ b/internal/common/idenfication.go@@ -16,6 +16,11 @@package common+import (+ "sort"+ "strings"+)+func Deduplicate(names []string) []string {if names == nil {return nil@@ -33,3 +38,60 @@ func Deduplicate(names []string) []string {return result}++// DeduplicateTrimmed removes surrounding whitespace, drops empty strings, and+// removes duplicates while preserving order. Returns nil when the result is+// empty.+func DeduplicateTrimmed(values []string) []string {+ seen := make(map[string]bool)+ result := make([]string, 0, len(values))+ for _, v := range values {+ v = strings.TrimSpace(v)+ if v == "" || seen[v] {+ continue+ }+ seen[v] = true+ result = append(result, v)+ }+ if len(result) == 0 {+ return nil+ }+ return result+}++// DeduplicateSorted drops empty strings, removes duplicates, and sorts the+// result. It does not mutate the input slice. Returns nil when empty.+func DeduplicateSorted(values []string) []string {+ seen := make(map[string]bool)+ result := make([]string, 0, len(values))+ for _, v := range values {+ if v == "" || seen[v] {+ continue+ }+ seen[v] = true+ result = append(result, v)+ }+ if len(result) == 0 {+ return nil+ }+ out := append([]string(nil), result...)+ sort.Strings(out)+ return out+}++// AppendUnique appends values not already present in dst (empty strings+// dropped), preserving order, and returns the extended dst.+func AppendUnique(dst, values []string) []string {+ seen := make(map[string]bool, len(dst)+len(values))+ for _, v := range dst {+ seen[v] = true+ }+ for _, v := range values {+ if v == "" || seen[v] {+ continue+ }+ seen[v] = true+ dst = append(dst, v)+ }+ return dst+}diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex a6117b0dbb..57a451df21 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -269,10 +269,14 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in)result := mergeChunks(inputs, compiled)if len(out.AffectedPageSlugs) > 0 {- result["wiki_affected_slugs"] = uniqueSorted(out.AffectedPageSlugs)+ affected := clog.DeduplicateTrimmed(out.AffectedPageSlugs)+ sort.Strings(affected)+ result["wiki_affected_slugs"] = affected}if len(out.RemovedPageSlugs) > 0 {- result["wiki_removed_slugs"] = uniqueSorted(out.RemovedPageSlugs)+ removed := clog.DeduplicateTrimmed(out.RemovedPageSlugs)+ sort.Strings(removed)+ result["wiki_removed_slugs"] = removed}if len(out.WikiActiveStates) > 0 {result["wiki_active_map_states"] = wikiActiveStateValues(out.WikiActiveStates)@@ -301,24 +305,6 @@ func wikiActiveStateValues(states []common.WikiMapActiveState) []map[string]anyreturn values}-func uniqueSorted(values []string) []string {- seen := make(map[string]struct{}, len(values))- result := make([]string, 0, len(values))- for _, value := range values {- value = strings.TrimSpace(value)- if value == "" {- continue- }- if _, ok := seen[value]; ok {- continue- }- seen[value] = struct{}{}- result = append(result, value)- }- sort.Strings(result)- return result-}-// resolveTemplateSpecs resolves the configured compilation template spec(s) to// their TemplateInfo rows. Priority: compilation_template_id >// compilation_template_group_id. The group path resolves the group to itsdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.goindex 946807382b..3868718732 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go@@ -166,8 +166,8 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []comcontent = page.ContentRaw}summary := summarizeMarkdown(content, title)- sourceChunkIDs := uniqueStrings(page.SourceChunkIDs)- sourceDocIDs := uniqueStrings(page.SourceDocIDs)+ sourceChunkIDs := appcommon.DeduplicateTrimmed(page.SourceChunkIDs)+ sourceDocIDs := appcommon.DeduplicateTrimmed(page.SourceDocIDs)if len(sourceDocIDs) == 0 && docID != "" {sourceDocIDs = []string{docID}}@@ -186,9 +186,9 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com"page_type": pageType,"topic": common.NormalizeWikiTopicPath(page.Topic),"summary": summary,- "entity_names": uniqueStrings(page.EntityNames),- "related_kb_pages": uniqueStrings(page.RelatedKBPages),- "outlinks": uniqueStrings(page.Outlinks),+ "entity_names": appcommon.DeduplicateTrimmed(page.EntityNames),+ "related_kb_pages": appcommon.DeduplicateTrimmed(page.RelatedKBPages),+ "outlinks": appcommon.DeduplicateTrimmed(page.Outlinks),"source_chunk_ids": sourceChunkIDs,"source_doc_ids": sourceDocIDs,"content_md_raw": page.ContentRaw,diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex f7fb750a89..64cc98ab15 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -768,7 +768,7 @@ func buildWikiEntityPlanPages(reduced wikiExtract) []wikiPlanPage {Title: name,PageType: "entity",Topic: selectWikiIdentityTopic(name, "entity", sourceChunkIDsForWikiIdentity(name, entity.SourceChunkIDs, reduced.Claims), reduced.Topics),- EntityNames: uniqueStrings(append([]string{name}, entity.Aliases...)),+ EntityNames: appcommon.DeduplicateTrimmed(append([]string{name}, entity.Aliases...)),Priority: len(pages) + 1,})}@@ -797,7 +797,7 @@ func sourceChunkIDsForWikiIdentity(name string, sourceChunkIDs []string, claimsids = append(ids, claim.SourceChunkIDs...)}}- return uniqueStrings(ids)+ return appcommon.DeduplicateTrimmed(ids)}func selectWikiIdentityTopic(name, pageType string, sourceChunkIDs []string, topics []wikiTopic) string {@@ -1065,8 +1065,8 @@ func (p *wikiPipeline) runRefinePage(PageType: firstNonEmpty(planItem.PageType, "concept"),Topic: topic,Action: firstNonEmpty(planItem.Action, "CREATE"),- EntityNames: uniqueStrings(planItem.EntityNames),- RelatedKBPages: uniqueStrings(planItem.RelatedKB),+ EntityNames: appcommon.DeduplicateTrimmed(planItem.EntityNames),+ RelatedKBPages: appcommon.DeduplicateTrimmed(planItem.RelatedKB),ContentRaw: contentRaw,Content: contentRendered,Outlinks: outlinks,@@ -1177,7 +1177,7 @@ func assembleWikiPlanRelatedPages(pages []wikiPlanPage, relations []wikiRelationpageByName := make(map[string][]int)for i, page := range pages {- names := uniqueStrings(append([]string{}, page.EntityNames...))+ names := appcommon.DeduplicateTrimmed(append([]string{}, page.EntityNames...))if len(names) == 0 && strings.TrimSpace(page.Title) != "" {names = []string{page.Title}}@@ -1271,7 +1271,7 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {if len(existing.RoutedEntityNames) > 0 {// An explicit route membership set is authoritative: this is a// migration, so do not union members that the LLM removed.- page.EntityNames = uniqueStrings(existing.RoutedEntityNames)+ page.EntityNames = appcommon.DeduplicateTrimmed(existing.RoutedEntityNames)} else {page.EntityNames = mergeStrings(page.EntityNames, existing.EntityNames)}@@ -1426,7 +1426,7 @@ func (p *wikiPipeline) sourceChunksForPlanPage(page wikiPlanPage) []string {chunks = append(chunks, relation.SourceChunkIDs...)}}- chunks = uniqueStrings(chunks)+ chunks = appcommon.DeduplicateTrimmed(chunks)if len(chunks) > wikiTopicSourceChunkMax {chunks = chunks[:wikiTopicSourceChunkMax]}@@ -1685,8 +1685,8 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {continue}item := e- item.Aliases = uniqueStrings(item.Aliases)- item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)+ item.Aliases = appcommon.DeduplicateTrimmed(item.Aliases)+ item.SourceChunkIDs = appcommon.DeduplicateTrimmed(item.SourceChunkIDs)entities[key] = &item}for _, c := range ex.Concepts {@@ -1701,11 +1701,11 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {item := citem.Term = strings.TrimSpace(item.Term)item.Definition = strings.TrimSpace(item.Definition)- item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)+ item.SourceChunkIDs = appcommon.DeduplicateTrimmed(item.SourceChunkIDs)concepts[key] = &item}for _, c := range ex.Claims {- c.SourceChunkIDs = uniqueStrings(c.SourceChunkIDs)+ c.SourceChunkIDs = appcommon.DeduplicateTrimmed(c.SourceChunkIDs)out.Claims = append(out.Claims, c)}for _, r := range ex.Relations {@@ -1714,7 +1714,7 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {out.Relations[cur].SourceChunkIDs = mergeStrings(out.Relations[cur].SourceChunkIDs, r.SourceChunkIDs)continue}- r.SourceChunkIDs = uniqueStrings(r.SourceChunkIDs)+ r.SourceChunkIDs = appcommon.DeduplicateTrimmed(r.SourceChunkIDs)relationIndexes[key] = len(out.Relations)out.Relations = append(out.Relations, r)}@@ -1732,7 +1732,7 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {continue}t.Description = strings.TrimSpace(t.Description)- t.SourceChunkIDs = uniqueStrings(t.SourceChunkIDs)+ t.SourceChunkIDs = appcommon.DeduplicateTrimmed(t.SourceChunkIDs)topic := ttopics[key] = &topic}@@ -1876,7 +1876,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlacontinue}if idx, ok := existing[page.Slug]; ok {- out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))+ out[idx].RelatedKB = appcommon.DeduplicateTrimmed(append(out[idx].RelatedKB, page.RelatedKB...))continue}key := wikiTitleKey(page.PageType, page.Title)@@ -1884,7 +1884,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlakey = page.PageType + "\x00" + page.Slug}if idx, ok := byTitle[key]; ok {- out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))+ out[idx].RelatedKB = appcommon.DeduplicateTrimmed(append(out[idx].RelatedKB, page.RelatedKB...))continue}idx := len(out)@@ -2041,8 +2041,8 @@ func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {if page.Topic == "" {page.Topic = common.GeneralWikiTopic}- page.EntityNames = uniqueStrings(page.EntityNames)- page.RelatedKB = uniqueStrings(page.RelatedKB)+ page.EntityNames = appcommon.DeduplicateTrimmed(page.EntityNames)+ page.RelatedKB = appcommon.DeduplicateTrimmed(page.RelatedKB)page.Sections = normalizeWikiPlanSections(page.Sections)if page.Priority <= 0 {page.Priority = 99@@ -2057,7 +2057,7 @@ func normalizeWikiPlanSections(sections []wikiPlanSection) []wikiPlanSection {out := make([]wikiPlanSection, 0, len(sections))for _, sec := range sections {sec.Heading = strings.TrimSpace(sec.Heading)- sec.Points = uniqueStrings(sec.Points)+ sec.Points = appcommon.DeduplicateTrimmed(sec.Points)if sec.Heading == "" {continue}@@ -2176,8 +2176,8 @@ func (p wikiPlan) firstPageCandidate() wikiPlanPage {Title: p.Title,PageType: p.PageType,Topic: p.Topic,- EntityNames: uniqueStrings(p.Entities),- RelatedKB: uniqueStrings(p.Related),+ EntityNames: appcommon.DeduplicateTrimmed(p.Entities),+ RelatedKB: appcommon.DeduplicateTrimmed(p.Related),Priority: 1,Lead: p.Lead,Sections: p.Sections,@@ -2414,7 +2414,7 @@ func normalizeWikiTopics(topics []wikiTopic) []wikiTopic {continue}topic.Description = strings.TrimSpace(topic.Description)- topic.SourceChunkIDs = uniqueStrings(topic.SourceChunkIDs)+ topic.SourceChunkIDs = appcommon.DeduplicateTrimmed(topic.SourceChunkIDs)if index, exists := indexes[key]; exists {if out[index].Description == "" {out[index].Description = topic.Description@@ -2468,7 +2468,7 @@ func firstNumber(v any) float64 {func parseWikiStrings(raw any) []string {switch v := raw.(type) {case []string:- return uniqueStrings(v)+ return appcommon.DeduplicateTrimmed(v)case []any:out := make([]string, 0, len(v))for _, item := range v {@@ -2476,7 +2476,7 @@ func parseWikiStrings(raw any) []string {out = append(out, s)}}- return uniqueStrings(out)+ return appcommon.DeduplicateTrimmed(out)case string:if s := strings.TrimSpace(v); s != "" {return []string{s}@@ -2551,7 +2551,7 @@ type wikiEvidenceItem struct {}func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityByName, conceptByTerm map[string]wikiExtractItem) []wikiEvidenceItem {- rawNames := uniqueStrings(append([]string{}, planItem.EntityNames...))+ rawNames := appcommon.DeduplicateTrimmed(append([]string{}, planItem.EntityNames...))if len(rawNames) == 0 {if t := strings.TrimSpace(planItem.Title); t != "" {rawNames = []string{t}@@ -2599,7 +2599,7 @@ func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityBStatement: claim.Statement,Subject: claim.Subject,Confidence: firstNonEmpty(claim.Confidence, "explicit"),- ChunkIDs: uniqueStrings(claim.SourceChunkIDs),+ ChunkIDs: appcommon.DeduplicateTrimmed(claim.SourceChunkIDs),})}if len(evidence) > 0 {@@ -2647,7 +2647,7 @@ type wikiExtractItem struct {func buildWikiEntityLookup(items []wikiEntity) map[string]wikiExtractItem {out := map[string]wikiExtractItem{}for _, item := range items {- hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}+ hit := wikiExtractItem{SourceChunkIDs: appcommon.DeduplicateTrimmed(item.SourceChunkIDs)}name := strings.TrimSpace(item.Name)if name != "" {out[strings.ToLower(name)] = hit@@ -2664,7 +2664,7 @@ func buildWikiEntityLookup(items []wikiEntity) map[string]wikiExtractItem {func buildWikiConceptLookup(items []wikiConcept) map[string]wikiExtractItem {out := map[string]wikiExtractItem{}for _, item := range items {- hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}+ hit := wikiExtractItem{SourceChunkIDs: appcommon.DeduplicateTrimmed(item.SourceChunkIDs)}term := strings.TrimSpace(item.Term)if term != "" {out[strings.ToLower(term)] = hit@@ -3073,25 +3073,8 @@ func firstNonEmpty(vals ...string) string {return ""}-func uniqueStrings(in []string) []string {- if len(in) == 0 {- return nil- }- seen := map[string]bool{}- out := make([]string, 0, len(in))- for _, s := range in {- s = strings.TrimSpace(s)- if s == "" || seen[s] {- continue- }- seen[s] = true- out = append(out, s)- }- return out-}-func mergeStrings(a, b []string) []string {- return uniqueStrings(append(append([]string{}, a...), b...))+ return appcommon.DeduplicateTrimmed(append(append([]string{}, a...), b...))}func normalizedStringSet(items []string) map[string]struct{} {diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 2347404c4a..4f4797caa6 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -672,7 +672,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, efor _, entry := range disabled {retractedDocIDs = append(retractedDocIDs, entry.DocID)}- retractedDocIDs = uniqueSortedStrings(retractedDocIDs)+ retractedDocIDs = common.DeduplicateSorted(retractedDocIDs)wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, retractedDocIDs)if err != nil {return err@@ -1237,7 +1237,7 @@ func completedDocsWithoutNavInput(completed []BacklogEntry, inputs []nav.UpsertDwithoutNav = append(withoutNav, entry.DocID)}}- return uniqueSortedStrings(withoutNav)+ return common.DeduplicateSorted(withoutNav)}func copyMeta(input map[string]any) map[string]any {@@ -1794,8 +1794,8 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {}b.Description += desc}- b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})- b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))+ b.SourceDocIDs = common.AppendUnique(b.SourceDocIDs, []string{p.DocID})+ b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))// mention_count_int: sum the per-entity mention counts (Python// _struct_merge_graph_entities sums mention_count across the merged entities).// Each entity product carries mention_count ≥ 1 (structure compile.go L231);@@ -1848,22 +1848,6 @@ func structureProductDescription(content string) string {return content}-// appendUnique appends only values not already present, preserving order.-func appendUnique(dst, values []string) []string {- seen := make(map[string]bool, len(dst)+len(values))- for _, v := range dst {- seen[v] = true- }- for _, v := range values {- if v == "" || seen[v] {- continue- }- seen[v] = true- dst = append(dst, v)- }- return dst-}-// upsertNavLocked places the batch's Tree/PageIndex summaries into the dataset// navigation tree under the per-dataset write/rebuild lock, with the claim-token// check inside the lock (B7: nav writes are claim-fenced like any otherdiff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.goindex d0f897a574..a65104ea48 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -25,6 +25,7 @@ import ("strings""sync"+ "ragflow/internal/common""ragflow/internal/engine"enginetypes "ragflow/internal/engine/types"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -272,29 +273,16 @@ func wikiContributionSignature(product kccommon.Product) string {Title: strings.TrimSpace(metaString(product.Meta, "title")),Topic: kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic")),Summary: strings.TrimSpace(metaString(product.Meta, "summary")),- EntityNames: sortedUniqueCopy(metaStringSlice(product.Meta, "entity_names")),- RelatedKBPages: sortedUniqueCopy(metaStringSlice(product.Meta, "related_kb_pages")),- Outlinks: sortedUniqueCopy(metaStringSlice(product.Meta, "outlinks")),- SourceChunkIDs: sortedUniqueCopy(metaStringSlice(product.Meta, "source_chunk_ids")),+ EntityNames: common.DeduplicateSorted(metaStringSlice(product.Meta, "entity_names")),+ RelatedKBPages: common.DeduplicateSorted(metaStringSlice(product.Meta, "related_kb_pages")),+ Outlinks: common.DeduplicateSorted(metaStringSlice(product.Meta, "outlinks")),+ SourceChunkIDs: common.DeduplicateSorted(metaStringSlice(product.Meta, "source_chunk_ids")),}payload, _ := json.Marshal(normalized)sum := sha256.Sum256(payload)return hex.EncodeToString(sum[:])}-func sortedUniqueCopy(values []string) []string {- copyValues := append([]string(nil), values...)- sort.Strings(copyValues)- out := copyValues[:0]- for _, value := range copyValues {- if value == "" || len(out) > 0 && out[len(out)-1] == value {- continue- }- out = append(out, value)- }- return out-}-type wikiContributionDiffResult struct {productsByDoc map[string][]kccommon.ProductcurrentByDoc map[string]wikiDocumentContributiondiff --git a/internal/ingestion/knowledge_compile/wiki_dirty.go b/internal/ingestion/knowledge_compile/wiki_dirty.goindex b09ebdca83..b186e4d6f3 100644--- a/internal/ingestion/knowledge_compile/wiki_dirty.go+++ b/internal/ingestion/knowledge_compile/wiki_dirty.go@@ -19,7 +19,6 @@ import ("context""encoding/json""fmt"- "sort""sync""time"@@ -196,27 +195,10 @@ func processWikiDirty(ctx context.Context, db *gorm.DB, owner string, row entityfunc parseDirtyChunkIDs(raw string) []string {var ids []string_ = json.Unmarshal([]byte(raw), &ids)- return uniqueSortedStrings(ids)+ return common.DeduplicateSorted(ids)}func marshalDirtyChunkIDs(ids []string) string {- payload, _ := json.Marshal(uniqueSortedStrings(ids))+ payload, _ := json.Marshal(common.DeduplicateSorted(ids))return string(payload)}--func uniqueSortedStrings(values []string) []string {- seen := make(map[string]struct{}, len(values))- out := make([]string, 0, len(values))- for _, value := range values {- if value == "" {- continue- }- if _, ok := seen[value]; ok {- continue- }- seen[value] = struct{}{}- out = append(out, value)- }- sort.Strings(out)- return out-}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 135000d768..f093e317f0 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -652,7 +652,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, bacompileKinds := make([]string, 0, len(buckets))for i := range buckets {byIdentity[structureBucketIdentity(buckets[i])] = i- compileKinds = appendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})+ compileKinds = common.AppendUnique(compileKinds, []string{structureCompileKind(buckets[i].CompileKwd)})}if len(byIdentity) == 0 {return nil, nil@@ -691,8 +691,8 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, baif !structureBucketWritable(*bucket) {continue}- bucket.SourceDocIDs = appendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))- bucket.SourceChunkIDs = appendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))+ bucket.SourceDocIDs = common.AppendUnique(bucket.SourceDocIDs, firstStringSlice(row["source_doc_ids"]))+ bucket.SourceChunkIDs = common.AppendUnique(bucket.SourceChunkIDs, firstStringSlice(row["source_chunk_ids"]))payload := structureRowPayload(row)if bucket.FromEntity == "" && bucket.ToEntity == "" {existingType := structureString(row["entity_type_kwd"])diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 0cec413340..df69504b0b 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -146,7 +146,7 @@ func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.Wikif len(affectedSlugs) == 0 && len(removedSlugs) == 0 {return persistDirtyWikiActiveStates(ctx, request, activeStates)}- return replaceDirtyWikiProducts(ctx, request, compiled, uniqueStrings(affectedSlugs), uniqueStrings(removedSlugs), activeStates, false)+ return replaceDirtyWikiProducts(ctx, request, compiled, appcommon.DeduplicateTrimmed(affectedSlugs), appcommon.DeduplicateTrimmed(removedSlugs), activeStates, false)}func loadWikiCompilerParams(ctx context.Context, doc *entity.Document) (map[string]any, error) {@@ -206,13 +206,13 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[strgroupIDs := stringValues(params["compilation_template_group_id"])groupIDs = append(groupIDs, stringValues(params["compilation_template_group_ids"])...)if len(groupIDs) > 0 {- resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, kcWiringDeps.DB, tenantID, uniqueStrings(groupIDs))+ resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, kcWiringDeps.DB, tenantID, appcommon.DeduplicateTrimmed(groupIDs))if err != nil {return nil, err}ids = append(ids, resolved...)}- ids = uniqueStrings(ids)+ ids = appcommon.DeduplicateTrimmed(ids)if len(ids) == 0 {return nil, nil}@@ -231,7 +231,7 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[strwikiIDs = append(wikiIDs, template.ID)}}- return uniqueStrings(wikiIDs), nil+ return appcommon.DeduplicateTrimmed(wikiIDs), nil}func loadActiveSourceChunks(ctx context.Context, request knowledge_compile.WikiDirtyRequest) ([]map[string]any, error) {@@ -353,7 +353,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikremovedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))}}- removedSlugs = uniqueStrings(removedSlugs)+ removedSlugs = appcommon.DeduplicateTrimmed(removedSlugs)}staleIDs := make([]string, 0)if existing != nil {@@ -516,23 +516,6 @@ func stringValues(value any) []string {return nil}-func uniqueStrings(values []string) []string {- seen := make(map[string]struct{}, len(values))- result := make([]string, 0, len(values))- for _, value := range values {- value = strings.TrimSpace(value)- if value == "" {- continue- }- if _, ok := seen[value]; ok {- continue- }- seen[value] = struct{}{}- result = append(result, value)- }- return result-}-// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed by// the compilation_template DAO. Without it, any config carrying// compilation_template_group_id would fail loud at runtime (the component