Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(retrieval): search each tenant index once to stop duplicate chunks (#20731)

Vedant2438c0d6e264
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index d301b8e319..fcf241c5fb 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -1288,14 +1288,16 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
}, nil
}
-// buildIndexNames creates index names for the given tenant IDs.
+// buildIndexNames creates one index name per distinct tenant ID.
// Each tenantID may be a comma-separated list.
func buildIndexNames(tenantIDs []string) []string {
var indexNames []string
+ seen := make(map[string]bool)
for _, tid := range tenantIDs {
for part := range strings.SplitSeq(tid, ",") {
part = strings.TrimSpace(part)
- if part != "" {
+ if part != "" && !seen[part] {
+ seen[part] = true
indexNames = append(indexNames, fmt.Sprintf("ragflow_%s", part))
}
}
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 76263f9c5e..8a6dc45861 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -635,3 +635,61 @@ func TestSearchKeepsPythonFusionWeightForElasticsearch(t *testing.T) {
t.Fatalf("expected Elasticsearch weights=%s, got %v", esFusionWeights, got)
}
}
+
+// perIndexEngine searches each listed index separately, like Elasticsearch and Infinity.
+type perIndexEngine struct {
+ retrievalCountEngine
+ indexNames []string
+}
+
+func (e *perIndexEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.indexNames = req.IndexNames
+ merged := &types.SearchResult{}
+ for range req.IndexNames {
+ result, err := e.retrievalCountEngine.Search(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ merged.Chunks = append(merged.Chunks, result.Chunks...)
+ merged.Total += result.Total
+ }
+ return merged, nil
+}
+
+func TestRetrievalSearchesEachTenantIndexOnce(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &perIndexEngine{retrievalCountEngine: retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "chunk-a", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
+ {"id": "chunk-b", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
+ }}}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ aggs := false
+
+ // Two datasets owned by the same tenant.
+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1", "tenant-1"},
+ KbIDs: []string{"kb-1", "kb-2"},
+ Page: 1,
+ PageSize: 10,
+ SimilarityThreshold: &threshold,
+ Aggs: &aggs,
+ })
+ if err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ var ids []string
+ for _, chunk := range result.Chunks {
+ ids = append(ids, chunk["chunk_id"].(string))
+ }
+ if !slices.Equal(ids, []string{"chunk-a", "chunk-b"}) || result.Total != 2 {
+ t.Fatalf("chunks = %v (total %d), want [chunk-a chunk-b] (total 2)", ids, result.Total)
+ }
+ if !slices.Equal(docEngine.indexNames, []string{"ragflow_tenant-1"}) {
+ t.Fatalf("index names = %v, want [ragflow_tenant-1]", docEngine.indexNames)
+ }
+}