RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): search each tenant index once to stop duplicate chunks (#20731)
Vedant2438c0d6e264
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex d301b8e319..fcf241c5fb 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -1288,14 +1288,16 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri}, nil}-// buildIndexNames creates index names for the given tenant IDs.+// buildIndexNames creates one index name per distinct tenant ID.// Each tenantID may be a comma-separated list.func buildIndexNames(tenantIDs []string) []string {var indexNames []string+ seen := make(map[string]bool)for _, tid := range tenantIDs {for part := range strings.SplitSeq(tid, ",") {part = strings.TrimSpace(part)- if part != "" {+ if part != "" && !seen[part] {+ seen[part] = trueindexNames = append(indexNames, fmt.Sprintf("ragflow_%s", part))}}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 76263f9c5e..8a6dc45861 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -635,3 +635,61 @@ func TestSearchKeepsPythonFusionWeightForElasticsearch(t *testing.T) {t.Fatalf("expected Elasticsearch weights=%s, got %v", esFusionWeights, got)}}++// perIndexEngine searches each listed index separately, like Elasticsearch and Infinity.+type perIndexEngine struct {+ retrievalCountEngine+ indexNames []string+}++func (e *perIndexEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.indexNames = req.IndexNames+ merged := &types.SearchResult{}+ for range req.IndexNames {+ result, err := e.retrievalCountEngine.Search(ctx, req)+ if err != nil {+ return nil, err+ }+ merged.Chunks = append(merged.Chunks, result.Chunks...)+ merged.Total += result.Total+ }+ return merged, nil+}++func TestRetrievalSearchesEachTenantIndexOnce(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &perIndexEngine{retrievalCountEngine: retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-a", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ {"id": "chunk-b", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},+ }}}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ aggs := false++ // Two datasets owned by the same tenant.+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1", "tenant-1"},+ KbIDs: []string{"kb-1", "kb-2"},+ Page: 1,+ PageSize: 10,+ SimilarityThreshold: &threshold,+ Aggs: &aggs,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ var ids []string+ for _, chunk := range result.Chunks {+ ids = append(ids, chunk["chunk_id"].(string))+ }+ if !slices.Equal(ids, []string{"chunk-a", "chunk-b"}) || result.Total != 2 {+ t.Fatalf("chunks = %v (total %d), want [chunk-a chunk-b] (total 2)", ids, result.Total)+ }+ if !slices.Equal(docEngine.indexNames, []string{"ragflow_tenant-1"}) {+ t.Fatalf("index names = %v, want [ragflow_tenant-1]", docEngine.indexNames)+ }+}