RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): bound rerank and KNN candidate windows (#21907)
ZZZZYc2808b14fee4
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex 79217a83eb..0ae7a8cbf2 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -37,59 +37,6 @@ type SearchResult struct {Field map[string]map[string]interface{} // id -> fields}-// Rerank performs reranking based on whether a reranker model is provided-// This implements the logic from rag/nlp/search.py L404-L429-// Parameters:-// - rerankModel: the reranker model (can be nil)-// - sres: search results-// - query: the query string-// - tkWeight: weight for token similarity-// - vtWeight: weight for vector similarity-// - useInfinity: whether using Infinity engine-// - cfield: content field name (default: "content_ltks")-// - qb: QueryBuilder instance for token processing-//-// Returns:-// - sim: combined similarity scores-// - tsim: token similarity scores-// - vsim: vector similarity scores-func Rerank(- ctx context.Context,- rerankModel *models.RerankModel,- chunks []map[string]interface{},- total int,- keywords []string,- questionVector []float64,- query string,- tkWeight, vtWeight float64,- useInfinity bool,- cfield string,- qb *QueryBuilder,- rankFeature map[string]float64,- language string,-) (sim []float64, tsim []float64, vsim []float64, err error) {- // If reranker model is provided and there are results, use model reranking- if rerankModel != nil && total > 0 {- return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)- }-- // Otherwise, use fallback logic based on engine type- if useInfinity {- // For Infinity: scores are already normalized before fusion- // Just extract the scores from results- if chunks == nil || total == 0 || len(chunks) == 0 {- return []float64{}, []float64{}, []float64{}, nil- }-- sim, tsim, vsim = RerankInfinityFallback(chunks)- return sim, tsim, vsim, nil- }-- // For Elasticsearch: need to perform reranking and apply rank features- sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)- return sim, tsim, vsim, nil-}-// RerankByModel performs reranking using a reranker modelfunc RerankByModel(ctx context.Context,@@ -320,32 +267,6 @@ func RerankStandard(return sim, tsim, vsim}-// RerankInfinityFallback is used as a fallback when no reranker model is provided for Infinity engine.-// Infinity can return scores in various field names (SCORE, score, SIMILARITY, etc.),-// so we check multiple possible field names. If no score is found, we default to 1.0-// to ensure the chunk passes through any similarity threshold filters.-func RerankInfinityFallback(chunks []map[string]interface{}) (sim []float64, tsim []float64, vsim []float64) {- common.Info("RerankInfinityFallback started", zap.Int("chunkCount", len(chunks)))-- sim = make([]float64, len(chunks))- for i, chunk := range chunks {- scoreFound := false- scoreFields := []string{"SCORE", "score", "SIMILARITY", "similarity", "_score", "score()", "similarity()"}- for _, field := range scoreFields {- if score, ok := chunk[field].(float64); ok {- sim[i] = score- scoreFound = true- break- }- }- if !scoreFound {- sim[i] = 1.0- }- }- common.Info("RerankInfinityFallback completed")- return sim, sim, sim-}-// HybridSimilarity calculates hybrid similarity between query and documentsfunc HybridSimilarity(avec []float64,diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex b5260c680e..32c0b6b5e0 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -85,9 +85,22 @@ type RetrievalResult struct {Total int64 // Threshold-valid matches across the retrieval candidate set}+// Retrieval window limits. They bound the candidate set one request may ask the+// shared doc engine (and the reranker) to score, so an oversized request cannot+// exhaust shared retrieval resources. The KNN limits mirror the HTTP validation+// in internal/handler/dataset.go, which only covers the dataset endpoints; this+// is the contract every other entry point (chat, agent tools, saved search+// configs) shares. maxRerankCandidates is 1024 because Dify retrieval defaults+// its window to top_k, which is itself capped at 1024.+const (+ maxRerankCandidates = 1024+ maxKNNTopK = 2048+ maxKNNNumCandidates = 10000+)+// Retrieval performs hybrid search + reranking + pagination// - Retrieve candidates for reranking-// - Perform reranking via Rerank()+// - Perform reranking via scoreSearchResult()// - Sort indices by score descending and filter by threshold// - Require Page * PageSize to fit within RerankCandidatesCount// - Support only the first page when a rerank model is configured@@ -143,6 +156,15 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)if rerankCandidatesCount <= 0 || req.Page > rerankCandidatesCount/pageSize {return nil, fmt.Errorf("rerank_candidates_count(%d) must be greater than or equal to page(%d) * page_size(%d)", rerankCandidatesCount, req.Page, pageSize)}+ if rerankCandidatesCount > maxRerankCandidates {+ return nil, fmt.Errorf("rerank_candidates_count(%d) must not exceed %d", rerankCandidatesCount, maxRerankCandidates)+ }+ if *req.KNNTopK > maxKNNTopK {+ return nil, fmt.Errorf("knn_top_k(%d) must not exceed %d", *req.KNNTopK, maxKNNTopK)+ }+ if *req.KNNNumCandidates > maxKNNNumCandidates {+ return nil, fmt.Errorf("knn_num_candidates(%d) must not exceed %d", *req.KNNNumCandidates, maxKNNNumCandidates)+ }if req.RerankModel != nil && req.Page != 1 {return nil, fmt.Errorf("Pagination is not supported when rerank_mdl is specified. Please set page=1 to retrieve the top %d results.", pageSize)}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 07b06b8900..3761e34587 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -178,6 +178,142 @@ func TestRetrievalRejectsPageBeyondCandidateWindow(t *testing.T) {}}+func TestRetrievalRejectsWindowsAboveTheCapBeforeSearch(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ cases := []struct {+ name string+ candidates *int+ knnTopK *int+ knnNumCandidates *int+ wantParam string+ wantBound int+ }{+ {+ name: "rerank candidates above the cap",+ candidates: new(maxRerankCandidates + 1),+ wantParam: "rerank_candidates_count",+ wantBound: maxRerankCandidates,+ },+ {+ name: "knn top k above the cap",+ knnTopK: new(maxKNNTopK + 1),+ wantParam: "knn_top_k",+ wantBound: maxKNNTopK,+ },+ {+ name: "knn num candidates above the cap",+ knnNumCandidates: new(maxKNNNumCandidates + 1),+ wantParam: "knn_num_candidates",+ wantBound: maxKNNNumCandidates,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ engine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ }}+ service := NewRetrievalService(engine, &dao.DocumentDAO{})+ top := 10+ if tc.knnTopK != nil {+ top = *tc.knnTopK+ }+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false++ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 1,+ RerankCandidatesCount: tc.candidates,+ KNNTopK: &top,+ KNNNumCandidates: tc.knnNumCandidates,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ })+ if err == nil {+ t.Fatalf("Retrieval accepted an oversized window: %#v", result)+ }+ if result != nil {+ t.Fatalf("result = %#v, want nil on rejection", result)+ }+ if !strings.Contains(err.Error(), tc.wantParam) || !strings.Contains(err.Error(), fmt.Sprintf("%d", tc.wantBound)) {+ t.Fatalf("error = %q, want parameter %q capped at %d", err, tc.wantParam, tc.wantBound)+ }+ if len(engine.searchLimits) != 0 {+ t.Fatalf("engine Search calls = %v, want none before window validation", engine.searchLimits)+ }+ })+ }+}++func TestRetrievalAcceptsWindowsAtTheCap(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ engine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ }}+ service := NewRetrievalService(engine, &dao.DocumentDAO{})+ candidates := maxRerankCandidates+ top := maxKNNTopK+ numCandidates := maxKNNNumCandidates+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false++ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 1,+ RerankCandidatesCount: &candidates,+ KNNTopK: &top,+ KNNNumCandidates: &numCandidates,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ })+ if err != nil {+ t.Fatalf("Retrieval rejected windows at the cap: %v", err)+ }+ if result == nil {+ t.Fatal("result = nil, want a retrieval result")+ }+ if !slices.Equal(engine.searchLimits, []int{maxRerankCandidates}) {+ t.Fatalf("search limits = %v, want [%d]", engine.searchLimits, maxRerankCandidates)+ }+}++func TestRetrievalWindowCapPrecedesRerankPaginationGuard(t *testing.T) {+ service := NewRetrievalService(nil, &dao.DocumentDAO{})+ candidates := maxRerankCandidates + 1+ top := 10+ threshold := 0.0+ vectorWeight := 1.0++ _, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 2,+ PageSize: 10,+ RerankCandidatesCount: &candidates,+ RerankModel: &modelModule.RerankModel{},+ KNNTopK: &top,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ })+ if err == nil || !strings.Contains(err.Error(), "rerank_candidates_count") || !strings.Contains(err.Error(), "must not exceed") {+ t.Fatalf("error = %v, want the window cap to reject before the pagination guard", err)+ }+}+func TestRetrievalRejectsPaginationWithRerankModelBeforeSearch(t *testing.T) {oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()