RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): recover dense candidates after empty hybrid search (#19502)
Sbaaoui Idriss5c91e65dde33
diff --git a/internal/agent/retrievalbridge/runtime_adapter.go b/internal/agent/retrievalbridge/runtime_adapter.goindex a2d20ff3ca..6dd41af4e4 100644--- a/internal/agent/retrievalbridge/runtime_adapter.go+++ b/internal/agent/retrievalbridge/runtime_adapter.go@@ -73,6 +73,7 @@ func (a *RuntimeAdapter) Search(ctx context.Context, db *gorm.DB, req agentrunt.KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,UseKG: req.UseKG,SimilarityThreshold: req.SimilarityThreshold,+ AllowDenseFallback: new(false),CrossLanguages: req.CrossLanguages,RetrievalFrom: req.RetrievalFrom,DocScope: req.DocScope,diff --git a/internal/agent/retrievalbridge/runtime_adapter_test.go b/internal/agent/retrievalbridge/runtime_adapter_test.gonew file mode 100644index 0000000000..e58c46b8af--- /dev/null+++ b/internal/agent/retrievalbridge/runtime_adapter_test.go@@ -0,0 +1,34 @@+package retrievalbridge++import (+ "context"+ "testing"++ agentrunt "ragflow/internal/agent/runtime"+ agenttool "ragflow/internal/agent/tool"++ "gorm.io/gorm"+)++type captureRetrievalService struct {+ req agenttool.RetrievalRequest+}++func (s *captureRetrievalService) Search(_ context.Context, _ *gorm.DB, req agenttool.RetrievalRequest) ([]agenttool.RetrievalChunk, error) {+ s.req = req+ return nil, nil+}++func TestRuntimeAdapterDisablesDenseFallback(t *testing.T) {+ previous := agenttool.GetRetrievalService()+ service := &captureRetrievalService{}+ agenttool.SetRetrievalService(service)+ t.Cleanup(func() { agenttool.SetRetrievalService(previous) })+ _, err := NewRuntimeAdapter().Search(t.Context(), nil, agentrunt.RetrievalRequest{Query: "test"})+ if err != nil {+ t.Fatal(err)+ }+ if service.req.AllowDenseFallback == nil || *service.req.AllowDenseFallback {+ t.Fatal("Agentic runtime retrieval must disable dense fallback")+ }+}diff --git a/internal/agent/tool/agentic_search.go b/internal/agent/tool/agentic_search.goindex 4586662397..df444d885c 100644--- a/internal/agent/tool/agentic_search.go+++ b/internal/agent/tool/agentic_search.go@@ -128,6 +128,7 @@ func (a *AgenticSearchTool) InvokableRun(ctx context.Context, argumentsInJSON stTopN: args.TopN,TopK: args.TopN * 4,SimilarityThreshold: &similarityThreshold,+ AllowDenseFallback: new(false),KeywordsSimilarityWeight: &weight,DocScope: args.DocScope,}diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex 5ad301e62b..27133d9d7a 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -322,15 +322,16 @@ func nlpRequestFromRetrieval(excludeCompiled bool,) *nlp.RetrievalRequest {nlpReq := &nlp.RetrievalRequest{- Question: req.Query,- TenantIDs: append([]string(nil), tenantIDs...),- KbIDs: append([]string(nil), req.DatasetIDs...),- DocIDs: append([]string(nil), compactStrings(req.DocScope)...),- Page: 1,- PageSize: topN,- EmbeddingModel: embeddingModel,- Aggs: boolPtr(false),- Highlight: boolPtr(false),+ Question: req.Query,+ TenantIDs: append([]string(nil), tenantIDs...),+ KbIDs: append([]string(nil), req.DatasetIDs...),+ DocIDs: append([]string(nil), compactStrings(req.DocScope)...),+ Page: 1,+ PageSize: topN,+ EmbeddingModel: embeddingModel,+ Aggs: boolPtr(false),+ Highlight: boolPtr(false),+ AllowDenseFallback: req.AllowDenseFallback,}if excludeCompiled {// Python hybrid_search excludes compiled products from plain retrievaldiff --git a/internal/agent/tool/retrieval_nlp_test.go b/internal/agent/tool/retrieval_nlp_test.goindex e57d497c0a..e194c70759 100644--- a/internal/agent/tool/retrieval_nlp_test.go+++ b/internal/agent/tool/retrieval_nlp_test.go@@ -246,11 +246,15 @@ func TestNLPRequestFromRetrieval_ThreadsSearchControls(t *testing.T) {TopK: 99,KeywordsSimilarityWeight: &keywordWeight,SimilarityThreshold: &similarityThreshold,+ AllowDenseFallback: new(false),}, []string{"tenant-a"}, 3, embeddingModel, false)if got.Question != "hi" {t.Fatalf("Question=%q want hi", got.Question)}+ if got.AllowDenseFallback == nil || *got.AllowDenseFallback {+ t.Fatal("dense fallback opt-out was not passed to nlp retrieval")+ }if len(got.TenantIDs) != 1 || got.TenantIDs[0] != "tenant-a" {t.Fatalf("TenantIDs=%v want [tenant-a]", got.TenantIDs)}@@ -287,6 +291,9 @@ func TestNLPRequestFromRetrieval_FallsBackToTopNHeadroom(t *testing.T) {if got.VectorSimilarityWeight != nil {t.Fatalf("VectorSimilarityWeight=%v want nil", got.VectorSimilarityWeight)}+ if got.AllowDenseFallback != nil {+ t.Fatal("ordinary retrieval must preserve default dense fallback policy")+ }}func TestNLPRequestFromRetrieval_PreservesExplicitZeroSimilarityThreshold(t *testing.T) {diff --git a/internal/agent/tool/retrieval_service.go b/internal/agent/tool/retrieval_service.goindex a6b1d670f6..cbd1c9435a 100644--- a/internal/agent/tool/retrieval_service.go+++ b/internal/agent/tool/retrieval_service.go@@ -68,6 +68,7 @@ type RetrievalRequest struct {DisableVectorLeg boolUseKG boolSimilarityThreshold *float64+ AllowDenseFallback *boolRerankID stringCrossLanguages []stringTOCEnhance booldiff --git a/internal/agent/tool/retrieval_test.go b/internal/agent/tool/retrieval_test.goindex 1a897a742d..5bb5563f23 100644--- a/internal/agent/tool/retrieval_test.go+++ b/internal/agent/tool/retrieval_test.go@@ -122,6 +122,27 @@ func TestRetrieval_EmptyArgsIsHandled(t *testing.T) {}}+func TestAgenticSearchDisablesDenseFallback(t *testing.T) {+ prev := GetRetrievalService()+ svc := &capturingRetrievalService{}+ SetRetrievalService(svc)+ t.Cleanup(func() { SetRetrievalService(prev) })+ state := runtime.NewCanvasState("run-1", "task-1")+ state.Sys["tenant_id"] = "tenant-1"+ ctx := runtime.WithState(t.Context(), state)+ for _, mode := range []string{toolHybridSearch, toolVectorSearch, toolBM25Search} {+ t.Run(mode, func(t *testing.T) {+ _, err := NewAgenticSearchTool(mode).InvokableRun(ctx, `{"query":"hello","kb_ids":["kb-1"]}`)+ if err != nil {+ t.Fatal(err)+ }+ if svc.req.AllowDenseFallback == nil || *svc.req.AllowDenseFallback {+ t.Fatal("Agentic search must disable dense fallback")+ }+ })+ }+}+func TestRetrieval_PassesTenantIDFromCanvasState(t *testing.T) {prev := GetRetrievalService()svc := &capturingRetrievalService{}diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex 24b47163fc..98f1dd3242 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -63,7 +63,13 @@ func TestSearchUsesConfiguredKNNNumCandidates(t *testing.T) {engine := &Engine{client: client}_, err = engine.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"ragflow_tenant"},+ KbIDs: []string{"kb-1"},Limit: 30,+ Filter: map[string]interface{}{+ "doc_id": []string{"doc-1"},+ "available_int": 1,+ "category_kwd": "allowed",+ },MatchExprs: []interface{}{&types.MatchDenseExpr{VectorColumnName: "q_2_vec",EmbeddingData: []float64{0.1, 0.2},@@ -81,6 +87,19 @@ func TestSearchUsesConfiguredKNNNumCandidates(t *testing.T) {if knn["k"] != float64(128) || knn["num_candidates"] != float64(4096) {t.Fatalf("KNN parameters = (%v, %v), want (128, 4096)", knn["k"], knn["num_candidates"])}+ filterJSON, err := json.Marshal(knn["filter"])+ if err != nil {+ t.Fatal(err)+ }+ filter := string(filterJSON)+ for _, scope := range []string{"kb-1", "doc-1", "available_int", "category_kwd", "allowed"} {+ if !strings.Contains(filter, scope) {+ t.Fatalf("KNN filter %s lost scope %q", filter, scope)+ }+ }+ if strings.Contains(filter, "query_string") {+ t.Fatalf("dense-only KNN filter contains lexical predicate: %s", filter)+ }}func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) {diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex d560207aeb..147fba4ebd 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -19,16 +19,18 @@ package nlpimport ("context""fmt"+ "maps""math"+ "slices"+ "sort"+ "strconv"+ "strings"+"ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types""ragflow/internal/entity/models"- "sort"- "strconv"- "strings"-"ragflow/internal/tokenizer""go.uber.org/zap"@@ -63,6 +65,7 @@ type RetrievalRequest struct {EmbeddingModel *models.EmbeddingModelAggs *boolHighlight *bool+ AllowDenseFallback *boolFilter map[string]interface{}}@@ -141,6 +144,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)RankFeature: *req.RankFeature,EmbeddingModel: req.EmbeddingModel,VectorSimilarityWeight: req.VectorSimilarityWeight,+ AllowDenseFallback: req.AllowDenseFallback,Filter: req.Filter,}searchResult, err := s.Search(ctx, searchReq)@@ -515,6 +519,7 @@ type RetrievalSearchRequest struct {Filter map[string]interface{}EmbeddingModel *models.EmbeddingModelVectorSimilarityWeight *float64+ AllowDenseFallback *bool}func buildInfinityFusionExpr(topn int, vectorSimilarityWeight *float64) *types.FusionExpr {@@ -568,6 +573,9 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchRequeif req.Highlight == nil {req.Highlight = func() *bool { v := false; return &v }()}+ if req.AllowDenseFallback == nil {+ req.AllowDenseFallback = new(true)+ }filters := req.GetFilters()if _, ok := filters["available_int"]; !ok {filters["available_int"] = 1@@ -658,6 +666,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchRequeif err != nil {return nil, fmt.Errorf("GetVector failed: %w", err)}+ denseTemplate := cloneDenseExpr(matchDense)// Execute search with fusionfusionExpr := buildRetrievalFusionExpr(s.docEngine.GetType(), knnTopK, req.VectorSimilarityWeight)@@ -670,7 +679,11 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque}searchRequest.SelectFields = searchSrc- searchRequest.MatchExprs = []interface{}{matchText, matchDense, fusionExpr}+ if matchText == nil {+ searchRequest.MatchExprs = []any{matchDense}+ } else {+ searchRequest.MatchExprs = []any{matchText, matchDense, fusionExpr}+ }searchRequest.RankFeature = req.RankFeatureengineResult, err = s.docEngine.Search(ctx, searchRequest)@@ -680,7 +693,18 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// If result is empty, retry with relaxed conditionsif engineResult.Total == 0 {_, hasDocIDFilter := filters["doc_id"]- if hasDocIDFilter {+ if matchText == nil {+ if *req.AllowDenseFallback {+ common.Debug("Retrieval dense-only fallback after empty initial search")+ matchDense = cloneDenseExpr(denseTemplate)+ matchDense.ExtraOptions["similarity"] = 0.17+ searchRequest.MatchExprs = []any{matchDense}+ engineResult, err = s.docEngine.Search(ctx, searchRequest)+ if err != nil {+ return nil, fmt.Errorf("dense-only fallback failed: %w", err)+ }+ }+ } else if hasDocIDFilter {// When a doc_id filter is present (e.g. from metadata filter like era=960)// and the hybrid search returns no results, fall back to a filter-only// search (no text match, no vector match). This ensures that when a@@ -707,14 +731,30 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// This provides a second chance for queries that were too strict// on the first attempt.matchText, _ := GetQueryBuilder().Question(req.Question, "qa", 0.1)+ matchDense = cloneDenseExpr(denseTemplate)matchDense.ExtraOptions["similarity"] = 0.17- searchRequest.MatchExprs = []interface{}{matchText, matchDense, fusionExpr}+ if matchText == nil {+ searchRequest.MatchExprs = []any{matchDense}+ } else {+ searchRequest.MatchExprs = []any{matchText, matchDense, fusionExpr}+ }searchRequest.RankFeature = req.RankFeatureengineResult, err = s.docEngine.Search(ctx, searchRequest)if err != nil {return nil, fmt.Errorf("search retry failed: %w", err)}+ // Zero-only by design: any lexical hit keeps the existing hybrid candidate semantics.+ if engineResult.Total == 0 && matchText != nil && *req.AllowDenseFallback {+ common.Debug("Retrieval dense-only fallback after empty hybrid retries")+ matchDense = cloneDenseExpr(denseTemplate)+ matchDense.ExtraOptions["similarity"] = 0.17+ searchRequest.MatchExprs = []any{matchDense}+ engineResult, err = s.docEngine.Search(ctx, searchRequest)+ if err != nil {+ return nil, fmt.Errorf("dense-only fallback failed: %w", err)+ }+ }}}@@ -778,6 +818,13 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque}, nil}+func cloneDenseExpr(source *types.MatchDenseExpr) *types.MatchDenseExpr {+ clone := *source+ clone.EmbeddingData = slices.Clone(source.EmbeddingData)+ clone.ExtraOptions = maps.Clone(source.ExtraOptions)+ return &clone+}+// GetVector computes query vector and returns MatchDenseExpr for hybrid searchfunc (s *RetrievalService) GetVector(ctx context.Context, txt string, embModel *models.EmbeddingModel, knnTopK, numCandidates int, similarity float64) (*types.MatchDenseExpr, error) {embeddingConfig := &models.EmbeddingConfig{diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 61afe6a909..3a3c3d4192 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -3,6 +3,8 @@ package nlpimport ("context""fmt"+ "maps"+ "slices""testing""ragflow/internal/common"@@ -252,6 +254,12 @@ func (e *captureSearchDocEngine) GetAggregation(_ []map[string]interface{}, _ stfunc (e *captureSearchDocEngine) GetHighlight(_ []map[string]interface{}, _ []string, _ string) map[string]string {return nil}+func (e *captureSearchDocEngine) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {+ return map[string]interface{}{}, nil+}+func (e *captureSearchDocEngine) GetScores(map[string]interface{}) map[string]float64 {+ return map[string]float64{}+}type captureEmbeddingDriver struct{ modelModule.ModelDriver }@@ -259,6 +267,149 @@ func (d *captureEmbeddingDriver) Embed(_ context.Context, _ *string, _ modelModureturn []modelModule.EmbeddingData{{Embedding: []float64{0.1, 0.2}}}, nil}+type retryCaptureEngine struct {+ engine.DocEngine+ engineType string+ totals []int64+ requests []*types.SearchRequest+ mutate bool+}++func (e *retryCaptureEngine) GetType() string { return e.engineType }+func (e *retryCaptureEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ requestCopy := *req+ requestCopy.Filter = maps.Clone(req.Filter)+ requestCopy.MatchExprs = slices.Clone(req.MatchExprs)+ for i, expression := range requestCopy.MatchExprs {+ if dense, ok := expression.(*types.MatchDenseExpr); ok {+ requestCopy.MatchExprs[i] = cloneDenseExpr(dense)+ }+ }+ e.requests = append(e.requests, &requestCopy)+ if e.mutate {+ for _, expression := range req.MatchExprs {+ if dense, ok := expression.(*types.MatchDenseExpr); ok {+ delete(dense.ExtraOptions, "num_candidates")+ dense.ExtraOptions["filter"] = "connector-added"+ }+ }+ }+ total := e.totals[0]+ e.totals = e.totals[1:]+ return &types.SearchResult{Total: total}, nil+}+func (e *retryCaptureEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }+func (e *retryCaptureEngine) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {+ return nil+}+func (e *retryCaptureEngine) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {+ return nil+}+func (e *retryCaptureEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {+ return nil+}++func TestSearchDenseFallbackContract(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }++ for _, engineType := range []string{string(engine.EngineElasticsearch), string(engine.EngineInfinity)} {+ for _, test := range []struct {+ name string+ totals []int64+ wantCalls int+ wantExprs int+ }{+ {name: "one weak lexical hit", totals: []int64{1}, wantCalls: 1, wantExprs: 3},+ {name: "relaxed hit", totals: []int64{0, 1}, wantCalls: 2, wantExprs: 3},+ {name: "dense recovery", totals: []int64{0, 0, 1}, wantCalls: 3, wantExprs: 1},+ {name: "dense recovery empty", totals: []int64{0, 0, 0}, wantCalls: 3, wantExprs: 1},+ } {+ t.Run(engineType+"/"+test.name, func(t *testing.T) {+ docEngine := &retryCaptureEngine{engineType: engineType, totals: slices.Clone(test.totals), mutate: true}+ service := NewRetrievalService(docEngine, nil)+ _, err := service.Search(t.Context(), &RetrievalSearchRequest{+ Question: "ปัญหาแจ้งระบบภาษี", TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"}, Page: 1, PageSize: 10,+ KNNTopK: 7, KNNNumCandidates: 19, Filter: map[string]interface{}{"category_kwd": "allowed"},+ EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}},+ })+ if err != nil {+ t.Fatal(err)+ }+ if len(docEngine.requests) != test.wantCalls {+ t.Fatalf("search calls = %d, want %d", len(docEngine.requests), test.wantCalls)+ }+ last := docEngine.requests[len(docEngine.requests)-1]+ if len(last.MatchExprs) != test.wantExprs {+ t.Fatalf("final expressions = %d, want %d", len(last.MatchExprs), test.wantExprs)+ }+ if test.wantCalls == 3 {+ dense := last.MatchExprs[0].(*types.MatchDenseExpr)+ if dense.TopN != 7 || dense.ExtraOptions["num_candidates"] != 19 || dense.ExtraOptions["similarity"] != 0.17 {+ t.Fatalf("dense fallback changed options: %#v", dense)+ }+ if _, contaminated := dense.ExtraOptions["filter"]; contaminated {+ t.Fatalf("dense fallback inherited connector mutation: %#v", dense.ExtraOptions)+ }+ if fmt.Sprint(last.Filter["kb_id"]) != "[kb-1]" || last.Filter["available_int"] != 1 || last.Filter["category_kwd"] != "allowed" {+ t.Fatalf("dense fallback lost scope filters: %#v", last.Filter)+ }+ }+ })+ }+ }+}++func TestSearchWithoutLexicalExpressionRetries(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }++ docEngine := &retryCaptureEngine{engineType: string(engine.EngineElasticsearch), totals: []int64{0, 1}, mutate: true}+ service := NewRetrievalService(docEngine, nil)+ _, err := service.Search(t.Context(), &RetrievalSearchRequest{+ Question: "!!!", TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"},+ EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}},+ })+ if err != nil {+ t.Fatal(err)+ }+ if len(docEngine.requests) != 2 || len(docEngine.requests[1].MatchExprs) != 1 {+ t.Fatalf("dense-only request did not retry: %#v", docEngine.requests)+ }+ dense := docEngine.requests[1].MatchExprs[0].(*types.MatchDenseExpr)+ if dense.ExtraOptions["similarity"] != 0.17 || dense.ExtraOptions["num_candidates"] != 2048 || dense.ExtraOptions["filter"] != nil {+ t.Fatalf("fallback options = %#v", dense.ExtraOptions)+ }+}++func TestRetrievalDisablesDenseFallback(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }+ for _, query := range []string{"!!!", "weak lexical query"} {+ t.Run(query, func(t *testing.T) {+ wantCalls := 2+ if query == "!!!" {+ wantCalls = 1+ }+ docEngine := &retryCaptureEngine{engineType: string(engine.EngineElasticsearch), totals: []int64{0, 0, 0}}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ _, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: query, TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"},+ EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}}, AllowDenseFallback: new(false),+ })+ if err != nil {+ t.Fatal(err)+ }+ if len(docEngine.requests) != wantCalls {+ t.Fatalf("calls = %d, want %d", len(docEngine.requests), wantCalls)+ }+ })+ }+}+func TestSearchPassesVectorSimilarityWeightToFusionExpr(t *testing.T) {if GetQueryBuilder() == nil {globalQueryBuilder = NewQueryBuilder()@@ -284,7 +435,7 @@ func TestRetrievalPassesVectorSimilarityWeightToSearch(t *testing.T) {top := 10docEngine := &captureSearchDocEngine{engineType: string(engine.EngineInfinity),- result: &types.SearchResult{Chunks: []map[string]interface{}{}, Total: 0},+ result: &types.SearchResult{Chunks: []map[string]interface{}{}, Total: 1},}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})_, err := service.Retrieval(t.Context(), &RetrievalRequest{diff --git a/rag/advanced_rag/agentic_rag.py b/rag/advanced_rag/agentic_rag.pyindex 53c7a6c4c9..3d35c724fe 100644--- a/rag/advanced_rag/agentic_rag.py+++ b/rag/advanced_rag/agentic_rag.py@@ -722,6 +722,7 @@ class RAGTools:doc_ids=doc_scope,rank_feature=label_question(question, self.kbs),rerank_candidates_count=rerank_candidates_count,+ allow_dense_fallback=False,)if not kbinfos:return {"chunks": [], "doc_aggs": []}diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex 7f7203868b..45dd790929 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -550,6 +550,7 @@ async def _content_recall_docs(tools, query: str, doc_scope: list[str] | None =doc_ids=doc_scope,aggs=True,highlight=False,+ allow_dense_fallback=False,)except Exception:_LOG.exception("[Dataset navigation] content-recall retrieval failed")@@ -1475,6 +1476,7 @@ async def _recall_chunk_ids_in_doc(tools, query: str, doc_id: str, top_n: int) -vector_similarity_weight=0.3 if embd_mdl else 0,doc_ids=[str(doc_id).strip()],highlight=False,+ allow_dense_fallback=False,# RAPTOR cluster nodes live in the chunk index (rechunk_kwd=tree, no# compile_kwd), so they survive this filter and stay retrievable —# which is the point, since they are the structure.diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex 9965b027c0..16a947f090 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -170,6 +170,7 @@ async def hybrid_search(doc_ids=doc_scope,must_not={"exists": "compile_kwd"}, # plain retrieval = document chunks only; compiled products have their own toolsrerank_candidates_count=rerank_candidates_count,+ allow_dense_fallback=False,)kbinfos = _normalize(kbinfos, tools.tenant_ids)# Preserve the RAW retrieved chunks in the central memory store BEFORE any@@ -235,6 +236,7 @@ async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_doc_ids=doc_scope,must_not={"exists": "compile_kwd"},rerank_candidates_count=rerank_candidates_count,+ allow_dense_fallback=False,)kbinfos = _normalize(kbinfos, tools.tenant_ids)try:@@ -272,6 +274,7 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:doc_ids=doc_scope,must_not={"exists": "compile_kwd"},rerank_candidates_count=rerank_candidates_count,+ allow_dense_fallback=False,)kbinfos = _normalize(kbinfos, tools.tenant_ids)try:diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 1d2246f612..a59cb724fb 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -13,6 +13,7 @@# See the License for the specific language governing permissions and# limitations under the License.#+import copyimport jsonimport loggingimport re@@ -303,7 +304,8 @@ class Dealer:total = self.dataStore.get_total(res)logging.debug("Dealer.search TOTAL: {}".format(total))else:- matchDense = await self.get_vector(qst, emb_mdl, top_k=knn_top_k, num_candidates=knn_num_candidates, similarity=req.get("similarity", 0.1))+ dense_template = await self.get_vector(qst, emb_mdl, top_k=knn_top_k, num_candidates=knn_num_candidates, similarity=req.get("similarity", 0.1))+ matchDense = copy.deepcopy(dense_template)q_vec = matchDense.embedding_data# ES path no longer fetches chunk vectors here. The clean# cosine score is recovered later via a second KNN-only call@@ -333,13 +335,34 @@ class Dealer:total = self.dataStore.get_total(res)logging.debug("Dealer.search TOTAL: {}".format(total))- # If result is empty, try again with lower min_match+ # If result is empty, try again with lower min_match or, for+ # dense-only queries, a lower vector threshold.if total == 0:- if filters.get("doc_id"):+ if not matchText:+ if req.get("allow_dense_fallback", True):+ matchDense = copy.deepcopy(dense_template)+ matchDense.extra_options["similarity"] = 0.17+ logging.debug("Dealer.search dense-only fallback after empty initial search")+ res = await thread_pool_exec(+ self.dataStore.search,+ src,+ highlightFields,+ filters,+ [matchDense],+ orderBy,+ offset,+ limit,+ idx_names,+ kb_ids,+ rank_feature=rank_feature,+ )+ total = self.dataStore.get_total(res)+ elif filters.get("doc_id"):res = await thread_pool_exec(self.dataStore.search, src, [], filters, [], orderBy, offset, limit, idx_names, kb_ids)total = self.dataStore.get_total(res)else:matchText, _ = self.qryr.question(qst, min_match=(0.1 if min_match else 0))+ matchDense = copy.deepcopy(dense_template)matchDense.extra_options["similarity"] = 0.17res = await thread_pool_exec(self.dataStore.search,@@ -355,6 +378,25 @@ class Dealer:rank_feature=rank_feature,)total = self.dataStore.get_total(res)+ # Zero-only by design: any lexical hit keeps the existing hybrid candidate semantics.+ if total == 0 and matchText and req.get("allow_dense_fallback", True):+ matchDense = copy.deepcopy(dense_template)+ matchDense.extra_options["similarity"] = 0.17+ logging.debug("Dealer.search dense-only fallback after empty hybrid retries")+ res = await thread_pool_exec(+ self.dataStore.search,+ src,+ [],+ filters,+ [matchDense],+ orderBy,+ offset,+ limit,+ idx_names,+ kb_ids,+ rank_feature=rank_feature,+ )+ total = self.dataStore.get_total(res)logging.debug("Dealer.search 2 TOTAL: {}".format(total))for k in keywords:@@ -683,6 +725,7 @@ class Dealer:rerank_candidates_count=64,knn_top_k=1024, # Advanced knn parameterknn_num_candidates=2048, # Advanced knn parameter+ allow_dense_fallback=True,):"""Pagination is neither efficient nor reliable for this retrieval when rerank is enabled because the system must:@@ -717,6 +760,7 @@ class Dealer:"vector_similarity_weight": vector_similarity_weight,"knn_top_k": knn_top_k,"knn_num_candidates": knn_num_candidates,+ "allow_dense_fallback": allow_dense_fallback,}if isinstance(must_not, dict) and must_not:req["must_not"] = must_notdiff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.pyindex 236fa6dbd0..1609eb46f6 100644--- a/rag/utils/infinity_conn.py+++ b/rag/utils/infinity_conn.py@@ -247,8 +247,9 @@ class InfinityConnection(InfinityConnectionBase):self.logger.debug(f"INFINITY search MatchTextExpr: {json.dumps(matchExpr.__dict__)}")elif isinstance(matchExpr, MatchDenseExpr):matchExpr.extra_options.pop("num_candidates", None)- if filter_fulltext and "filter" not in matchExpr.extra_options:- matchExpr.extra_options.update({"filter": filter_fulltext})+ dense_filter = filter_fulltext or filter_cond+ if dense_filter and "filter" not in matchExpr.extra_options:+ matchExpr.extra_options.update({"filter": dense_filter})# dense_filter = _build_dense_filter(filter_cond, filter_fulltext, vector_similarity_weight)# if dense_filter and "filter" not in matchExpr.extra_options:# matchExpr.extra_options.update({"filter": dense_filter})diff --git a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py b/test/unit_test/rag/nlp/test_gaussdb_retrieval.pyindex 057684293b..4238aa5221 100644--- a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py+++ b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py@@ -13,6 +13,7 @@# See the License for the specific language governing permissions and# limitations under the License.#+import copyimport sysimport typesfrom unittest.mock import AsyncMock, Mock@@ -208,6 +209,128 @@ class FakeQueryer:return MatchTextExpr(["content_with_weight"], text, 1024), [text]+class RetryStore(FakeGaussDBStore):+ def __init__(self, totals, mutate=False):+ self.totals = iter(totals)+ self.calls = []+ self.mutate = mutate++ def search(self, *args, **kwargs):+ self.calls.append(copy.deepcopy((args, kwargs)))+ if self.mutate:+ dense = next((expr for expr in args[3] if isinstance(expr, MatchDenseExpr)), None)+ if dense:+ dense.extra_options.pop("num_candidates", None)+ dense.extra_options["filter"] = "connector-added"+ return type("SearchResult", (), {"total": next(self.totals), "chunks": []})()+++async def run_retry_search(dealer_cls, totals, *, queryer=None, mutate=False, doc_ids=None):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = queryer or FakeQueryer()+ dealer.dataStore = RetryStore(totals, mutate)++ async def fake_get_vector(_text, _emb_mdl, top_k=10, num_candidates=20, similarity=0.1):+ return MatchDenseExpr(+ "q_4_vec",+ [0.1, 0.2, 0.3, 0.4],+ "float",+ "cosine",+ top_k,+ {"similarity": similarity, "num_candidates": num_candidates, "future_option": "kept"},+ )++ dealer.get_vector = fake_get_vector+ req = {+ "question": "ปัญหาแจ้งระบบภาษี",+ "page": 1,+ "size": 5,+ "kb_ids": ["kb1"],+ "available_int": 1,+ "must_not": {"exists": "compile_kwd"},+ "knn_top_k": 7,+ "knn_num_candidates": 19,+ }+ if doc_ids:+ req["doc_ids"] = doc_ids+ await dealer.search(req, "ragflow_tenant", ["kb1"], emb_mdl=object(), highlight=True)+ return dealer.dataStore.calls+++ ("totals", "expected_calls", "final_expression_count"),+ [([1], 1, 3), ([0, 1], 2, 3), ([0, 0, 1], 3, 1), ([0, 0, 0], 3, 1)],+ ids=["one-weak-lexical-hit", "relaxed-hit", "dense-recovery", "dense-recovery-empty"],+)+async def test_dense_fallback_runs_only_after_empty_hybrid_attempts(dealer_cls, totals, expected_calls, final_expression_count):+ calls = await run_retry_search(dealer_cls, totals)++ assert len(calls) == expected_calls+ assert len(calls[-1][0][3]) == final_expression_count+ if expected_calls == 3:+ args, kwargs = calls[-1]+ dense = args[3][0]+ assert args[1] == []+ assert args[2] == {"kb_id": ["kb1"], "available_int": 1, "must_not": {"exists": "compile_kwd"}}+ assert args[8] == ["kb1"]+ assert kwargs == {"rank_feature": None}+ assert dense.topn == 7+ assert dense.extra_options == {"similarity": 0.17, "num_candidates": 19, "future_option": "kept"}+++async def test_dense_fallback_uses_pristine_expression(dealer_cls):+ calls = await run_retry_search(dealer_cls, [0, 0, 1], mutate=True)++ for index, (args, _) in enumerate(calls):+ dense = next(expr for expr in args[3] if isinstance(expr, MatchDenseExpr))+ assert dense.extra_options == {+ "similarity": 0.1 if index == 0 else 0.17,+ "num_candidates": 19,+ "future_option": "kept",+ }+++async def test_no_lexical_expression_stays_single_dense_search(dealer_cls):+ calls = await run_retry_search(dealer_cls, [0, 0], queryer=types.SimpleNamespace(question=lambda *_args, **_kwargs: (None, [])))++ assert len(calls) == 2+ assert len(calls[-1][0][3]) == 1+ assert isinstance(calls[-1][0][3][0], MatchDenseExpr)+ assert calls[-1][0][3][0].extra_options["similarity"] == 0.17+++async def test_dense_only_fallback_can_be_disabled(dealer_cls):+ dealer = make_dealer(dealer_cls)+ dealer.qryr = types.SimpleNamespace(question=lambda *_args, **_kwargs: (None, []))+ dealer.dataStore = RetryStore([0])++ async def fake_get_vector(_text, _emb_mdl, top_k=10, num_candidates=20, similarity=0.1):+ return MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", top_k, {"similarity": similarity, "num_candidates": num_candidates})++ dealer.get_vector = fake_get_vector+ await dealer.search(+ {"question": "query", "page": 1, "size": 5, "kb_ids": ["kb1"], "allow_dense_fallback": False},+ "ragflow_tenant",+ ["kb1"],+ emb_mdl=object(),+ )++ assert len(dealer.dataStore.calls) == 1+++async def test_document_scope_keeps_existing_filter_only_retry(dealer_cls):+ calls = await run_retry_search(dealer_cls, [0, 1], doc_ids=["doc-1"])++ assert len(calls) == 2+ assert calls[1][0][2]["doc_id"] == ["doc-1"]+ assert calls[1][0][3] == []++def retrieval_chunk(score, doc_id, doc_name, content="risk contract"):return {"_score": score,diff --git a/test/unit_test/rag/test_search_pagination.py b/test/unit_test/rag/test_search_pagination.pyindex 85980f0035..f8f6df7b39 100644--- a/test/unit_test/rag/test_search_pagination.py+++ b/test/unit_test/rag/test_search_pagination.py@@ -17,6 +17,7 @@import sysimport types+from unittest.mock import MagicMockimport pytest@@ -36,6 +37,7 @@ sys.modules.setdefault("rag.nlp.rag_tokenizer", _fake_tokenizer)sys.modules.setdefault("common.settings", types.ModuleType("common.settings"))from rag.nlp.search import Dealer, settings # noqa: E402+from common.doc_store.doc_store_base import MatchDenseExpr, OrderByExpr # noqa: E402def _search_result(total):@@ -54,6 +56,42 @@ def _search_result(total):return Dealer.SearchResult(total=total, ids=ids, query_vector=[0.1], field=fields, highlight={})+def test_elasticsearch_dense_only_filter_keeps_scope_without_lexical_predicate():+ from rag.utils import es_conn++ factory = es_conn.ESConnection+ cls = next(cell.cell_contents for cell in factory.__closure__ or () if isinstance(cell.cell_contents, type))+ conn = cls.__new__(cls)+ conn.logger = MagicMock()+ captured = {}++ def search_once(_index_names, query, **_kwargs):+ captured["query"] = query+ return {"timed_out": False, "hits": {"total": {"value": 0}, "hits": []}}++ conn._es_search_once = search_once+ dense = MatchDenseExpr("q_2_vec", [0.1, 0.2], "float", "cosine", 7, {"similarity": 0.17, "num_candidates": 19})+ conn.search(+ ["id"],+ [],+ {"doc_id": ["doc-1"], "available_int": 1, "category_kwd": "allowed"},+ [dense],+ OrderByExpr(),+ 0,+ 5,+ "ragflow_tenant",+ ["kb-1"],+ )++ knn = captured["query"]["knn"]+ knn = knn[0] if isinstance(knn, list) else knn+ assert knn["k"] == 7+ assert knn["num_candidates"] == 19+ assert "query_string" not in str(knn["filter"])+ for scope in ("kb-1", "doc-1", "available_int", "category_kwd", "allowed"):+ assert scope in str(knn["filter"])++@pytest.mark.asyncio@pytest.mark.parametrize(("page", "page_size", "rerank_candidates_count"), [(1, 10, 64), (6, 10, 64), (7, 10, 70)])async def test_retrieval_fetches_rerank_candidates_once_and_slices_requested_page(monkeypatch, page, page_size, rerank_candidates_count):diff --git a/test/unit_test/rag/utils/test_infinity_conn_helpers.py b/test/unit_test/rag/utils/test_infinity_conn_helpers.pyindex 1c7f368356..0bed65f857 100644--- a/test/unit_test/rag/utils/test_infinity_conn_helpers.py+++ b/test/unit_test/rag/utils/test_infinity_conn_helpers.py@@ -31,7 +31,9 @@ pool and exercise the routing logic. Run with::import loggingfrom unittest.mock import MagicMock, patch+import pandas as pdimport pytest+from common.doc_store.doc_store_base import MatchDenseExpr, OrderByExprpytestmark = pytest.mark.p2@@ -55,6 +57,33 @@ def _resolve_infinity_class():_Inf = _resolve_infinity_class()+def test_dense_search_keeps_scope_filter_without_fulltext():+ conn = _Inf.__new__(_Inf)+ conn.dbName = "default_db"+ conn.logger = logging.getLogger("test.infinity_dense_scope")+ conn.equivalent_condition_to_str = lambda *_args: "doc_id IN ('doc-1') AND available_int=1"++ builder = MagicMock()+ for method in ("match_dense", "offset", "limit", "option"):+ getattr(builder, method).return_value = builder+ builder.to_df.return_value = (pd.DataFrame([{"id": "chunk-1", "SIMILARITY": 0.9, "pagerank_fea": 0.0}]), {"total_hits_count": 1})+ table = MagicMock()+ table.output.return_value = builder+ db = MagicMock()+ db.get_table.return_value = table+ inf_conn = MagicMock()+ inf_conn.get_database.return_value = db+ conn.connPool = MagicMock()+ conn.connPool.get_conn.return_value = inf_conn++ dense = MatchDenseExpr("q_2_vec", [0.1, 0.2], "float", "cosine", 7, {"similarity": 0.17, "num_candidates": 19})+ conn.search(["id"], [], {"doc_id": ["doc-1"], "available_int": 1}, [dense], OrderByExpr(), 0, 5, "ragflow_tenant", ["kb-1"])++ options = builder.match_dense.call_args.args[-1]+ assert options["filter"] == "doc_id IN ('doc-1') AND available_int=1"+ assert "filter_fulltext" not in options["filter"]++# ---------------------------------------------------------------------------# ``InfinityConnection.get`` — doc-meta special case# ---------------------------------------------------------------------------