RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: pass Infinity vector similarity weight (#17453)
Linpeng chenga9323ca5549c
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 0a09754d29..33c956e195 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -137,16 +137,17 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)// Execute search via Search()searchReq := &RetrievalSearchRequest{- TenantIDs: req.TenantIDs,- Question: req.Question,- KbIDs: req.KbIDs,- DocIDs: req.DocIDs,- Page: searchPage,- PageSize: rerankLimit,- Top: *req.Top,- RankFeature: *req.RankFeature,- EmbeddingModel: req.EmbeddingModel,- Filter: req.Filter,+ TenantIDs: req.TenantIDs,+ Question: req.Question,+ KbIDs: req.KbIDs,+ DocIDs: req.DocIDs,+ Page: searchPage,+ PageSize: rerankLimit,+ Top: *req.Top,+ RankFeature: *req.RankFeature,+ EmbeddingModel: req.EmbeddingModel,+ VectorSimilarityWeight: req.VectorSimilarityWeight,+ Filter: req.Filter,}searchResult, err := s.Search(ctx, searchReq)if err != nil {@@ -546,19 +547,48 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval// RetrievalSearchRequest is the request struct for RetrievalService.Search()type RetrievalSearchRequest struct {- Question string- TenantIDs []string- KbIDs []string- DocIDs []string- Top int- Page int- PageSize int- Sort bool- Highlight *bool- SimilarityThreshold float64- RankFeature map[string]float64- Filter map[string]interface{}- EmbeddingModel *models.EmbeddingModel+ Question string+ TenantIDs []string+ KbIDs []string+ DocIDs []string+ Top int+ Page int+ PageSize int+ Sort bool+ Highlight *bool+ SimilarityThreshold float64+ RankFeature map[string]float64+ Filter map[string]interface{}+ EmbeddingModel *models.EmbeddingModel+ VectorSimilarityWeight *float64+}++func buildInfinityFusionExpr(topn int, vectorSimilarityWeight *float64) *types.FusionExpr {+ vectorWeight := 0.3+ if vectorSimilarityWeight != nil {+ vectorWeight = *vectorSimilarityWeight+ }+ termWeight := math.Round((1.0-vectorWeight)*10000) / 10000++ return &types.FusionExpr{+ Method: "weighted_sum",+ TopN: topn,+ FusionParams: map[string]interface{}{+ "weights": fmt.Sprintf("%g,%g", termWeight, vectorWeight),+ },+ }+}++func buildRetrievalFusionExpr(docEngineType string, topn int, vectorSimilarityWeight *float64) *types.FusionExpr {+ if docEngineType == string(engine.EngineInfinity) {+ return buildInfinityFusionExpr(topn, vectorSimilarityWeight)+ }++ return &types.FusionExpr{+ Method: "weighted_sum",+ TopN: topn,+ FusionParams: map[string]interface{}{"weights": "0.05,0.95"},+ }}type RetrievalSearchResult struct {@@ -671,11 +701,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque}// Execute search with fusion- fusionExpr := &types.FusionExpr{- Method: "weighted_sum",- TopN: topk,- FusionParams: map[string]interface{}{"weights": "0.05,0.95"},- }+ fusionExpr := buildRetrievalFusionExpr(s.docEngine.GetType(), topk, req.VectorSimilarityWeight)// Build source with vector column for ESsearchSrc := make([]string, len(searchRequest.SelectFields))diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 1cb7f782d1..1db4372c9f 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -5,8 +5,11 @@ import ("fmt""testing"+ "ragflow/internal/common""ragflow/internal/dao"+ "ragflow/internal/engine""ragflow/internal/engine/types"+ modelModule "ragflow/internal/entity/models""gorm.io/gorm")@@ -185,3 +188,167 @@ func (e *retrievalCountEngine) RunSQL(context.Context, string, string, []string,func (e *retrievalCountEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {return nil}++func TestBuildInfinityFusionExprUsesVectorSimilarityWeight(t *testing.T) {+ tests := []struct {+ name string+ vectorSimilarityWeight *float64+ expectedWeights string+ }{+ {name: "default", vectorSimilarityWeight: nil, expectedWeights: "0.7,0.3"},+ {name: "text only", vectorSimilarityWeight: float64Ptr(0), expectedWeights: "1,0"},+ {name: "balanced", vectorSimilarityWeight: float64Ptr(0.5), expectedWeights: "0.5,0.5"},+ {name: "vector only", vectorSimilarityWeight: float64Ptr(1), expectedWeights: "0,1"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ expr := buildInfinityFusionExpr(10, tt.vectorSimilarityWeight)+ if expr.Method != "weighted_sum" {+ t.Fatalf("expected weighted_sum, got %q", expr.Method)+ }+ if expr.TopN != 10 {+ t.Fatalf("expected TopN=10, got %d", expr.TopN)+ }+ weights, ok := expr.FusionParams["weights"].(string)+ if !ok || weights != tt.expectedWeights {+ t.Fatalf("expected weights=%q, got %v", tt.expectedWeights, expr.FusionParams["weights"])+ }+ })+ }+}++func float64Ptr(value float64) *float64 { return &value }+func intPtr(value int) *int { return &value }++type captureSearchDocEngine struct {+ engine.DocEngine+ engineType string+ searchRequest *types.SearchRequest+ result *types.SearchResult+}++func (e *captureSearchDocEngine) GetType() string {+ return e.engineType+}++func (e *captureSearchDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.searchRequest = req+ if e.result != nil {+ return e.result, nil+ }+ return &types.SearchResult{Chunks: []map[string]interface{}{{"id": "chunk-1"}}, Total: 1}, nil+}+func (e *captureSearchDocEngine) GetChunkIDs(_ []map[string]interface{}) []string {+ return []string{"chunk-1"}+}+func (e *captureSearchDocEngine) GetFields(_ []map[string]interface{}, _ []string) map[string]map[string]interface{} {+ return map[string]map[string]interface{}{}+}+func (e *captureSearchDocEngine) GetAggregation(_ []map[string]interface{}, _ string) []map[string]interface{} {+ return []map[string]interface{}{}+}+func (e *captureSearchDocEngine) GetHighlight(_ []map[string]interface{}, _ []string, _ string) map[string]string {+ return nil+}++type captureEmbeddingDriver struct{ modelModule.ModelDriver }++func (d *captureEmbeddingDriver) Embed(_ context.Context, _ *string, _ modelModule.EmbedRequest, _ *modelModule.APIConfig, _ *modelModule.EmbeddingConfig, _ *common.ModelUsage) ([]modelModule.EmbeddingData, error) {+ return []modelModule.EmbeddingData{{Embedding: []float64{0.1, 0.2}}}, nil+}++func TestSearchPassesVectorSimilarityWeightToFusionExpr(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }+ vectorWeight := 0.8+ docEngine := &captureSearchDocEngine{engineType: string(engine.EngineInfinity)}+ service := NewRetrievalService(docEngine, nil)+ _, err := service.Search(context.Background(), &RetrievalSearchRequest{+ Question: "test question", TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"}, Page: 1, PageSize: 10, Top: 10,+ RankFeature: map[string]float64{}, EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}}, VectorSimilarityWeight: &vectorWeight,+ })+ if err != nil {+ t.Fatalf("Search failed: %v", err)+ }+ assertFusionWeights(t, docEngine.searchRequest, "0.2,0.8")+}++func TestRetrievalPassesVectorSimilarityWeightToSearch(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }+ vectorWeight := 0.8+ top := 10+ docEngine := &captureSearchDocEngine{+ engineType: string(engine.EngineInfinity),+ result: &types.SearchResult{Chunks: []map[string]interface{}{}, Total: 0},+ }+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ _, err := service.Retrieval(context.Background(), &RetrievalRequest{+ Question: "test question", TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"}, Page: 1, PageSize: 10, Top: &top,+ RankFeature: &map[string]float64{}, EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}}, VectorSimilarityWeight: &vectorWeight,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ assertFusionWeights(t, docEngine.searchRequest, "0.2,0.8")+}++func assertFusionWeights(t *testing.T, request *types.SearchRequest, want string) {+ t.Helper()+ if request == nil || len(request.MatchExprs) != 3 {+ t.Fatalf("expected three match expressions, got %#v", request)+ }+ fusionExpr, ok := request.MatchExprs[2].(*types.FusionExpr)+ if !ok {+ t.Fatalf("expected third match expression to be FusionExpr, got %T", request.MatchExprs[2])+ }+ if got := fusionExpr.FusionParams["weights"]; got != want {+ t.Fatalf("expected weights=%s, got %v", want, got)+ }+}++func TestBuildRetrievalFusionExprKeepsLegacyWeightsOutsideInfinity(t *testing.T) {+ expr := buildRetrievalFusionExpr(string(engine.EngineElasticsearch), 10, float64Ptr(0.8))++ if got := expr.FusionParams["weights"]; got != "0.05,0.95" {+ t.Fatalf("expected Elasticsearch weights=0.05,0.95, got %v", got)+ }+}++func TestSearchKeepsLegacyFusionWeightForElasticsearch(t *testing.T) {+ if GetQueryBuilder() == nil {+ globalQueryBuilder = NewQueryBuilder()+ }++ vectorWeight := 0.8+ docEngine := &captureSearchDocEngine{engineType: string(engine.EngineElasticsearch)}+ service := NewRetrievalService(docEngine, nil)++ _, err := service.Search(context.Background(), &RetrievalSearchRequest{+ Question: "test question",+ TenantIDs: []string{"tenant-1"},+ KbIDs: []string{"kb-1"},+ Page: 1,+ PageSize: 10,+ Top: 10,+ RankFeature: map[string]float64{},+ EmbeddingModel: &modelModule.EmbeddingModel{ModelDriver: &captureEmbeddingDriver{}},+ VectorSimilarityWeight: &vectorWeight,+ })+ if err != nil {+ t.Fatalf("Search failed: %v", err)+ }++ if docEngine.searchRequest == nil || len(docEngine.searchRequest.MatchExprs) != 3 {+ t.Fatalf("expected three match expressions, got %#v", docEngine.searchRequest)+ }+ fusionExpr, ok := docEngine.searchRequest.MatchExprs[2].(*types.FusionExpr)+ if !ok {+ t.Fatalf("expected third match expression to be FusionExpr, got %T", docEngine.searchRequest.MatchExprs[2])+ }+ if got := fusionExpr.FusionParams["weights"]; got != "0.05,0.95" {+ t.Fatalf("expected Elasticsearch weights=0.05,0.95, got %v", got)+ }+}diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex fd73e2ea09..be34d5b6a7 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -32,6 +32,16 @@ from common import settingsfrom common.misc_utils import thread_pool_exec+def build_fusion_expr(topn: int, vector_similarity_weight: float = 0.3) -> FusionExpr:+ """Build the Infinity weighted-sum expression from the vector weight."""+ term_similarity_weight = 1 - vector_similarity_weight+ return FusionExpr(+ "weighted_sum",+ topn,+ {"weights": f"{term_similarity_weight:g},{vector_similarity_weight:g}"},+ )++def index_name(uid):return f"ragflow_{uid}"@@ -207,7 +217,15 @@ class Dealer:if settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_SERENEDB:src.append(f"q_{len(q_vec)}_vec")- if settings.DOC_ENGINE_GAUSSDB:+ if settings.DOC_ENGINE_INFINITY:+ vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))+ logging.debug(+ "Dealer.search fusion: topk=%s vector_similarity_weight=%s",+ topk,+ vector_similarity_weight,+ )+ fusionExpr = build_fusion_expr(topk, vector_similarity_weight)+ elif settings.DOC_ENGINE_GAUSSDB:vector_weight = req.get("vector_similarity_weight", 0.3)fusionExpr = FusionExpr("weighted_sum", topk, {"weights": f"{1 - float(vector_weight)},{float(vector_weight)}"})else:diff --git a/test/unit_test/rag/test_search_fusion_weight.py b/test/unit_test/rag/test_search_fusion_weight.pynew file mode 100644index 0000000000..fa04b203d4--- /dev/null+++ b/test/unit_test/rag/test_search_fusion_weight.py@@ -0,0 +1,190 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import importlib.util+import logging+from pathlib import Path+import sys+import types++import pytest++from common.doc_store.doc_store_base import MatchTextExpr, FusionExpr+++class _DummyFulltextQueryer:+ def question(self, text, min_match=0):+ return MatchTextExpr(fields=["content_ltks"], matching_text=text, topn=10, extra_options={}), ["keyword"]++ def token_similarity(self, keywords, documents):+ return [0.4] * len(documents)+++_ROOT = Path(__file__).parents[3]+++def search_environment(monkeypatch):+ fake_rag_nlp = types.ModuleType("rag.nlp")+ fake_rag_nlp.__path__ = []++ fake_tokenizer = types.ModuleType("rag.nlp.rag_tokenizer")+ fake_tokenizer.fine_grained_tokenize = lambda text: text+ fake_rag_nlp.rag_tokenizer = fake_tokenizer++ fake_query = types.ModuleType("rag.nlp.query")+ fake_query.FulltextQueryer = _DummyFulltextQueryer+ fake_rag_nlp.query = fake_query++ fake_settings = types.ModuleType("common.settings")+ fake_settings.DOC_ENGINE_OCEANBASE = False+ fake_settings.DOC_ENGINE_GAUSSDB = False+ fake_settings.DOC_ENGINE_SERENEDB = False+ fake_settings.DOC_ENGINE_INFINITY = True++ fake_token_utils = types.ModuleType("common.token_utils")+ fake_token_utils.num_tokens_from_string = lambda text: len(text.split())++ import common++ monkeypatch.setattr(common, "settings", fake_settings, raising=False)+ monkeypatch.setattr(common, "token_utils", fake_token_utils, raising=False)+ monkeypatch.setitem(sys.modules, "common.settings", fake_settings)+ monkeypatch.setitem(sys.modules, "common.token_utils", fake_token_utils)+ monkeypatch.setitem(sys.modules, "rag.nlp", fake_rag_nlp)+ monkeypatch.setitem(sys.modules, "rag.nlp.rag_tokenizer", fake_tokenizer)+ monkeypatch.setitem(sys.modules, "rag.nlp.query", fake_query)++ search_spec = importlib.util.spec_from_file_location("rag.nlp.search", _ROOT / "rag" / "nlp" / "search.py")+ search_module = importlib.util.module_from_spec(search_spec)+ assert search_spec.loader is not None+ monkeypatch.setitem(sys.modules, "rag.nlp.search", search_module)+ search_spec.loader.exec_module(search_module)++ return search_module+++class _FakeEmbeddingModel:+ def encode_queries(self, text):+ return [0.1, 0.2], None+++class _CapturingDataStore:+ def __init__(self):+ self.match_expressions = None++ def search(self, *args, **kwargs):+ if len(args[3]) == 3:+ self.match_expressions = args[3]+ return object()++ def get_total(self, result):+ return 1++ def get_doc_ids(self, result):+ return ["chunk-1"]++ def get_fields(self, result, fields):+ return {+ "chunk-1": {+ "_score": 1.0,+ "content_ltks": "test content",+ "content_with_weight": "test content",+ "kb_id": "kb-1",+ }+ }++ def get_scores(self, result):+ return {"chunk-1": 0.5}++ def get_highlight(self, result, keywords, field_name):+ return {}++ def get_aggregation(self, result, field_name):+ return []+++async def test_dealer_retrieval_passes_vector_similarity_weight_to_fusion_expr(search_environment, caplog):+ caplog.set_level(logging.DEBUG)+ data_store = _CapturingDataStore()+ dealer = search_environment.Dealer(data_store)++ await dealer.retrieval(+ question="test question",+ embd_mdl=_FakeEmbeddingModel(),+ tenant_ids=["tenant-1"],+ kb_ids=["kb-1"],+ page=1,+ page_size=10,+ similarity_threshold=0.0,+ vector_similarity_weight=0.8,+ top=10,+ aggs=False,+ )++ assert data_store.match_expressions is not None+ assert len(data_store.match_expressions) == 3+ fusion_expr = data_store.match_expressions[2]+ assert isinstance(fusion_expr, FusionExpr)+ assert fusion_expr.fusion_params["weights"] == "0.2,0.8"+ assert "Dealer.search fusion" in caplog.text+ assert "vector_similarity_weight=0.8" in caplog.text+++async def test_dealer_retrieval_keeps_elasticsearch_fusion_weight(search_environment):+ data_store = _CapturingDataStore()+ dealer = search_environment.Dealer(data_store)++ import common++ common.settings.DOC_ENGINE_INFINITY = False++ await dealer.retrieval(+ question="test question",+ embd_mdl=_FakeEmbeddingModel(),+ tenant_ids=["tenant-1"],+ kb_ids=["kb-1"],+ page=1,+ page_size=10,+ similarity_threshold=0.0,+ vector_similarity_weight=0.8,+ top=10,+ aggs=False,+ )++ assert data_store.match_expressions is not None+ fusion_expr = data_store.match_expressions[2]+ assert isinstance(fusion_expr, FusionExpr)+ assert fusion_expr.fusion_params["weights"] == "0.001,1"+++ ("vector_similarity_weight", "expected_weights"),+ [+ (0.0, "1,0"),+ (0.3, "0.7,0.3"),+ (0.5, "0.5,0.5"),+ (1.0, "0,1"),+ ],+)+def test_build_fusion_expr_uses_vector_similarity_weight(search_environment, vector_similarity_weight, expected_weights):+ fusion_expr = search_environment.build_fusion_expr(topn=10, vector_similarity_weight=vector_similarity_weight)++ assert fusion_expr.method == "weighted_sum"+ assert fusion_expr.topn == 10+ assert fusion_expr.fusion_params["weights"] == expected_weights