RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(memory): put the keyword weight in the text slot of the fusion expression (#18989)
MarMar Labscd27ae4c6ef5
diff --git a/api/db/joint_services/memory_message_service.py b/api/db/joint_services/memory_message_service.pyindex 1405a3a423..7a01644369 100644--- a/api/db/joint_services/memory_message_service.py+++ b/api/db/joint_services/memory_message_service.py@@ -289,7 +289,11 @@ def query_message(filter_dict: dict, params: dict):match_dense = get_vector(question, embd_model, similarity=params["similarity_threshold"])match_text, _ = MsgTextQuery().question(question, min_match=params["similarity_threshold"])keywords_similarity_weight = params.get("keywords_similarity_weight", 0.7)- fusion_expr = FusionExpr("weighted_sum", params["top_n"], {"weights": ",".join([str(1 - keywords_similarity_weight), str(keywords_similarity_weight)])})+ # Slot order is [text, vector]. Every adapter behind msgStoreConn reads slot 1 as+ # the vector weight and derives the text weight from it: see memory/utils/es_conn.py,+ # which then sets the text query's boost to 1 - that value, and the note in+ # memory/utils/gaussdb_conn.py, "FusionExpr orders weights as text_weight,vector_weight".+ fusion_expr = FusionExpr("weighted_sum", params["top_n"], {"weights": f"{keywords_similarity_weight:g},{1 - keywords_similarity_weight:g}"})return MessageService.search_message(memory_ids, condition_dict, uids, [match_text, match_dense, fusion_expr], params["top_n"])diff --git a/internal/service/memory.go b/internal/service/memory.goindex a9ec3f9a07..fc34a0f173 100644--- a/internal/service/memory.go+++ b/internal/service/memory.go@@ -1129,7 +1129,7 @@ func (s *MemoryService) queryMessage(ctx context.Context, memories []*entity.MemMethod: "weighted_sum",TopN: topN,FusionParams: map[string]interface{}{- "weights": fmt.Sprintf("%g,%g", 1-keywordsSimilarityWeight, keywordsSimilarityWeight),+ "weights": memoryFusionWeights(keywordsSimilarityWeight),},}matchExprs = append(matchExprs, matchText, matchDense, fusionExpr)@@ -1365,6 +1365,21 @@ func memoryMessageTextExpr(question string, similarityThreshold float64) *enginereturn matchText}+// memoryFusionWeights formats FusionExpr weights, whose slot order is [text, vector]:+// the Elasticsearch, OceanBase and SereneDB adapters all read slot 1 as the vector+// weight, and Elasticsearch then boosts the text query by 1 - that value. The keyword+// weight is the text weight, so it belongs in slot 0. Sending it to slot 1 gave every+// memory search the inverse of the requested hybrid balance.+//+// Six significant digits rather than %g for the same reason serenedb's formatWeight+// rounds: 1 - 0.7 would otherwise render as 0.30000000000000004, which is also what the+// Python side deliberately formats away.+func memoryFusionWeights(keywordsSimilarityWeight float64) string {+ textWeight := keywordsSimilarityWeight+ vectorWeight := 1 - keywordsSimilarityWeight+ return fmt.Sprintf("%.6g,%.6g", textWeight, vectorWeight)+}+func (s *MemoryService) memoryMessageDenseExpr(ctx context.Context, question string, memory *entity.Memory, topN int, similarityThreshold float64) (*enginetypes.MatchDenseExpr, error) {driver, modelName, apiConfig, maxTokens, err := NewModelProviderService().ResolveModelConfig(ctx, memory.TenantID, entity.ModelTypeEmbedding, memory.EmbdID)if err != nil {diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.goindex 6a35f01c63..29ac48f49e 100644--- a/internal/service/memory_message_test.go+++ b/internal/service/memory_message_test.go@@ -4,7 +4,9 @@ import ("context""errors""fmt"+ "math""reflect"+ "strconv""strings""testing""time"@@ -39,6 +41,60 @@ func TestMemoryIndexNameMatchesPythonPrefix(t *testing.T) {}}+// The FusionExpr weight slots are [text, vector], and keywords_similarity_weight is the+// text weight, so it belongs in slot 0. Both this and Python's+// api/db/joint_services/memory_message_service.py emitted the pair reversed, which handed+// every memory search the inverse of the requested hybrid balance. The weights below are+// asymmetric on purpose: an even split cannot tell the two orders apart.+func TestMemoryFusionWeightsPutTheKeywordWeightInTheTextSlot(t *testing.T) {+ for _, keywordsSimilarityWeight := range []float64{0.7, 0.9, 0.2} {+ weights := memoryFusionWeights(keywordsSimilarityWeight)+ parts := strings.Split(weights, ",")+ if len(parts) != 2 {+ t.Fatalf("memoryFusionWeights(%v) = %q, want two comma-separated weights", keywordsSimilarityWeight, weights)+ }++ textWeight, err := strconv.ParseFloat(parts[0], 64)+ if err != nil {+ t.Fatalf("text weight %q: %v", parts[0], err)+ }+ vectorWeight, err := strconv.ParseFloat(parts[1], 64)+ if err != nil {+ t.Fatalf("vector weight %q: %v", parts[1], err)+ }++ // Compared with a tolerance, not for equality: the slot each weight lands in is+ // the invariant, and %.2f is as valid a rendering here as %.6g.+ if math.Abs(textWeight-keywordsSimilarityWeight) > 1e-9 {+ t.Fatalf("text weight = %v, want %v (from %q)", textWeight, keywordsSimilarityWeight, weights)+ }+ if math.Abs(vectorWeight-(1-keywordsSimilarityWeight)) > 1e-9 {+ t.Fatalf("vector weight = %v, want %v (from %q)", vectorWeight, 1-keywordsSimilarityWeight, weights)+ }+ }+}++// The slot test above deliberately accepts any numeric rendering, so nothing there+// would notice a quiet return to %g. This pins the other half: the strings below are+// what Python's :g emits for the same inputs in+// api/db/joint_services/memory_message_service.py, so a regression that reintroduces+// 0.30000000000000004 on the Go side turns this red. 0.1234567 is here because it is+// the case that actually exercises the six-digit rounding.+func TestMemoryFusionWeightsMatchesPythonFormatting(t *testing.T) {+ for _, testCase := range []struct {+ keywordsSimilarityWeight float64+ want string+ }{+ {keywordsSimilarityWeight: 0.7, want: "0.7,0.3"},+ {keywordsSimilarityWeight: 0.9, want: "0.9,0.1"},+ {keywordsSimilarityWeight: 0.1234567, want: "0.123457,0.876543"},+ } {+ if got := memoryFusionWeights(testCase.keywordsSimilarityWeight); got != testCase.want {+ t.Fatalf("memoryFusionWeights(%v) = %q, want %q", testCase.keywordsSimilarityWeight, got, testCase.want)+ }+ }+}+func TestValidateMemorySearchModels(t *testing.T) {firstTenantEmbeddingID := "tenant-embedding-1"secondTenantEmbeddingID := "tenant-embedding-2"diff --git a/test/unit_test/api/db/joint_services/test_memory_message_service.py b/test/unit_test/api/db/joint_services/test_memory_message_service.pyindex 3ce266aea3..aafa478138 100644--- a/test/unit_test/api/db/joint_services/test_memory_message_service.py+++ b/test/unit_test/api/db/joint_services/test_memory_message_service.py@@ -80,3 +80,36 @@ async def test_save_extracted_to_memory_only_forwards_custom_extraction_prompts(assert result == (True, "No memory extracted from raw message.")assert_custom_prompts_forwarded(extract_by_llm)+++def test_query_message_puts_the_keyword_weight_in_the_text_slot(monkeypatch):+ """FusionExpr weights are "text,vector", so keywords_similarity_weight belongs first.++ The adapters behind msgStoreConn all read slot 1 as the vector weight, so sending+ the keyword weight there inverted hybrid ranking for every memory search.+ """+ captured = {}++ def fake_search_message(_memory_ids, _condition_dict, _uid_list, match_expressions, _top_n):+ captured["match_expressions"] = match_expressions+ return []++ memory = SimpleNamespace(id="memory-1", tenant_id="tenant-1", embd_id="embd-1")+ monkeypatch.setattr(memory_message_service.MemoryService, "get_by_ids", lambda _ids: [memory])+ monkeypatch.setattr(memory_message_service, "resolve_model_config", lambda *_a, **_k: {})+ monkeypatch.setattr(memory_message_service, "LLMBundle", lambda *_a, **_k: object())+ monkeypatch.setattr(memory_message_service, "get_vector", lambda *_a, **_k: SimpleNamespace())+ monkeypatch.setattr(memory_message_service, "MsgTextQuery", lambda: SimpleNamespace(question=lambda *_a, **_k: (SimpleNamespace(), None)))+ monkeypatch.setattr(memory_message_service.MessageService, "search_message", fake_search_message)++ memory_message_service.query_message(+ {"memory_id": ["memory-1"]},+ {"query": "hello", "similarity_threshold": 0.2, "keywords_similarity_weight": 0.7, "top_n": 5},+ )++ fusion_expr = captured["match_expressions"][2]+ text_weight, vector_weight = (float(part) for part in fusion_expr.fusion_params["weights"].split(","))++ assert text_weight == pytest.approx(0.7)+ assert vector_weight == pytest.approx(0.3)