RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(utils): round minimum_should_match fraction instead of truncating (#19031)
SYED ALI ABBAS RAHILd33368cd30df
diff --git a/common/float_utils.py b/common/float_utils.pyindex d7ef42fbbe..de4681e643 100644--- a/common/float_utils.py+++ b/common/float_utils.py@@ -56,3 +56,28 @@ def normalize_overlapped_percent(overlapped_percent):value *= 100value = int(value)return max(0, min(value, 90))+++def format_minimum_should_match_percent(fraction: float) -> str:+ """Convert a fraction to a percentage string for full-text search.++ Floating-point fractions such as 0.29 are not exactly representable in+ IEEE-754, so `int(fraction * 100)` truncates toward zero and can produce a+ string one percentage point stricter than requested. This helper rounds+ half-up to the nearest whole percent instead.++ Args:+ fraction: A fraction in the range [0, 1].++ Returns:+ A percentage string like "29%".++ Examples:+ >>> format_minimum_should_match_percent(0.29)+ '29%'+ >>> format_minimum_should_match_percent(0.57)+ '57%'+ >>> format_minimum_should_match_percent(0.125)+ '13%'+ """+ return str(int(fraction * 100 + 0.5)) + "%"diff --git a/internal/common/format.go b/internal/common/format.goindex 5cd807a0e9..98996edc79 100644--- a/internal/common/format.go+++ b/internal/common/format.go@@ -19,6 +19,7 @@ package commonimport ("encoding/base64""fmt"+ "math""regexp""slices""strconv"@@ -121,6 +122,16 @@ func FormatBytes(bytes int64) string {}}+// FormatMinimumShouldMatchPercent converts a fractional minimum_should_match+// value to a percentage string using half-up rounding, mirroring Python's+// common.float_utils.format_minimum_should_match_percent.+func FormatMinimumShouldMatchPercent(fraction float64) string {+ // Add a tiny epsilon so values whose true product ends in .5 but are+ // represented slightly below it (e.g. 0.285*100 == 28.499...) still+ // round half-up instead of being truncated.+ return fmt.Sprintf("%d%%", int(math.Floor(fraction*100+0.5+1e-9)))+}+func FormatNumber(n int64) string {s := fmt.Sprintf("%d", n)parts := []string{}diff --git a/internal/common/format_test.go b/internal/common/format_test.goindex 24bc1f0bcf..936c896b25 100644--- a/internal/common/format_test.go+++ b/internal/common/format_test.go@@ -54,6 +54,28 @@ func TestChunkID_PreservesLeadingZero(t *testing.T) {}}+func TestFormatMinimumShouldMatchPercent(t *testing.T) {+ tests := []struct {+ name string+ fraction float64+ want string+ }{+ {"exact percent", 0.29, "29%"},+ {"half rounds up", 0.125, "13%"},+ {"just over half rounds up", 0.135, "14%"},+ {"floating-point .5 rounds up", 0.285, "29%"},+ {"zero", 0.0, "0%"},+ {"one hundred", 1.0, "100%"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := FormatMinimumShouldMatchPercent(tt.fraction); got != tt.want {+ t.Fatalf("FormatMinimumShouldMatchPercent(%g) = %q, want %q", tt.fraction, got, tt.want)+ }+ })+ }+}+func TestBaseModelName(t *testing.T) {tests := []struct {name stringdiff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 9a5cbfa1cd..4831a3da8b 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1950,7 +1950,7 @@ func buildQueryStringQuery(matchText *types.MatchTextExpr, vectorSimilarityWeighminimumShouldMatch := "0%"if matchText.ExtraOptions != nil {if msm, ok := matchText.ExtraOptions["minimum_should_match"].(float64); ok {- minimumShouldMatch = fmt.Sprintf("%d%%", int(msm*100))+ minimumShouldMatch = common.FormatMinimumShouldMatchPercent(msm)}}diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.goindex 7d159806e0..6222a10014 100644--- a/internal/engine/elasticsearch/chunk_test.go+++ b/internal/engine/elasticsearch/chunk_test.go@@ -24,6 +24,7 @@ import ("testing""ragflow/internal/common"+ "ragflow/internal/engine/types")// makeResponse builds a SearchResponse with `n` synthetic hits whose id@@ -581,3 +582,26 @@ func TestFormatOrderedTagFeas(t *testing.T) {t.Fatalf("encoded JSON does not preserve score-descending order:\n%s", encodedStr)}}++func TestBuildQueryStringQueryMinimumShouldMatchHalfUp(t *testing.T) {+ tests := []struct {+ fraction float64+ want string+ }{+ {0.29, "29%"},+ {0.125, "13%"},+ {0.135, "14%"},+ {0.0, "0%"},+ {1.0, "100%"},+ }+ for _, tc := range tests {+ query := buildQueryStringQuery(&types.MatchTextExpr{+ MatchingText: "hello",+ ExtraOptions: map[string]interface{}{"minimum_should_match": tc.fraction},+ }, 0.5, false, false)+ got := query["query_string"].(map[string]interface{})["minimum_should_match"].(string)+ if got != tc.want {+ t.Errorf("buildQueryStringQuery minimum_should_match for %g = %q, want %q", tc.fraction, got, tc.want)+ }+ }+}diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex c8a13e5953..5299fbb9a9 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -977,7 +977,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S// Add text match if question is providedif hasTextMatch {extraOptions := map[string]string{- "minimum_should_match": fmt.Sprintf("%d%%", int(minMatch*100)),+ "minimum_should_match": common.FormatMinimumShouldMatchPercent(minMatch),}if filterStr != "" {diff --git a/internal/engine/oceanbase/compatibility_test.go b/internal/engine/oceanbase/compatibility_test.goindex a93101e775..ce660853c5 100644--- a/internal/engine/oceanbase/compatibility_test.go+++ b/internal/engine/oceanbase/compatibility_test.go@@ -247,6 +247,33 @@ func TestDBMSHybridBodyMatchesPythonSemantics(t *testing.T) {}}+func TestDBMSHybridBodyMinimumShouldMatchHalfUp(t *testing.T) {+ plan := searchPlan{+ text: &types.MatchTextExpr{MatchingText: "hello", TopN: 10, ExtraOptions: map[string]interface{}{"minimum_should_match": 0.285}},+ dense: &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, EmbeddingDataType: "float", TopN: 8, ExtraOptions: map[string]interface{}{"similarity": 0.42}},+ fusion: &types.FusionExpr{Method: "weighted_sum", FusionParams: map[string]interface{}{"weights": "0.25,0.75"}},+ }+ body, ok := buildDBMSBody("chunk", map[string]interface{}{"kb_id": []string{"kb-1"}, "available_int": 0}, &types.SearchRequest{+ Offset: 2, Limit: 5, RankFeature: map[string]float64{"pagerank_fea": 0.1},+ }, plan)+ if !ok {+ t.Fatal("hybrid body unexpectedly required SQL fallback")+ }+ root := body["query"].(map[string]interface{})+ query := root["bool"].(map[string]interface{})+ must := query["must"].([]interface{})[0].(map[string]interface{})["query_string"].(map[string]interface{})+ if got := must["minimum_should_match"]; got != "29%" {+ t.Fatalf("minimum_should_match for 0.285 = %q, want 29%%", got)+ }+ knn, ok := body["knn"].(map[string]interface{})+ if !ok {+ t.Fatalf("hybrid knn leg = %#v", body["knn"])+ }+ if knn["k"] != 8 || knn["num_candidates"] != 16 || knn["similarity"] != 0.42 {+ t.Fatalf("hybrid vector leg = %#v", knn)+ }+}+func TestMetadataJSONPushdownOperators(t *testing.T) {predicate, args, err := buildMetaPushdownPredicate([]map[string]interface{}{{"key": "author", "op": "contains", "value": "Alice"},diff --git a/internal/engine/oceanbase/search.go b/internal/engine/oceanbase/search.goindex 939bcdcc5e..a1f0aa8376 100644--- a/internal/engine/oceanbase/search.go+++ b/internal/engine/oceanbase/search.go@@ -729,9 +729,9 @@ func buildDBMSBody(kind string, condition map[string]interface{}, req *types.Sea}}if value, ok := minimumShouldMatch.(float64); ok {- minimumShouldMatch = strconv.Itoa(int(value*100)) + "%"+ minimumShouldMatch = common.FormatMinimumShouldMatchPercent(value)} else if value, ok := minimumShouldMatch.(float32); ok {- minimumShouldMatch = strconv.Itoa(int(value*100)) + "%"+ minimumShouldMatch = common.FormatMinimumShouldMatchPercent(float64(value))}boolQuery := map[string]interface{}{"must": []interface{}{map[string]interface{}{"query_string": map[string]interface{}{diff --git a/memory/utils/es_conn.py b/memory/utils/es_conn.pyindex f49e39705f..70e2666416 100644--- a/memory/utils/es_conn.py+++ b/memory/utils/es_conn.py@@ -25,7 +25,7 @@ from elastic_transport import ConnectionTimeoutfrom common.decorator import singletonfrom common.doc_store.doc_store_base import MatchExpr, OrderByExpr, MatchTextExpr, MatchDenseExpr, FusionExprfrom common.doc_store.es_conn_base import ESConnectionBase-from common.float_utils import get_float+from common.float_utils import format_minimum_should_match_percent, get_floatfrom common.constants import PAGERANK_FLD, TAG_FLDfrom rag.nlp.rag_tokenizer import tokenize, fine_grained_tokenize@@ -171,7 +171,7 @@ class ESConnection(ESConnectionBase):if isinstance(m, MatchTextExpr):minimum_should_match = m.extra_options.get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ minimum_should_match = format_minimum_should_match_percent(minimum_should_match)bool_query.must.append(Q("query_string",diff --git a/memory/utils/infinity_conn.py b/memory/utils/infinity_conn.pyindex feeabe591f..069548c5aa 100644--- a/memory/utils/infinity_conn.py+++ b/memory/utils/infinity_conn.py@@ -25,6 +25,7 @@ import pandas as pdfrom common.doc_store.doc_store_base import MatchExpr, MatchTextExpr, MatchDenseExpr, FusionExpr, OrderByExprfrom common.doc_store.infinity_conn_base import InfinityConnectionBasefrom common.time_utils import date_string_to_timestamp+from common.float_utils import format_minimum_should_match_percent@singleton@@ -187,7 +188,7 @@ class InfinityConnection(InfinityConnectionBase):filter_fulltext = f"({filter_cond}) AND {filter_fulltext}"minimum_should_match = matchExpr.extra_options.get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- str_minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ str_minimum_should_match = format_minimum_should_match_percent(minimum_should_match)matchExpr.extra_options["minimum_should_match"] = str_minimum_should_matchfor k, v in matchExpr.extra_options.items():diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.pyindex 1919ee7e1b..4cb66d6af5 100644--- a/rag/utils/es_conn.py+++ b/rag/utils/es_conn.py@@ -26,7 +26,7 @@ from common.constants import PAGERANK_FLD, TAG_FLDfrom common.decorator import singletonfrom common.doc_store.doc_store_base import FusionExpr, MatchDenseExpr, MatchExpr, MatchTextExpr, OrderByExprfrom common.doc_store.es_conn_base import ESConnectionBase-from common.float_utils import get_float+from common.float_utils import format_minimum_should_match_percent, get_floatATTEMPT_TIME = 2MAX_RESULT_WINDOW = 10000@@ -246,7 +246,7 @@ class ESConnection(ESConnectionBase):if isinstance(m, MatchTextExpr):minimum_should_match = (m.extra_options or {}).get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ minimum_should_match = format_minimum_should_match_percent(minimum_should_match)bool_query.must.append(Q("query_string", fields=m.fields, type="best_fields", query=m.matching_text, minimum_should_match=minimum_should_match, boost=1))bool_query.boost = 1.0 - vector_similarity_weightdiff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.pyindex 947220bb19..236fa6dbd0 100644--- a/rag/utils/infinity_conn.py+++ b/rag/utils/infinity_conn.py@@ -24,7 +24,7 @@ import pandas as pdfrom common.constants import PAGERANK_FLD, TAG_FLDfrom common.doc_store.doc_store_base import MatchExpr, MatchTextExpr, MatchDenseExpr, FusionExpr, OrderByExprfrom common.doc_store.infinity_conn_base import InfinityConnectionBase-from common.float_utils import get_float+from common.float_utils import format_minimum_should_match_percent, get_floatDENSE_FILTER_FULLTEXT_WEIGHT_THRESHOLD = 0.8@@ -227,7 +227,7 @@ class InfinityConnection(InfinityConnectionBase):filter_fulltext = f"({filter_cond}) AND {filter_fulltext}"minimum_should_match = matchExpr.extra_options.get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- str_minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ str_minimum_should_match = format_minimum_should_match_percent(minimum_should_match)matchExpr.extra_options["minimum_should_match"] = str_minimum_should_match# Add rank_feature supportdiff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.pyindex 45a8e05a4a..4205215a23 100644--- a/rag/utils/ob_conn.py+++ b/rag/utils/ob_conn.py@@ -40,7 +40,7 @@ from common.doc_store.ob_conn_base import (doc_meta_column_names,doc_meta_column_types,)-from common.float_utils import get_float+from common.float_utils import format_minimum_should_match_percent, get_floatfrom rag.nlp import rag_tokenizerlogger = logging.getLogger("ragflow.ob_conn")@@ -635,7 +635,7 @@ class OBConnection(OBConnectionBase):if isinstance(m, MatchTextExpr):minimum_should_match = m.extra_options.get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ minimum_should_match = format_minimum_should_match_percent(minimum_should_match)bqry.must.append(Q("query_string", fields=FTS_COLUMNS_TKS, type="best_fields", query=m.matching_text, minimum_should_match=minimum_should_match, boost=1))bqry.boost = 1.0 - vector_similarity_weightdiff --git a/rag/utils/opensearch_conn.py b/rag/utils/opensearch_conn.pyindex fc3ae1dd0b..b29b236781 100644--- a/rag/utils/opensearch_conn.py+++ b/rag/utils/opensearch_conn.py@@ -30,6 +30,7 @@ from common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, Orderfrom rag.nlp import is_english, rag_tokenizerfrom common.constants import PAGERANK_FLD, TAG_FLDfrom common import settings+from common.float_utils import format_minimum_should_match_percentATTEMPT_TIME = 2@@ -385,7 +386,7 @@ class OSConnection(DocStoreConnection):use_text = Trueminimum_should_match = m.extra_options.get("minimum_should_match", 0.0)if isinstance(minimum_should_match, float):- minimum_should_match = str(int(minimum_should_match * 100)) + "%"+ minimum_should_match = format_minimum_should_match_percent(minimum_should_match)bqry.must.append(Q("query_string", fields=m.fields, type="best_fields", query=m.matching_text, minimum_should_match=minimum_should_match, boost=1))bqry.boost = 1.0 - vector_similarity_weightdiff --git a/test/unit_test/common/test_float_utils.py b/test/unit_test/common/test_float_utils.pyindex a33d9c8009..5068959508 100644--- a/test/unit_test/common/test_float_utils.py+++ b/test/unit_test/common/test_float_utils.py@@ -15,7 +15,7 @@#import math-from common.float_utils import get_float+from common.float_utils import format_minimum_should_match_percent, get_floatclass TestGetFloat:@@ -86,3 +86,24 @@ class TestGetFloat:result = get_float(" invalid ")assert math.isinf(result)assert result < 0+++class TestFormatMinimumShouldMatchPercent:+ def test_fractions_round_to_nearest_percent(self):+ """Test that floating fractions round to the nearest whole percent."""+ assert format_minimum_should_match_percent(0.29) == "29%"+ assert format_minimum_should_match_percent(0.57) == "57%"+ assert format_minimum_should_match_percent(0.58) == "58%"++ def test_half_up_rounding(self):+ """Half percent values must round up, not use banker's rounding."""+ assert format_minimum_should_match_percent(0.125) == "13%"+ assert format_minimum_should_match_percent(0.135) == "14%"++ def test_zero_and_one(self):+ assert format_minimum_should_match_percent(0.0) == "0%"+ assert format_minimum_should_match_percent(1.0) == "100%"++ def test_representational_error_not_truncated(self):+ """0.29 * 100 is 28.999... in IEEE-754; must round to 29, not truncate."""+ assert format_minimum_should_match_percent(0.29) != "28%"