RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: honor minimum_should_match in GaussDB search (#18456)
Sevenzuof1e6b22c4ba5
diff --git a/common/doc_store/gaussdb_conn_base.py b/common/doc_store/gaussdb_conn_base.pyindex 684248b6d0..95bbe2a90e 100644--- a/common/doc_store/gaussdb_conn_base.py+++ b/common/doc_store/gaussdb_conn_base.py@@ -1041,6 +1041,7 @@ class GaussDBSearchBuilder:offset: int,limit: int,similarity_threshold: float | None = None,+ minimum_should_match: float | str | None = None,topn: int | None = None,highlight_fields: list[str] | None = None,order_by: OrderByExpr | None = None,@@ -1064,6 +1065,7 @@ class GaussDBSearchBuilder:vector_dim=vector_dim,vector_weight=vector_weight,similarity_threshold=similarity_threshold,+ minimum_should_match=minimum_should_match,candidate_limit=candidate_limit,offset=effective_offset,limit=page_limit,@@ -1089,6 +1091,7 @@ class GaussDBSearchBuilder:select_fields=select_fields,condition=condition,keywords=query_keywords,+ minimum_should_match=minimum_should_match,offset=effective_offset,limit=page_limit,highlight_fields=highlight_fields,@@ -1158,10 +1161,15 @@ class GaussDBSearchBuilder:params.append(value)return " AND ".join(fragments), params- def build_text_score_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:+ def build_text_score_expr(+ self,+ keywords: list[str],+ minimum_should_match: float | str | None = None,+ ) -> tuple[str, list[Any]]:score_exprs: list[str] = []params: list[Any] = []- for config, query_text in self._text_queries(keywords):+ text_queries = self._text_queries(keywords) if minimum_should_match is None else self._text_query_terms(keywords)+ for config, query_text in text_queries:weighted_score = " + ".join(f"{weight} * COALESCE(ts_rank(to_tsvector('{config}', coalesce({column}, ' ')), plainto_tsquery('{config}', %s)), 0)" for column, weight in self.FTS_WEIGHTS.items())@@ -1273,6 +1281,7 @@ class GaussDBSearchBuilder:select_fields: list[str],condition: dict,keywords: list[str],+ minimum_should_match: float | str | None,offset: int,limit: int,highlight_fields: list[str] | None,@@ -1280,9 +1289,9 @@ class GaussDBSearchBuilder:) -> tuple[str, list[Any]]:table_name = self.ddl.qualified_name(table)columns = self.normalize_select_fields(select_fields)- score_expr, score_params = self.build_text_score_expr(keywords)+ score_expr, score_params = self.build_text_score_expr(keywords, minimum_should_match)score_expr, pagerank_params = self._score_with_pagerank(score_expr, pagerank_weight)- match_expr, match_params = self._build_text_match_expr(keywords)+ match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match)where_sql, where_params = self.build_condition_where(condition)where_parts = [part for part in (where_sql, match_expr) if part]select_exprs = [*self._select_exprs(columns), f"{score_expr} AS _score", "COUNT(*) OVER() AS __total"]@@ -1344,6 +1353,7 @@ class GaussDBSearchBuilder:vector_dim: int,vector_weight: float,similarity_threshold: float | None,+ minimum_should_match: float | str | None,candidate_limit: int,offset: int,limit: int,@@ -1356,8 +1366,8 @@ class GaussDBSearchBuilder:dim = self.ddl.validate_vector_dim(vector_dim)vector_col = self.ddl.vector_column_name(dim)valid_col = self.ddl.vector_valid_column_name(dim)- text_score_expr, text_score_params = self.build_text_score_expr(keywords)- match_expr, match_params = self._build_text_match_expr(keywords)+ text_score_expr, text_score_params = self.build_text_score_expr(keywords, minimum_should_match)+ match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match)where_sql, where_params = self.build_condition_where(condition)base_where = where_sql or "TRUE"fts_where = " AND ".join([base_where, match_expr])@@ -1413,7 +1423,26 @@ class GaussDBSearchBuilder:offset,]- def _build_text_match_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:+ def _build_text_match_expr(+ self,+ keywords: list[str],+ minimum_should_match: float | str | None = None,+ ) -> tuple[str, list[Any]]:+ if minimum_should_match is not None:+ text_query_terms = self._text_query_terms(keywords)+ predicates = [f"{self.build_fts_vector_expr(config)} @@ plainto_tsquery('{config}', %s)" for config, _query_text in text_query_terms]+ params = [query_text for _config, query_text in text_query_terms]+ required = self._minimum_should_match_count(minimum_should_match, len(predicates))+ if not predicates:+ return "FALSE", []+ if required <= 1:+ return "(" + " OR ".join(predicates) + ")", params+ if required >= len(predicates):+ return "(" + " AND ".join(predicates) + ")", params+ any_match = " OR ".join(predicates)+ match_count = " + ".join(f"CASE WHEN {predicate} THEN 1 ELSE 0 END" for predicate in predicates)+ return f"(({any_match}) AND ({match_count}) >= {required})", [*params, *params]+match_exprs: list[str] = []params: list[Any] = []for config, query_text in self._text_queries(keywords):@@ -1528,6 +1557,41 @@ class GaussDBSearchBuilder:queries.append(("ngram", self._text_query_param(ngram_terms)))return queries+ def _text_query_terms(self, keywords: list[str]) -> list[tuple[str, str]]:+ simple_terms, ngram_terms = self.split_text_query_terms(keywords)+ terms: list[tuple[str, str]] = []+ seen: set[tuple[str, str]] = set()+ for config, values in (("simple", simple_terms), ("ngram", ngram_terms)):+ for value in values:+ key = (config, value.casefold())+ if key in seen:+ continue+ seen.add(key)+ terms.append((config, value))+ return terms++ @staticmethod+ def _minimum_should_match_count(minimum_should_match: float | str, term_count: int) -> int:+ if term_count <= 0:+ return 0++ value: float | str = minimum_should_match+ if isinstance(value, str):+ text = value.strip()+ try:+ if text.endswith("%"):+ value = float(text[:-1]) / 100.0+ else:+ value = float(text)+ except ValueError:+ return 1++ if isinstance(value, float) and 0.0 <= value <= 1.0:+ required = int(term_count * value)+ else:+ required = int(value)+ return min(term_count, max(1, required))+def _vector_param(self, vector: list[float] | tuple[float, ...], vector_dim: int | None) -> str:if vector_dim is None:raise ValueError("vector_dim is required for vector search")diff --git a/rag/utils/gaussdb_conn.py b/rag/utils/gaussdb_conn.pyindex 0ce2958dec..91f73a0b44 100644--- a/rag/utils/gaussdb_conn.py+++ b/rag/utils/gaussdb_conn.py@@ -824,6 +824,7 @@ class GaussDBConnection(GaussDBConnectionBase):vector_dim=parsed["vector_dim"],vector_weight=parsed["vector_weight"],similarity_threshold=parsed["similarity_threshold"],+ minimum_should_match=parsed["minimum_should_match"],topn=parsed["topn"],offset=offset,limit=limit,@@ -845,6 +846,7 @@ class GaussDBConnection(GaussDBConnectionBase):vector_dim=parsed["vector_dim"],vector_weight=parsed["vector_weight"],similarity_threshold=parsed["similarity_threshold"],+ minimum_should_match=parsed["minimum_should_match"],topn=parsed["topn"],offset=0,limit=1,@@ -897,12 +899,14 @@ class GaussDBConnection(GaussDBConnectionBase):topn = Nonevector_weight = Nonesimilarity_threshold = None+ minimum_should_match = Nonepagerank_weight = 10.0for expr in match_expressions or []:if isinstance(expr, MatchTextExpr):query_text = (expr.extra_options or {}).get("original_query") or expr.matching_text or ""keywords = _tokenize_query_terms(query_text)+ minimum_should_match = (expr.extra_options or {}).get("minimum_should_match")topn = expr.topn if topn is None else min(topn, expr.topn)elif isinstance(expr, MatchDenseExpr):if expr.embedding_data_type != "float":@@ -931,6 +935,7 @@ class GaussDBConnection(GaussDBConnectionBase):"vector_dim": vector_dim,"vector_weight": vector_weight,"similarity_threshold": similarity_threshold,+ "minimum_should_match": minimum_should_match,"topn": topn,"pagerank_weight": pagerank_weight,}diff --git a/test/unit_test/rag/utils/test_gaussdb_conn.py b/test/unit_test/rag/utils/test_gaussdb_conn.pyindex 03f885b2c1..dc39a29bf5 100644--- a/test/unit_test/rag/utils/test_gaussdb_conn.py+++ b/test/unit_test/rag/utils/test_gaussdb_conn.py@@ -2822,7 +2822,7 @@ def test_tc_ret_805_parse_fusion_vector_weight_returns_none_for_missing_or_bad_wdef test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaults(monkeypatch):monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", lambda query: str(query).split())conn = make_conn(RecordingCursor())- text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello"})+ text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello", "minimum_should_match": 0.3})dense = MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", 10, {"similarity": 0.0})fusion = FusionExpr("weighted_sum", 6, {"weights": "0.7,0.3"})@@ -2832,7 +2832,9 @@ def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaulhybrid_explicit = conn._parse_match_expressions([text, dense, fusion], rank_feature={"pagerank_fea": 7})assert text_only["vector_weight"] == 0.0+ assert text_only["minimum_should_match"] == 0.3assert vector_only["vector_weight"] == 1.0+ assert vector_only["minimum_should_match"] is Noneassert hybrid_default["vector_weight"] == 0.5assert {text_only["pagerank_weight"], vector_only["pagerank_weight"], hybrid_default["pagerank_weight"]} == {10.0}assert hybrid_explicit["vector_weight"] == 0.3diff --git a/test/unit_test/rag/utils/test_gaussdb_search_sql.py b/test/unit_test/rag/utils/test_gaussdb_search_sql.pyindex 8c27422d1a..b4308cb6e6 100644--- a/test/unit_test/rag/utils/test_gaussdb_search_sql.py+++ b/test/unit_test/rag/utils/test_gaussdb_search_sql.py@@ -176,6 +176,28 @@ def test_tc_ret_109_chinese_ngram_query_preserves_english_semantics_and_highlighassert conn.get_highlight(result, ["深圳数据", "深圳"], "content_with_weight") == {"overlap": "<em>深圳数据</em>库审计"}+def test_tc_ret_111_minimum_should_match_uses_partial_unique_terms_and_term_scores():+ builder = GaussDBSearchBuilder(schema="public")+ keywords = ["你好", "一下", "王楠", "简历", "大概", "介绍", "一下"]++ match_sql, match_params = builder._build_text_match_expr(keywords, minimum_should_match=0.3)+ score_sql, score_params = builder.build_text_score_expr(keywords, minimum_should_match=0.3)++ unique_terms = ["你好", "一下", "王楠", "简历", "大概", "介绍"]+ assert " OR " in match_sql+ assert match_params == unique_terms+ assert " / 6.0)" in score_sql+ assert score_params == [term for term in unique_terms for _field in builder.FTS_WEIGHTS]+ assert builder._minimum_should_match_count(0.3, 6) == 1+ assert builder._minimum_should_match_count("50%", 6) == 3+ assert builder._minimum_should_match_count(2, 6) == 2++ two_match_sql, two_match_params = builder._build_text_match_expr(keywords, minimum_should_match=2)+ assert "CASE WHEN" in two_match_sql+ assert ">= 2" in two_match_sql+ assert two_match_params == unique_terms * 2++def test_tc_ret_203_vector_search_filters_invalid_placeholder_vectors():builder = GaussDBSearchBuilder(schema="public")sql, params = builder.build_search_sql(