Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: honor minimum_should_match in GaussDB search (#18456)

Sevenzuof1e6b22c4ba5
diff --git a/common/doc_store/gaussdb_conn_base.py b/common/doc_store/gaussdb_conn_base.py
index 684248b6d0..95bbe2a90e 100644
--- a/common/doc_store/gaussdb_conn_base.py
+++ b/common/doc_store/gaussdb_conn_base.py
@@ -1041,6 +1041,7 @@ class GaussDBSearchBuilder:
offset: int,
limit: int,
similarity_threshold: float | None = None,
+ minimum_should_match: float | str | None = None,
topn: int | None = None,
highlight_fields: list[str] | None = None,
order_by: OrderByExpr | None = None,
@@ -1064,6 +1065,7 @@ class GaussDBSearchBuilder:
vector_dim=vector_dim,
vector_weight=vector_weight,
similarity_threshold=similarity_threshold,
+ minimum_should_match=minimum_should_match,
candidate_limit=candidate_limit,
offset=effective_offset,
limit=page_limit,
@@ -1089,6 +1091,7 @@ class GaussDBSearchBuilder:
select_fields=select_fields,
condition=condition,
keywords=query_keywords,
+ minimum_should_match=minimum_should_match,
offset=effective_offset,
limit=page_limit,
highlight_fields=highlight_fields,
@@ -1158,10 +1161,15 @@ class GaussDBSearchBuilder:
params.append(value)
return " AND ".join(fragments), params
- def build_text_score_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:
+ def build_text_score_expr(
+ self,
+ keywords: list[str],
+ minimum_should_match: float | str | None = None,
+ ) -> tuple[str, list[Any]]:
score_exprs: list[str] = []
params: list[Any] = []
- for config, query_text in self._text_queries(keywords):
+ text_queries = self._text_queries(keywords) if minimum_should_match is None else self._text_query_terms(keywords)
+ for config, query_text in text_queries:
weighted_score = " + ".join(
f"{weight} * COALESCE(ts_rank(to_tsvector('{config}', coalesce({column}, ' ')), plainto_tsquery('{config}', %s)), 0)" for column, weight in self.FTS_WEIGHTS.items()
)
@@ -1273,6 +1281,7 @@ class GaussDBSearchBuilder:
select_fields: list[str],
condition: dict,
keywords: list[str],
+ minimum_should_match: float | str | None,
offset: int,
limit: int,
highlight_fields: list[str] | None,
@@ -1280,9 +1289,9 @@ class GaussDBSearchBuilder:
) -> tuple[str, list[Any]]:
table_name = self.ddl.qualified_name(table)
columns = self.normalize_select_fields(select_fields)
- score_expr, score_params = self.build_text_score_expr(keywords)
+ score_expr, score_params = self.build_text_score_expr(keywords, minimum_should_match)
score_expr, pagerank_params = self._score_with_pagerank(score_expr, pagerank_weight)
- match_expr, match_params = self._build_text_match_expr(keywords)
+ match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match)
where_sql, where_params = self.build_condition_where(condition)
where_parts = [part for part in (where_sql, match_expr) if part]
select_exprs = [*self._select_exprs(columns), f"{score_expr} AS _score", "COUNT(*) OVER() AS __total"]
@@ -1344,6 +1353,7 @@ class GaussDBSearchBuilder:
vector_dim: int,
vector_weight: float,
similarity_threshold: float | None,
+ minimum_should_match: float | str | None,
candidate_limit: int,
offset: int,
limit: int,
@@ -1356,8 +1366,8 @@ class GaussDBSearchBuilder:
dim = self.ddl.validate_vector_dim(vector_dim)
vector_col = self.ddl.vector_column_name(dim)
valid_col = self.ddl.vector_valid_column_name(dim)
- text_score_expr, text_score_params = self.build_text_score_expr(keywords)
- match_expr, match_params = self._build_text_match_expr(keywords)
+ text_score_expr, text_score_params = self.build_text_score_expr(keywords, minimum_should_match)
+ match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match)
where_sql, where_params = self.build_condition_where(condition)
base_where = where_sql or "TRUE"
fts_where = " AND ".join([base_where, match_expr])
@@ -1413,7 +1423,26 @@ class GaussDBSearchBuilder:
offset,
]
- def _build_text_match_expr(self, keywords: list[str]) -> tuple[str, list[Any]]:
+ def _build_text_match_expr(
+ self,
+ keywords: list[str],
+ minimum_should_match: float | str | None = None,
+ ) -> tuple[str, list[Any]]:
+ if minimum_should_match is not None:
+ text_query_terms = self._text_query_terms(keywords)
+ predicates = [f"{self.build_fts_vector_expr(config)} @@ plainto_tsquery('{config}', %s)" for config, _query_text in text_query_terms]
+ params = [query_text for _config, query_text in text_query_terms]
+ required = self._minimum_should_match_count(minimum_should_match, len(predicates))
+ if not predicates:
+ return "FALSE", []
+ if required <= 1:
+ return "(" + " OR ".join(predicates) + ")", params
+ if required >= len(predicates):
+ return "(" + " AND ".join(predicates) + ")", params
+ any_match = " OR ".join(predicates)
+ match_count = " + ".join(f"CASE WHEN {predicate} THEN 1 ELSE 0 END" for predicate in predicates)
+ return f"(({any_match}) AND ({match_count}) >= {required})", [*params, *params]
+
match_exprs: list[str] = []
params: list[Any] = []
for config, query_text in self._text_queries(keywords):
@@ -1528,6 +1557,41 @@ class GaussDBSearchBuilder:
queries.append(("ngram", self._text_query_param(ngram_terms)))
return queries
+ def _text_query_terms(self, keywords: list[str]) -> list[tuple[str, str]]:
+ simple_terms, ngram_terms = self.split_text_query_terms(keywords)
+ terms: list[tuple[str, str]] = []
+ seen: set[tuple[str, str]] = set()
+ for config, values in (("simple", simple_terms), ("ngram", ngram_terms)):
+ for value in values:
+ key = (config, value.casefold())
+ if key in seen:
+ continue
+ seen.add(key)
+ terms.append((config, value))
+ return terms
+
+ @staticmethod
+ def _minimum_should_match_count(minimum_should_match: float | str, term_count: int) -> int:
+ if term_count <= 0:
+ return 0
+
+ value: float | str = minimum_should_match
+ if isinstance(value, str):
+ text = value.strip()
+ try:
+ if text.endswith("%"):
+ value = float(text[:-1]) / 100.0
+ else:
+ value = float(text)
+ except ValueError:
+ return 1
+
+ if isinstance(value, float) and 0.0 <= value <= 1.0:
+ required = int(term_count * value)
+ else:
+ required = int(value)
+ return min(term_count, max(1, required))
+
def _vector_param(self, vector: list[float] | tuple[float, ...], vector_dim: int | None) -> str:
if vector_dim is None:
raise ValueError("vector_dim is required for vector search")
diff --git a/rag/utils/gaussdb_conn.py b/rag/utils/gaussdb_conn.py
index 0ce2958dec..91f73a0b44 100644
--- a/rag/utils/gaussdb_conn.py
+++ b/rag/utils/gaussdb_conn.py
@@ -824,6 +824,7 @@ class GaussDBConnection(GaussDBConnectionBase):
vector_dim=parsed["vector_dim"],
vector_weight=parsed["vector_weight"],
similarity_threshold=parsed["similarity_threshold"],
+ minimum_should_match=parsed["minimum_should_match"],
topn=parsed["topn"],
offset=offset,
limit=limit,
@@ -845,6 +846,7 @@ class GaussDBConnection(GaussDBConnectionBase):
vector_dim=parsed["vector_dim"],
vector_weight=parsed["vector_weight"],
similarity_threshold=parsed["similarity_threshold"],
+ minimum_should_match=parsed["minimum_should_match"],
topn=parsed["topn"],
offset=0,
limit=1,
@@ -897,12 +899,14 @@ class GaussDBConnection(GaussDBConnectionBase):
topn = None
vector_weight = None
similarity_threshold = None
+ minimum_should_match = None
pagerank_weight = 10.0
for expr in match_expressions or []:
if isinstance(expr, MatchTextExpr):
query_text = (expr.extra_options or {}).get("original_query") or expr.matching_text or ""
keywords = _tokenize_query_terms(query_text)
+ minimum_should_match = (expr.extra_options or {}).get("minimum_should_match")
topn = expr.topn if topn is None else min(topn, expr.topn)
elif isinstance(expr, MatchDenseExpr):
if expr.embedding_data_type != "float":
@@ -931,6 +935,7 @@ class GaussDBConnection(GaussDBConnectionBase):
"vector_dim": vector_dim,
"vector_weight": vector_weight,
"similarity_threshold": similarity_threshold,
+ "minimum_should_match": minimum_should_match,
"topn": topn,
"pagerank_weight": pagerank_weight,
}
diff --git a/test/unit_test/rag/utils/test_gaussdb_conn.py b/test/unit_test/rag/utils/test_gaussdb_conn.py
index 03f885b2c1..dc39a29bf5 100644
--- a/test/unit_test/rag/utils/test_gaussdb_conn.py
+++ b/test/unit_test/rag/utils/test_gaussdb_conn.py
@@ -2822,7 +2822,7 @@ def test_tc_ret_805_parse_fusion_vector_weight_returns_none_for_missing_or_bad_w
def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaults(monkeypatch):
monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", lambda query: str(query).split())
conn = make_conn(RecordingCursor())
- text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello"})
+ text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello", "minimum_should_match": 0.3})
dense = MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", 10, {"similarity": 0.0})
fusion = FusionExpr("weighted_sum", 6, {"weights": "0.7,0.3"})
@@ -2832,7 +2832,9 @@ def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaul
hybrid_explicit = conn._parse_match_expressions([text, dense, fusion], rank_feature={"pagerank_fea": 7})
assert text_only["vector_weight"] == 0.0
+ assert text_only["minimum_should_match"] == 0.3
assert vector_only["vector_weight"] == 1.0
+ assert vector_only["minimum_should_match"] is None
assert hybrid_default["vector_weight"] == 0.5
assert {text_only["pagerank_weight"], vector_only["pagerank_weight"], hybrid_default["pagerank_weight"]} == {10.0}
assert hybrid_explicit["vector_weight"] == 0.3
diff --git a/test/unit_test/rag/utils/test_gaussdb_search_sql.py b/test/unit_test/rag/utils/test_gaussdb_search_sql.py
index 8c27422d1a..b4308cb6e6 100644
--- a/test/unit_test/rag/utils/test_gaussdb_search_sql.py
+++ b/test/unit_test/rag/utils/test_gaussdb_search_sql.py
@@ -176,6 +176,28 @@ def test_tc_ret_109_chinese_ngram_query_preserves_english_semantics_and_highligh
assert conn.get_highlight(result, ["深圳数据", "深圳"], "content_with_weight") == {"overlap": "<em>深圳数据</em>库审计"}
+def test_tc_ret_111_minimum_should_match_uses_partial_unique_terms_and_term_scores():
+ builder = GaussDBSearchBuilder(schema="public")
+ keywords = ["你好", "一下", "王楠", "简历", "大概", "介绍", "一下"]
+
+ match_sql, match_params = builder._build_text_match_expr(keywords, minimum_should_match=0.3)
+ score_sql, score_params = builder.build_text_score_expr(keywords, minimum_should_match=0.3)
+
+ unique_terms = ["你好", "一下", "王楠", "简历", "大概", "介绍"]
+ assert " OR " in match_sql
+ assert match_params == unique_terms
+ assert " / 6.0)" in score_sql
+ assert score_params == [term for term in unique_terms for _field in builder.FTS_WEIGHTS]
+ assert builder._minimum_should_match_count(0.3, 6) == 1
+ assert builder._minimum_should_match_count("50%", 6) == 3
+ assert builder._minimum_should_match_count(2, 6) == 2
+
+ two_match_sql, two_match_params = builder._build_text_match_expr(keywords, minimum_should_match=2)
+ assert "CASE WHEN" in two_match_sql
+ assert ">= 2" in two_match_sql
+ assert two_match_params == unique_terms * 2
+
+
def test_tc_ret_203_vector_search_filters_invalid_placeholder_vectors():
builder = GaussDBSearchBuilder(schema="public")
sql, params = builder.build_search_sql(