RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(opensearch): stop zeroing vector scores by mapping similarity to knn boost (#18662)
Andras BARTHA7c455fba1835
diff --git a/rag/utils/opensearch_conn.py b/rag/utils/opensearch_conn.pyindex 30d195b3e5..791e9faea9 100644--- a/rag/utils/opensearch_conn.py+++ b/rag/utils/opensearch_conn.py@@ -395,9 +395,9 @@ class OSConnection(DocStoreConnection):# Besides, Opensearch's DSL for KNN_search query syntax differs from that in Elasticsearch, I also made some adaptions for itelif isinstance(m, MatchDenseExpr):assert bqry is not None- similarity = 0.0- if "similarity" in m.extra_options:- similarity = m.extra_options["similarity"]+ explicit_boost = None+ if isinstance(m.extra_options, dict) and "boost" in m.extra_options:+ explicit_boost = m.extra_options["boost"]use_knn = Truevector_column_name = m.vector_column_nameknn_query[vector_column_name] = {}@@ -410,7 +410,8 @@ class OSConnection(DocStoreConnection):bool_inner = bqry.to_dict().get("bool", {})if bool_inner.get("filter"):knn_query[vector_column_name]["filter"] = {"bool": {"filter": bool_inner["filter"]}}- knn_query[vector_column_name]["boost"] = similarity+ if explicit_boost is not None:+ knn_query[vector_column_name]["boost"] = explicit_boostif bqry and rank_feature:for fld, sc in rank_feature.items():diff --git a/test/unit_test/rag/utils/test_opensearch_hybrid_search.py b/test/unit_test/rag/utils/test_opensearch_hybrid_search.pyindex fb1f2159c9..49fa0a286b 100644--- a/test/unit_test/rag/utils/test_opensearch_hybrid_search.py+++ b/test/unit_test/rag/utils/test_opensearch_hybrid_search.py@@ -122,14 +122,19 @@ def _text_expr():return MatchTextExpr(fields=["content_ltks"], matching_text="what is kubernetes", topn=10, extra_options={})-def _dense_expr():+_DEFAULT_DENSE_OPTIONS = object()+++def _dense_expr(extra_options=_DEFAULT_DENSE_OPTIONS):+ if extra_options is _DEFAULT_DENSE_OPTIONS:+ extra_options = {"similarity": 0.0}return MatchDenseExpr(vector_column_name="q_1024_vec",embedding_data=[0.1] * 8,embedding_data_type="float",distance_type="cosine",topn=5,- extra_options={"similarity": 0.0},+ extra_options=extra_options,)@@ -214,6 +219,51 @@ class TestHybridSearchDSL:assert "knn" in body["query"], "must fall back to a pure knn query"assert params is None, "must not reference a search_pipeline when disabled"+ def test_knn_does_not_implicitly_set_boost_from_similarity(self):+ """similarity=0.0 is a threshold input and must not zero-out knn scores+ by being copied into boost."""+ conn = _make_os_connection()+ body, _ = _call_search(conn, [_dense_expr({"similarity": 0.0})])++ knn_clause = body["query"]["knn"]+ vec_params = next(iter(knn_clause.values()))+ assert "boost" not in vec_params, "knn boost must be omitted unless explicitly configured"++ def test_knn_honors_explicit_boost(self):+ conn = _make_os_connection()+ body, _ = _call_search(conn, [_dense_expr({"similarity": 0.0, "boost": 0.25})])++ knn_clause = body["query"]["knn"]+ vec_params = next(iter(knn_clause.values()))+ assert vec_params.get("boost") == 0.25++ def test_knn_accepts_none_extra_options(self):+ conn = _make_os_connection()+ body, _ = _call_search(conn, [_dense_expr(extra_options=None)])++ knn_clause = body["query"]["knn"]+ vec_params = next(iter(knn_clause.values()))+ assert "boost" not in vec_params, "knn boost must be omitted when extra_options is None"+++class TestOpenSearchVectorScoreExtraction:+ def test_get_scores_keeps_nonzero_knn_scores(self):+ """Vector similarity in retrieval comes from get_scores(_score) in the+ second knn pass; nonzero engine scores must survive unchanged."""+ conn = _make_os_connection()+ res = {+ "hits": {+ "hits": [+ {"_id": "chunk-1", "_score": 0.8123},+ {"_id": "chunk-2", "_score": 0.1034},+ ]+ }+ }++ scores = conn.get_scores(res)+ assert scores["chunk-1"] == pytest.approx(0.8123)+ assert scores["chunk-2"] == pytest.approx(0.1034)+if __name__ == "__main__":raise SystemExit(pytest.main([__file__, "-v"]))