OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: apply the search score threshold before DLS adds its own score (#2520)
Rico Furtado572d04903410
diff --git a/src/services/search_service.py b/src/services/search_service.pyindex 762f4c671..b124d6f01 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -616,6 +616,15 @@ class SearchService:}}+ # Apply the threshold to the hybrid score itself. A top-level+ # min_score also counts the constant-score clause OpenSearch DLS+ # adds to every user-scoped search, which lifts every visible hit+ # over the threshold and turns the filter off.+ if score_threshold > 0:+ query_block = {+ "function_score": {"query": query_block, "min_score": score_threshold}+ }+search_body: dict[str, Any] = {"query": query_block,"aggs": _build_file_facet_aggregations(),@@ -662,19 +671,15 @@ class SearchService:"require_field_match": False,}- # Add score threshold only for hybrid (not meaningful for match_all)- if not is_wildcard_match_all and score_threshold > 0:- search_body["min_score"] = score_threshold-# Prepare fallback search body without num_candidates for clusters that don't support it.# Only relevant when we actually dispatched KNN queries.fallback_search_body: dict[str, Any] | None = Noneif not is_wildcard_match_all and query_embeddings:try:fallback_search_body = copy.deepcopy(search_body)- knn_query_blocks = fallback_search_body["query"]["bool"]["should"][0]["dis_max"][- "queries"- ]+ hybrid_query = fallback_search_body["query"]+ hybrid_query = hybrid_query.get("function_score", {}).get("query", hybrid_query)+ knn_query_blocks = hybrid_query["bool"]["should"][0]["dis_max"]["queries"]for query_candidate in knn_query_blocks:knn_section = query_candidate.get("knn")if isinstance(knn_section, dict):diff --git a/tests/unit/test_search_service_score_threshold.py b/tests/unit/test_search_service_score_threshold.pynew file mode 100644index 000000000..f65eea600--- /dev/null+++ b/tests/unit/test_search_service_score_threshold.py@@ -0,0 +1,157 @@+"""+Unit tests for how services/search_service.py applies the score threshold.++OpenSearch DLS adds its role query to every user-scoped search as a+constant-score clause, so a top-level min_score is met by every visible hit.+The threshold must therefore sit inside the query, on the hybrid score.++Verifies:+- a positive threshold wraps the hybrid query in function_score.min_score+- no top-level min_score is sent+- threshold 0 and wildcard queries are not wrapped+- the num_candidates retry still strips num_candidates inside the wrapper+"""++from types import SimpleNamespace++import pytest+from opensearchpy.exceptions import RequestError++from auth_context import set_auth_context, set_score_threshold, set_search_filters, set_search_limit+from services.search_service import SearchService+++class _OpenSearch:+ """Minimal OpenSearch stub that records every non-agg search body."""++ def __init__(self, reject_num_candidates: bool = False):+ """Initialize request recording and optional rejection of the first search."""+ self.bodies: list[dict] = []+ self._reject_num_candidates = reject_num_candidates++ async def search(self, *, index, body, params):+ """Return empty results, recording searches and optionally simulating a retry error."""+ # Agg-only probes (embedding space detection, wildcard facets)+ if body.get("size") == 0:+ return {+ "aggregations": {+ "embedding_spaces": {"buckets": []},+ "legacy_embedding_models": {"buckets": []},+ }+ }+ self.bodies.append(body)+ if self._reject_num_candidates and len(self.bodies) == 1:+ raise RequestError(+ 400,+ "x_content_parse_exception",+ {"error": {"root_cause": [{"reason": "[knn] unknown field [num_candidates]"}]}},+ )+ return {"hits": {"hits": []}, "aggregations": {}}+++def _make_service(opensearch: _OpenSearch, monkeypatch) -> SearchService:+ """Build a search service with the supplied client and deterministic embedding stubs."""+ service = object.__new__(SearchService)+ service.session_manager = SimpleNamespace(+ get_user_opensearch_client=lambda user_id, jwt_token: opensearch+ )+ service.models_service = None++ async def _no_embed(body, **_kwargs):+ """Return a fixed query embedding without contacting an embedding provider."""+ return {"data": [{"embedding": [0.1, 0.2]}]}++ monkeypatch.setattr("services.search_service.gateway_embeddings", _no_embed)+ monkeypatch.setattr("services.search_service.get_index_name", lambda: "documents")+ monkeypatch.setattr(+ "services.search_service.get_embedding_model", lambda: "text-embedding-3-small"+ )+ monkeypatch.setattr(+ "services.search_service.get_openrag_config",+ lambda: SimpleNamespace(+ knowledge=SimpleNamespace(embedding_provider="openai"),+ providers=SimpleNamespace(ollama=SimpleNamespace(endpoint=None)),+ ),+ )+ monkeypatch.setattr(+ "services.search_service.get_declared_default_embedding_model",+ lambda _: "text-embedding-3-small",+ )+ return service+++def _set_context(score_threshold: float) -> None:+ """Set a test user's search context with the requested score threshold."""+ set_auth_context("user-1", None)+ set_search_filters({})+ set_search_limit(100)+ set_score_threshold(score_threshold)+++def _knn_params(query: dict) -> list[dict]:+ """Extract kNN parameters from a hybrid query with or without a score wrapper."""+ hybrid = query.get("function_score", {}).get("query", query)+ knn_queries = hybrid["bool"]["should"][0]["dis_max"]["queries"]+ return [params for knn in knn_queries for params in knn["knn"].values()]+++async def test_positive_threshold_applies_inside_the_query(monkeypatch):+ """Verify a positive threshold filters the hybrid score inside function_score."""+ os_client = _OpenSearch()+ service = _make_service(os_client, monkeypatch)+ _set_context(1.0)++ await service.search_tool("MCP")++ body = os_client.bodies[-1]+ assert "min_score" not in body+ function_score = body["query"]["function_score"]+ assert function_score["min_score"] == 1.0+ assert "functions" not in function_score+ assert function_score["query"]["bool"]["minimum_should_match"] == 1+++async def test_zero_threshold_leaves_the_query_unwrapped(monkeypatch):+ """Verify a zero threshold leaves the hybrid query unwrapped and without min_score."""+ os_client = _OpenSearch()+ service = _make_service(os_client, monkeypatch)+ _set_context(0)++ await service.search_tool("MCP")++ body = os_client.bodies[-1]+ assert "min_score" not in body+ assert "function_score" not in body["query"]+ assert "bool" in body["query"]+++async def test_wildcard_query_ignores_the_threshold(monkeypatch):+ """Verify wildcard searches use match_all even when the score threshold is positive."""+ os_client = _OpenSearch()+ service = _make_service(os_client, monkeypatch)+ _set_context(1.0)++ await service.search_tool("*")++ body = os_client.bodies[-1]+ assert "min_score" not in body+ assert body["query"] == {"match_all": {}}+++async def test_num_candidates_retry_keeps_the_threshold(monkeypatch):+ """Verify a retry removes unsupported num_candidates while preserving the threshold."""+ os_client = _OpenSearch(reject_num_candidates=True)+ service = _make_service(os_client, monkeypatch)+ _set_context(1.0)++ await service.search_tool("MCP")++ first, retry = os_client.bodies+ assert all("num_candidates" in params for params in _knn_params(first["query"]))+ assert all("num_candidates" not in params for params in _knn_params(retry["query"]))+ assert retry["query"]["function_score"]["min_score"] == 1.0+ assert "min_score" not in retry