Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

reset search service and opensearch queries

Edwin Josefb7083075c7a
diff --git a/src/services/search_service.py b/src/services/search_service.py
index cefdc6cc1..d6bb06a63 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -75,10 +75,10 @@ class SearchService:
if filters:
# Map frontend filter names to backend field names
field_mapping = {
- "data_sources": "filename.keyword",
- "document_types": "mimetype.keyword",
+ "data_sources": "filename",
+ "document_types": "mimetype",
"owners": "owner_name.keyword",
- "connector_types": "connector_type.keyword",
+ "connector_types": "connector_type",
}
for filter_key, values in filters.items():
@@ -234,10 +234,10 @@ class SearchService:
if filters:
# Map frontend filter names to backend field names
field_mapping = {
- "data_sources": "filename.keyword",
- "document_types": "mimetype.keyword",
+ "data_sources": "filename",
+ "document_types": "mimetype",
"owners": "owner_name.keyword",
- "connector_types": "connector_type.keyword",
+ "connector_types": "connector_type",
}
for filter_key, values in filters.items():
@@ -330,10 +330,10 @@ class SearchService:
search_body = {
"query": query_block,
"aggs": {
- "data_sources": {"terms": {"field": "filename.keyword", "size": 20}},
- "document_types": {"terms": {"field": "mimetype.keyword", "size": 10}},
+ "data_sources": {"terms": {"field": "filename", "size": 20}},
+ "document_types": {"terms": {"field": "mimetype", "size": 10}},
"owners": {"terms": {"field": "owner_name.keyword", "size": 10}},
- "connector_types": {"terms": {"field": "connector_type.keyword", "size": 10}},
+ "connector_types": {"terms": {"field": "connector_type", "size": 10}},
"embedding_models": {"terms": {"field": "embedding_model", "size": 10}},
},
"_source": [
@@ -395,119 +395,17 @@ class SearchService:
search_params = {"terminate_after": 0}
- def is_knn_field_error(message: str) -> bool:
- lowered = message.lower()
- # OpenSearch variants seen in practice:
- # - "Field 'x' is not knn_vector type."
- # - "failed to create query: Field 'x' ..."
- # - generic knn query construction errors
- return (
- "not knn_vector type" in lowered
- or "failed to create query: field" in lowered
- or "knn_vector" in lowered
- or ("failed to create query" in lowered and "knn" in lowered)
- )
-
- def is_fielddata_error(message: str) -> bool:
- lowered = message.lower()
- return "text fields are not optimised" in lowered or "fielddata" in lowered
-
- async def run_keyword_only_fallback(reason: str):
- logger.warning(
- "Falling back to keyword-only search",
- reason=reason,
- )
- if not (query or "").strip():
- # Keep empty-query behavior deterministic in fallback paths.
- return {"hits": {"hits": []}, "aggregations": {}}
- keyword_query = {
- "bool": {
- "must": [
- {
- "multi_match": {
- "query": query,
- "fields": ["text^2", "filename^1.5"],
- "type": "best_fields",
- "fuzziness": "AUTO",
- }
- }
- ],
- "filter": filter_clauses,
- }
- }
- keyword_search_body = copy.deepcopy(search_body)
- keyword_search_body["query"] = keyword_query
- keyword_search_body.pop("min_score", None)
- try:
- return await opensearch_client.search(
- index=get_index_name(),
- body=keyword_search_body,
- params=search_params,
- )
- except Exception as keyword_error:
- keyword_error_message = str(keyword_error)
- if is_fielddata_error(keyword_error_message):
- # Some clusters can reject terms aggregations for text-mapped fields;
- # retry keyword-only query without aggregations to keep retrieval alive.
- keyword_without_aggs = copy.deepcopy(keyword_search_body)
- keyword_without_aggs.pop("aggs", None)
- return await opensearch_client.search(
- index=get_index_name(),
- body=keyword_without_aggs,
- params=search_params,
- )
- raise
-
- async def run_legacy_vector_fallback(reason: str, *, include_num_candidates: bool):
- logger.warning(
- "Attempting legacy vector fallback",
- reason=reason,
- include_num_candidates=include_num_candidates,
- )
- fallback_vector = next(iter(query_embeddings.values()), None)
- if fallback_vector is None:
- return await run_keyword_only_fallback(
- f"{reason}; no query embeddings available"
- )
-
- fallback_field = "chunk_embedding"
- legacy_search_body = copy.deepcopy(search_body)
- legacy_search_body.pop("min_score", None)
- legacy_search_body["query"]["bool"]["filter"] = [
- *filter_clauses,
- {"exists": {"field": fallback_field}},
- ]
-
- knn_legacy = {
- "knn": {
- fallback_field: {
- "vector": fallback_vector,
- "k": 50,
- }
- }
- }
- if include_num_candidates:
- knn_legacy["knn"][fallback_field]["num_candidates"] = 1000
- legacy_search_body["query"]["bool"]["should"][0]["dis_max"]["queries"] = [knn_legacy]
-
- return await opensearch_client.search(
- index=get_index_name(),
- body=legacy_search_body,
- params=search_params,
- )
-
try:
index_name = get_index_name()
logger.info(f"Sending query to index '{index_name}'..")
results = await opensearch_client.search(
index=index_name, body=search_body, params=search_params
)
- except Exception as e:
+ except RequestError as e:
error_message = str(e)
- lowered_error = error_message.lower()
if (
fallback_search_body is not None
- and "unknown field [num_candidates]" in lowered_error
+ and "unknown field [num_candidates]" in error_message.lower()
):
logger.warning(
"OpenSearch cluster does not support num_candidates; retrying without it"
@@ -518,51 +416,24 @@ class SearchService:
body=fallback_search_body,
params=search_params,
)
- except Exception as retry_error:
- retry_error_message = str(retry_error)
- if not is_wildcard_match_all:
- # Be resilient here: when the retry still fails, prefer degraded retrieval
- # over surfacing a hard 500 to callers.
- if is_knn_field_error(retry_error_message):
- try:
- results = await run_legacy_vector_fallback(
- f"knn field error after num_candidates fallback: {retry_error_message}",
- include_num_candidates=False,
- )
- except Exception as legacy_retry_error:
- results = await run_keyword_only_fallback(
- "legacy vector fallback failed after num_candidates fallback: "
- f"{legacy_retry_error}"
- )
- else:
- results = await run_keyword_only_fallback(
- f"search failed after num_candidates fallback: {retry_error_message}"
- )
- else:
- logger.error(
- "OpenSearch retry without num_candidates failed",
- error=retry_error_message,
- search_body=fallback_search_body,
- )
- raise
- elif (
- not is_wildcard_match_all
- and is_knn_field_error(error_message)
- ):
- try:
- results = await run_legacy_vector_fallback(
- f"knn field error: {error_message}",
- include_num_candidates=False,
- )
- except Exception as legacy_error:
- results = await run_keyword_only_fallback(
- f"legacy vector fallback failed: {legacy_error}"
+ except RequestError as retry_error:
+ logger.error(
+ "OpenSearch retry without num_candidates failed",
+ error=str(retry_error),
+ search_body=fallback_search_body,
)
+ raise
else:
logger.error(
"OpenSearch query failed", error=error_message, search_body=search_body
)
raise
+ except Exception as e:
+ logger.error(
+ "OpenSearch query failed", error=str(e), search_body=search_body
+ )
+ # Re-raise the exception so the API returns the error to frontend
+ raise
# Transform results (keep for backward compatibility)
chunks = []
@@ -635,4 +506,4 @@ class SearchService:
set_search_limit(limit)
set_score_threshold(score_threshold)
- return await self.search_tool(query, embedding_model=embedding_model)
+ return await self.search_tool(query, embedding_model=embedding_model)
\ No newline at end of file
diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.py
index a8c411c76..eadafd376 100644
--- a/src/utils/opensearch_queries.py
+++ b/src/utils/opensearch_queries.py
@@ -14,18 +14,9 @@ def build_filename_query(filename: str) -> dict:
Returns:
A dict containing the OpenSearch query body
"""
- # Be resilient to mapping differences across clusters:
- # - keyword subfield for exact match (preferred)
- # - raw text field term for older mappings
- # - match_phrase fallback when filename is text-only
return {
- "bool": {
- "should": [
- {"term": {"filename.keyword": filename}},
- {"term": {"filename": filename}},
- {"match_phrase": {"filename": filename}},
- ],
- "minimum_should_match": 1,
+ "term": {
+ "filename": filename
}
}
@@ -61,4 +52,4 @@ def build_filename_delete_body(filename: str) -> dict:
"""
return {
"query": build_filename_query(filename)
- }
+ }
\ No newline at end of file