OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: surface clear error when OpenSearch disk usage exceeds 90% threshold
Tejas Kumar2fd1773dac29
diff --git a/src/api/search.py b/src/api/search.pyindex 3259c1ccf..2bff17eb5 100644--- a/src/api/search.py+++ b/src/api/search.py@@ -4,6 +4,7 @@ from fastapi import Dependsfrom pydantic import BaseModel, Fieldfrom fastapi.responses import JSONResponsefrom utils.logging_config import get_logger+from utils.opensearch_utils import OpenSearchDiskSpaceErrorfrom dependencies import get_search_service, get_session_manager, get_current_userfrom session_manager import User@@ -50,6 +51,8 @@ async def search(score_threshold=body.scoreThreshold,)return JSONResponse(result, status_code=200)+ except OpenSearchDiskSpaceError as e:+ return JSONResponse({"error": str(e)}, status_code=507)except Exception as e:error_msg = str(e)if (diff --git a/src/api/v1/search.py b/src/api/v1/search.pyindex ef71ebfe1..4c2bc9c78 100644--- a/src/api/v1/search.py+++ b/src/api/v1/search.py@@ -10,6 +10,7 @@ from fastapi import Dependsfrom pydantic import BaseModelfrom fastapi.responses import JSONResponsefrom utils.logging_config import get_logger+from utils.opensearch_utils import OpenSearchDiskSpaceErrorfrom dependencies import get_search_service, get_api_key_user_asyncfrom session_manager import User@@ -65,6 +66,9 @@ async def search_endpoint(return JSONResponse({"results": results})+ except OpenSearchDiskSpaceError as e:+ logger.error("Search blocked by disk space constraint", error=str(e), user_id=user.user_id)+ return JSONResponse({"error": str(e)}, status_code=507)except Exception as e:error_msg = str(e)logger.error("Search failed", error=error_msg, user_id=user.user_id)diff --git a/src/services/search_service.py b/src/services/search_service.pyindex ef88bb9d4..fe0d8d309 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -392,6 +392,7 @@ class SearchService:)from opensearchpy.exceptions import RequestError+ from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error, DISK_SPACE_ERROR_MESSAGEsearch_params = {"terminate_after": 0}@@ -403,6 +404,12 @@ class SearchService:)except RequestError as e:error_message = str(e)+ if is_disk_space_error(e):+ logger.error(+ "OpenSearch query blocked by disk space constraint",+ error=error_message,+ )+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from eif (fallback_search_body is not Noneand "unknown field [num_candidates]" in error_message.lower()@@ -417,6 +424,12 @@ class SearchService:params=search_params,)except RequestError as retry_error:+ if is_disk_space_error(retry_error):+ logger.error(+ "OpenSearch retry blocked by disk space constraint",+ error=str(retry_error),+ )+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from retry_errorlogger.error("OpenSearch retry without num_candidates failed",error=str(retry_error),@@ -428,7 +441,15 @@ class SearchService:"OpenSearch query failed", error=error_message, search_body=search_body)raise+ except OpenSearchDiskSpaceError:+ raiseexcept Exception as e:+ if is_disk_space_error(e):+ logger.error(+ "OpenSearch query blocked by disk space constraint",+ error=str(e),+ )+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from elogger.error("OpenSearch query failed", error=str(e), search_body=search_body)diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.pyindex b504933b5..cf2b343ad 100644--- a/src/utils/opensearch_utils.py+++ b/src/utils/opensearch_utils.py@@ -5,9 +5,50 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)+DISK_SPACE_ERROR_MESSAGE = (+ "OpenSearch disk usage has exceeded the critical threshold (>90%). "+ "Search and indexing operations are blocked. "+ "Please free up disk space to restore OpenRAG functionality."+)++# Error strings emitted by OpenSearch when disk watermark thresholds are breached+_DISK_SPACE_INDICATORS = [+ "disk watermark",+ "flood_stage",+ "flood stage",+ "disk usage exceeded",+ "index read-only",+ "no space left on device",+ "cluster_block_exception",+ "forbidden/12",+ "too_many_requests/12",+]++class OpenSearchNotReadyError(Exception):"""Raised when OpenSearch fails to become ready within the retry limit."""++class OpenSearchDiskSpaceError(Exception):+ """Raised when OpenSearch operations fail due to insufficient disk space (>90% used)."""+++def is_disk_space_error(error: Exception) -> bool:+ """Check whether an exception is caused by OpenSearch disk space constraints.++ OpenSearch blocks write and search operations when disk usage crosses+ the high-watermark (default 90%) or flood-stage watermark (default 95%).+ This function detects those error signatures.++ Args:+ error: The exception to inspect.++ Returns:+ True if the error is disk-space related, False otherwise.+ """+ error_str = str(error).lower()+ return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)+async def wait_for_opensearch(opensearch_client: AsyncOpenSearch,max_retries: int = 15,diff --git a/tests/unit/test_opensearch_disk_space.py b/tests/unit/test_opensearch_disk_space.pynew file mode 100644index 000000000..05e4c380e--- /dev/null+++ b/tests/unit/test_opensearch_disk_space.py@@ -0,0 +1,101 @@+"""+Unit tests for OpenSearch disk space error detection.++When OpenSearch disk usage exceeds the high-watermark (90%) or flood-stage+watermark (95%), it blocks operations and returns errors with specific+signatures. These tests verify that those signatures are correctly detected+and surfaced as OpenSearchDiskSpaceError.+"""+import pytest+from unittest.mock import AsyncMock, MagicMock, patch++from utils.opensearch_utils import (+ OpenSearchDiskSpaceError,+ is_disk_space_error,+ DISK_SPACE_ERROR_MESSAGE,+)+++# ---------------------------------------------------------------------------+# is_disk_space_error+# ---------------------------------------------------------------------------++class TestIsDiskSpaceError:+ def test_disk_watermark_message(self):+ assert is_disk_space_error(Exception("high disk watermark exceeded on node"))++ def test_flood_stage_underscore(self):+ assert is_disk_space_error(Exception("flood_stage disk watermark exceeded"))++ def test_flood_stage_space(self):+ assert is_disk_space_error(Exception("flood stage disk watermark exceeded"))++ def test_disk_usage_exceeded(self):+ assert is_disk_space_error(Exception("disk usage exceeded flood-stage watermark"))++ def test_index_read_only(self):+ assert is_disk_space_error(Exception("index read-only / allow delete (api)"))++ def test_no_space_left_on_device(self):+ assert is_disk_space_error(Exception("no space left on device"))++ def test_cluster_block_exception(self):+ assert is_disk_space_error(Exception("cluster_block_exception: blocked by index"))++ def test_forbidden_12(self):+ assert is_disk_space_error(Exception("[FORBIDDEN/12/index read-only]"))++ def test_too_many_requests_12(self):+ assert is_disk_space_error(Exception("[TOO_MANY_REQUESTS/12/disk usage exceeded]"))++ def test_case_insensitive(self):+ assert is_disk_space_error(Exception("DISK WATERMARK EXCEEDED"))++ def test_unrelated_error_returns_false(self):+ assert not is_disk_space_error(Exception("Connection refused"))++ def test_auth_error_returns_false(self):+ assert not is_disk_space_error(Exception("AuthenticationException: access denied"))++ def test_timeout_error_returns_false(self):+ assert not is_disk_space_error(Exception("ConnectionTimeout caused by timeout"))++ def test_empty_error_returns_false(self):+ assert not is_disk_space_error(Exception(""))+++# ---------------------------------------------------------------------------+# DISK_SPACE_ERROR_MESSAGE content+# ---------------------------------------------------------------------------++class TestDiskSpaceErrorMessage:+ def test_message_mentions_threshold(self):+ assert "90%" in DISK_SPACE_ERROR_MESSAGE++ def test_message_mentions_disk_space(self):+ assert "disk" in DISK_SPACE_ERROR_MESSAGE.lower()++ def test_message_mentions_resolution(self):+ assert "free up" in DISK_SPACE_ERROR_MESSAGE.lower()+++# ---------------------------------------------------------------------------+# OpenSearchDiskSpaceError is an Exception subclass+# ---------------------------------------------------------------------------++class TestOpenSearchDiskSpaceError:+ def test_is_exception(self):+ err = OpenSearchDiskSpaceError("test")+ assert isinstance(err, Exception)++ def test_carries_message(self):+ err = OpenSearchDiskSpaceError("disk full")+ assert str(err) == "disk full"++ def test_can_chain_cause(self):+ cause = ValueError("original")+ err = OpenSearchDiskSpaceError("disk full")+ try:+ raise err from cause+ except OpenSearchDiskSpaceError as caught:+ assert caught.__cause__ is cause