Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: surface clear error when OpenSearch disk usage exceeds 90% threshold

Tejas Kumar2fd1773dac29
diff --git a/src/api/search.py b/src/api/search.py
index 3259c1ccf..2bff17eb5 100644
--- a/src/api/search.py
+++ b/src/api/search.py
@@ -4,6 +4,7 @@ from fastapi import Depends
from pydantic import BaseModel, Field
from fastapi.responses import JSONResponse
from utils.logging_config import get_logger
+from utils.opensearch_utils import OpenSearchDiskSpaceError
from dependencies import get_search_service, get_session_manager, get_current_user
from session_manager import User
@@ -50,6 +51,8 @@ async def search(
score_threshold=body.scoreThreshold,
)
return JSONResponse(result, status_code=200)
+ except OpenSearchDiskSpaceError as e:
+ return JSONResponse({"error": str(e)}, status_code=507)
except Exception as e:
error_msg = str(e)
if (
diff --git a/src/api/v1/search.py b/src/api/v1/search.py
index ef71ebfe1..4c2bc9c78 100644
--- a/src/api/v1/search.py
+++ b/src/api/v1/search.py
@@ -10,6 +10,7 @@ from fastapi import Depends
from pydantic import BaseModel
from fastapi.responses import JSONResponse
from utils.logging_config import get_logger
+from utils.opensearch_utils import OpenSearchDiskSpaceError
from dependencies import get_search_service, get_api_key_user_async
from session_manager import User
@@ -65,6 +66,9 @@ async def search_endpoint(
return JSONResponse({"results": results})
+ except OpenSearchDiskSpaceError as e:
+ logger.error("Search blocked by disk space constraint", error=str(e), user_id=user.user_id)
+ return JSONResponse({"error": str(e)}, status_code=507)
except Exception as e:
error_msg = str(e)
logger.error("Search failed", error=error_msg, user_id=user.user_id)
diff --git a/src/services/search_service.py b/src/services/search_service.py
index ef88bb9d4..fe0d8d309 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -392,6 +392,7 @@ class SearchService:
)
from opensearchpy.exceptions import RequestError
+ from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error, DISK_SPACE_ERROR_MESSAGE
search_params = {"terminate_after": 0}
@@ -403,6 +404,12 @@ class SearchService:
)
except RequestError as e:
error_message = str(e)
+ if is_disk_space_error(e):
+ logger.error(
+ "OpenSearch query blocked by disk space constraint",
+ error=error_message,
+ )
+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
if (
fallback_search_body is not None
and "unknown field [num_candidates]" in error_message.lower()
@@ -417,6 +424,12 @@ class SearchService:
params=search_params,
)
except RequestError as retry_error:
+ if is_disk_space_error(retry_error):
+ logger.error(
+ "OpenSearch retry blocked by disk space constraint",
+ error=str(retry_error),
+ )
+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from retry_error
logger.error(
"OpenSearch retry without num_candidates failed",
error=str(retry_error),
@@ -428,7 +441,15 @@ class SearchService:
"OpenSearch query failed", error=error_message, search_body=search_body
)
raise
+ except OpenSearchDiskSpaceError:
+ raise
except Exception as e:
+ if is_disk_space_error(e):
+ logger.error(
+ "OpenSearch query blocked by disk space constraint",
+ error=str(e),
+ )
+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
logger.error(
"OpenSearch query failed", error=str(e), search_body=search_body
)
diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.py
index b504933b5..cf2b343ad 100644
--- a/src/utils/opensearch_utils.py
+++ b/src/utils/opensearch_utils.py
@@ -5,9 +5,50 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+DISK_SPACE_ERROR_MESSAGE = (
+ "OpenSearch disk usage has exceeded the critical threshold (>90%). "
+ "Search and indexing operations are blocked. "
+ "Please free up disk space to restore OpenRAG functionality."
+)
+
+# Error strings emitted by OpenSearch when disk watermark thresholds are breached
+_DISK_SPACE_INDICATORS = [
+ "disk watermark",
+ "flood_stage",
+ "flood stage",
+ "disk usage exceeded",
+ "index read-only",
+ "no space left on device",
+ "cluster_block_exception",
+ "forbidden/12",
+ "too_many_requests/12",
+]
+
+
class OpenSearchNotReadyError(Exception):
"""Raised when OpenSearch fails to become ready within the retry limit."""
+
+class OpenSearchDiskSpaceError(Exception):
+ """Raised when OpenSearch operations fail due to insufficient disk space (>90% used)."""
+
+
+def is_disk_space_error(error: Exception) -> bool:
+ """Check whether an exception is caused by OpenSearch disk space constraints.
+
+ OpenSearch blocks write and search operations when disk usage crosses
+ the high-watermark (default 90%) or flood-stage watermark (default 95%).
+ This function detects those error signatures.
+
+ Args:
+ error: The exception to inspect.
+
+ Returns:
+ True if the error is disk-space related, False otherwise.
+ """
+ error_str = str(error).lower()
+ return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)
+
async def wait_for_opensearch(
opensearch_client: AsyncOpenSearch,
max_retries: int = 15,
diff --git a/tests/unit/test_opensearch_disk_space.py b/tests/unit/test_opensearch_disk_space.py
new file mode 100644
index 000000000..05e4c380e
--- /dev/null
+++ b/tests/unit/test_opensearch_disk_space.py
@@ -0,0 +1,101 @@
+"""
+Unit tests for OpenSearch disk space error detection.
+
+When OpenSearch disk usage exceeds the high-watermark (90%) or flood-stage
+watermark (95%), it blocks operations and returns errors with specific
+signatures. These tests verify that those signatures are correctly detected
+and surfaced as OpenSearchDiskSpaceError.
+"""
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from utils.opensearch_utils import (
+ OpenSearchDiskSpaceError,
+ is_disk_space_error,
+ DISK_SPACE_ERROR_MESSAGE,
+)
+
+
+# ---------------------------------------------------------------------------
+# is_disk_space_error
+# ---------------------------------------------------------------------------
+
+class TestIsDiskSpaceError:
+ def test_disk_watermark_message(self):
+ assert is_disk_space_error(Exception("high disk watermark exceeded on node"))
+
+ def test_flood_stage_underscore(self):
+ assert is_disk_space_error(Exception("flood_stage disk watermark exceeded"))
+
+ def test_flood_stage_space(self):
+ assert is_disk_space_error(Exception("flood stage disk watermark exceeded"))
+
+ def test_disk_usage_exceeded(self):
+ assert is_disk_space_error(Exception("disk usage exceeded flood-stage watermark"))
+
+ def test_index_read_only(self):
+ assert is_disk_space_error(Exception("index read-only / allow delete (api)"))
+
+ def test_no_space_left_on_device(self):
+ assert is_disk_space_error(Exception("no space left on device"))
+
+ def test_cluster_block_exception(self):
+ assert is_disk_space_error(Exception("cluster_block_exception: blocked by index"))
+
+ def test_forbidden_12(self):
+ assert is_disk_space_error(Exception("[FORBIDDEN/12/index read-only]"))
+
+ def test_too_many_requests_12(self):
+ assert is_disk_space_error(Exception("[TOO_MANY_REQUESTS/12/disk usage exceeded]"))
+
+ def test_case_insensitive(self):
+ assert is_disk_space_error(Exception("DISK WATERMARK EXCEEDED"))
+
+ def test_unrelated_error_returns_false(self):
+ assert not is_disk_space_error(Exception("Connection refused"))
+
+ def test_auth_error_returns_false(self):
+ assert not is_disk_space_error(Exception("AuthenticationException: access denied"))
+
+ def test_timeout_error_returns_false(self):
+ assert not is_disk_space_error(Exception("ConnectionTimeout caused by timeout"))
+
+ def test_empty_error_returns_false(self):
+ assert not is_disk_space_error(Exception(""))
+
+
+# ---------------------------------------------------------------------------
+# DISK_SPACE_ERROR_MESSAGE content
+# ---------------------------------------------------------------------------
+
+class TestDiskSpaceErrorMessage:
+ def test_message_mentions_threshold(self):
+ assert "90%" in DISK_SPACE_ERROR_MESSAGE
+
+ def test_message_mentions_disk_space(self):
+ assert "disk" in DISK_SPACE_ERROR_MESSAGE.lower()
+
+ def test_message_mentions_resolution(self):
+ assert "free up" in DISK_SPACE_ERROR_MESSAGE.lower()
+
+
+# ---------------------------------------------------------------------------
+# OpenSearchDiskSpaceError is an Exception subclass
+# ---------------------------------------------------------------------------
+
+class TestOpenSearchDiskSpaceError:
+ def test_is_exception(self):
+ err = OpenSearchDiskSpaceError("test")
+ assert isinstance(err, Exception)
+
+ def test_carries_message(self):
+ err = OpenSearchDiskSpaceError("disk full")
+ assert str(err) == "disk full"
+
+ def test_can_chain_cause(self):
+ cause = ValueError("original")
+ err = OpenSearchDiskSpaceError("disk full")
+ try:
+ raise err from cause
+ except OpenSearchDiskSpaceError as caught:
+ assert caught.__cause__ is cause