Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

chore: add node check and manager count check for opensearch health (#1913) (#1959)

Edwin Jose86bb09655593
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index d041cdbe2..84dcb8c73 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -66,6 +66,8 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}
+ # Single-node E2E cluster never reaches the multi-node readiness counts
+ OPENSEARCH_NODE_COUNT_CHECK: "false"
LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}
LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}
LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index 360e1aa1d..a42eee86c 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -164,6 +164,8 @@ jobs:
GOOGLE_OAUTH_CLIENT_SECRET: ""
# Disable startup ingest noise unless a test enables it
DISABLE_STARTUP_INGEST: "true"
+ # Single-node test cluster never reaches the multi-node readiness counts
+ OPENSEARCH_NODE_COUNT_CHECK: "false"
run: |
OPENRAG_VERSION="${CI_IMAGE_TAG}" TEST_SUITE="${{ matrix.suite }}" make test-ci-suite
echo "Keys directory after tests:"
diff --git a/docker-compose.yml b/docker-compose.yml
index 7c63f8a15..412d5f5ed 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -73,6 +73,9 @@ services:
- LANGFLOW_OPENSEARCH_PORT=${LANGFLOW_OPENSEARCH_PORT:-${OPENSEARCH_PORT:-9200}}
- OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME:-admin}
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+ # Single-node compose clusters don't meet the multi-node readiness counts,
+ # so default the node-count check off here (override to re-enable).
+ - OPENSEARCH_NODE_COUNT_CHECK=${OPENSEARCH_NODE_COUNT_CHECK:-false}
- OPENAI_API_KEY=${OPENAI_API_KEY}
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
- WATSONX_API_KEY=${WATSONX_API_KEY}
diff --git a/src/app/lifespan.py b/src/app/lifespan.py
index 42d45e426..846580685 100644
--- a/src/app/lifespan.py
+++ b/src/app/lifespan.py
@@ -17,6 +17,7 @@ from config.settings import (
JWT_CLAIMS_CACHE_MAX_SIZE,
JWT_CLAIMS_CACHE_TTL_SECONDS,
OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,
+ OPENSEARCH_WAIT_MAX_RETRIES,
RBAC_CACHE_BACKEND,
RBAC_PERMISSION_CACHE_TTL_SECONDS,
UVICORN_WORKER_COUNT,
@@ -292,7 +293,7 @@ async def run_startup(app: FastAPI):
)
try:
logger.info("Verifying OpenSearch readiness before bootstrap")
- await wait_for_opensearch(opensearch_client)
+ await wait_for_opensearch(opensearch_client, max_retries=OPENSEARCH_WAIT_MAX_RETRIES)
logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)
await setup_opensearch_security(opensearch_client, admin_username=admin_username)
logger.info("OpenSearch security bootstrap completed", admin_username=admin_username)
diff --git a/src/config/settings.py b/src/config/settings.py
index bfc52280b..239cf27fc 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -39,6 +39,27 @@ LANGFLOW_OPENSEARCH_PORT = get_env_int("LANGFLOW_OPENSEARCH_PORT", OPENSEARCH_PO
OPENSEARCH_USERNAME = os.getenv("OPENSEARCH_USERNAME", "admin")
OPENSEARCH_PASSWORD = os.getenv("OPENSEARCH_PASSWORD")
+# Gate the OpenSearch node-count readiness check ("OS node_count" flag).
+# Enabled by default; set to false on small/single-node clusters.
+OPENSEARCH_NODE_COUNT_CHECK_ENABLED = os.getenv(
+ "OPENSEARCH_NODE_COUNT_CHECK", "true"
+).strip().lower() in ("true", "1", "yes")
+
+# Expected cluster size, used only when the node-count check is enabled.
+OPENSEARCH_EXPECTED_DATA_NODE_COUNT = get_env_int("OPENSEARCH_EXPECTED_DATA_NODE_COUNT", 3)
+# Minimum reachable cluster-manager (master) nodes, gated by the same flag.
+OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT = get_env_int(
+ "OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT", 3
+)
+# Minimum reachable coordinating-only nodes, gated by the same flag.
+OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT = get_env_int(
+ "OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT", 3
+)
+# Max readiness-probe attempts for the lifespan startup bootstrap (exponential
+# backoff between tries). Higher than other callers because a large cluster can
+# take longer to fully form; raise further for very large clusters.
+OPENSEARCH_WAIT_MAX_RETRIES = get_env_int("OPENSEARCH_WAIT_MAX_RETRIES", 100)
+
def get_opensearch_username() -> str:
"""OpenSearch admin/basic-auth username, read per-call so runtime/test
diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.py
index a87bd1630..e794503e2 100644
--- a/src/utils/opensearch_init.py
+++ b/src/utils/opensearch_init.py
@@ -102,7 +102,7 @@ async def _ensure_field_mappings(
)
-async def wait_for_opensearch(opensearch_client=None):
+async def wait_for_opensearch(opensearch_client=None, max_retries: int = 30):
"""Wait for OpenSearch to be ready, delegating to the shared utility."""
from utils.opensearch_utils import (
OpenSearchNotReadyError,
@@ -112,7 +112,9 @@ async def wait_for_opensearch(opensearch_client=None):
)
try:
- await _wait_for_opensearch(opensearch_client or clients.opensearch)
+ logger.info("Waiting for OpenSearch to be ready...", max_retries=max_retries)
+ await _wait_for_opensearch(opensearch_client or clients.opensearch, max_retries=max_retries)
+ logger.info("OpenSearch is ready!")
await TelemetryClient.send_event(
Category.OPENSEARCH_SETUP, MessageId.ORB_OS_CONN_ESTABLISHED
)
diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.py
index d01ac780e..1c5c5e20f 100644
--- a/src/utils/opensearch_utils.py
+++ b/src/utils/opensearch_utils.py
@@ -132,7 +132,7 @@ def opensearch_error_fields(error: Exception) -> dict[str, Any]:
async def wait_for_opensearch(
opensearch_client: AsyncOpenSearch,
- max_retries: int = 15,
+ max_retries: int = 30,
base_delay: float = 2.0,
max_delay: float = 30.0,
) -> None:
@@ -163,12 +163,60 @@ async def wait_for_opensearch(
health = await opensearch_client.cluster.health()
status = health.get("status")
if status in ["green", "yellow"]:
- logger.info(
- "Successfully verified that OpenSearch is ready.",
- attempt=display_attempt,
- status=status,
+ from config.settings import (
+ OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,
+ OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,
+ OPENSEARCH_EXPECTED_DATA_NODE_COUNT,
+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED,
)
- return
+
+ if OPENSEARCH_NODE_COUNT_CHECK_ENABLED:
+ data_node_count = health.get("number_of_data_nodes", 0)
+ # Reachable cluster-manager (master) nodes.
+ cm_info = await opensearch_client.transport.perform_request(
+ "GET", "/_nodes/cluster_manager:true/process,transport"
+ )
+ cluster_manager_count = cm_info.get("_nodes", {}).get("successful", 0)
+ # Reachable coordinating-only nodes.
+ coord_info = await opensearch_client.transport.perform_request(
+ "GET", "/_nodes/coordinating_only:true/process,transport"
+ )
+ coordinating_count = coord_info.get("_nodes", {}).get("successful", 0)
+
+ if (
+ data_node_count < OPENSEARCH_EXPECTED_DATA_NODE_COUNT
+ or cluster_manager_count < OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT
+ or coordinating_count < OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT
+ ):
+ logger.warning(
+ "OpenSearch healthy but cluster has not reached expected node count.",
+ attempt=display_attempt,
+ status=status,
+ number_of_data_nodes=data_node_count,
+ cluster_manager_nodes=cluster_manager_count,
+ coordinating_nodes=coordinating_count,
+ expected_data_nodes=OPENSEARCH_EXPECTED_DATA_NODE_COUNT,
+ expected_cluster_managers=OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,
+ expected_coordinating=OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,
+ )
+ # Fall through to the retry/backoff below until nodes join.
+ else:
+ logger.info(
+ "Successfully verified that OpenSearch is ready.",
+ attempt=display_attempt,
+ status=status,
+ number_of_data_nodes=data_node_count,
+ cluster_manager_nodes=cluster_manager_count,
+ coordinating_nodes=coordinating_count,
+ )
+ return
+ else:
+ logger.info(
+ "Successfully verified that OpenSearch is ready.",
+ attempt=display_attempt,
+ status=status,
+ )
+ return
else:
logger.warning(
"OpenSearch is up but cluster health is red.",
diff --git a/tests/unit/test_opensearch_init_wait_retries.py b/tests/unit/test_opensearch_init_wait_retries.py
new file mode 100644
index 000000000..8d28d51df
--- /dev/null
+++ b/tests/unit/test_opensearch_init_wait_retries.py
@@ -0,0 +1,46 @@
+"""The opensearch_init.wait_for_opensearch wrapper forwards max_retries to the
+underlying readiness util.
+
+ * No max_retries arg -> wrapper default of 30 (used by startup_orchestrator /
+ init_index).
+ * Explicit max_retries -> forwarded verbatim (lifespan passes the configured 100).
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from utils import opensearch_init # noqa: E402
+
+
+async def test_wrapper_defaults_to_30(monkeypatch):
+ inner = AsyncMock()
+ monkeypatch.setattr(opensearch_init, "TelemetryClient", MagicMock(send_event=AsyncMock()))
+ monkeypatch.setattr("utils.opensearch_utils.wait_for_opensearch", inner)
+
+ fake_client = MagicMock()
+ await opensearch_init.wait_for_opensearch(fake_client)
+
+ inner.assert_awaited_once()
+ assert inner.await_args.kwargs["max_retries"] == 30
+
+
+async def test_wrapper_forwards_explicit_value(monkeypatch):
+ inner = AsyncMock()
+ monkeypatch.setattr(opensearch_init, "TelemetryClient", MagicMock(send_event=AsyncMock()))
+ monkeypatch.setattr("utils.opensearch_utils.wait_for_opensearch", inner)
+
+ fake_client = MagicMock()
+ await opensearch_init.wait_for_opensearch(fake_client, max_retries=100)
+
+ inner.assert_awaited_once()
+ assert inner.await_args.kwargs["max_retries"] == 100
diff --git a/tests/unit/test_opensearch_wait_node_count.py b/tests/unit/test_opensearch_wait_node_count.py
new file mode 100644
index 000000000..c5efbdc0e
--- /dev/null
+++ b/tests/unit/test_opensearch_wait_node_count.py
@@ -0,0 +1,94 @@
+"""The OpenSearch readiness probe (wait_for_opensearch) gates on data-node,
+cluster-manager, and coordinating-node counts behind the OPENSEARCH_NODE_COUNT_CHECK
+flag.
+
+Cases:
+ * Flag on, all counts met (>= expected): returns without raising.
+ * Flag on, data-node count short: raises OpenSearchNotReadyError.
+ * Flag on, cluster-manager count short: raises OpenSearchNotReadyError.
+ * Flag on, coordinating count short: raises OpenSearchNotReadyError.
+ * Flag off: returns regardless of counts.
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from utils.opensearch_utils import ( # noqa: E402
+ OpenSearchNotReadyError,
+ wait_for_opensearch,
+)
+
+
+def _fake_os_client(
+ data_nodes: int = 2, cluster_managers: int = 2, coordinating: int = 2
+) -> MagicMock:
+ client = MagicMock()
+ client.ping = AsyncMock(return_value=True)
+ client.cluster.health = AsyncMock(
+ return_value={"status": "green", "number_of_data_nodes": data_nodes}
+ )
+
+ async def _perform_request(method, path, *args, **kwargs):
+ if "cluster_manager:true" in path:
+ return {"_nodes": {"successful": cluster_managers}}
+ if "coordinating_only:true" in path:
+ return {"_nodes": {"successful": coordinating}}
+ raise AssertionError(f"unexpected request path: {path}")
+
+ client.transport.perform_request = AsyncMock(side_effect=_perform_request)
+ return client
+
+
+def _patch_settings(monkeypatch, *, enabled=True, data=2, managers=2, coordinating=2):
+ monkeypatch.setattr("config.settings.OPENSEARCH_NODE_COUNT_CHECK_ENABLED", enabled)
+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_DATA_NODE_COUNT", data)
+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT", managers)
+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT", coordinating)
+
+
+async def test_returns_when_all_counts_met(monkeypatch):
+ _patch_settings(monkeypatch)
+ client = _fake_os_client(data_nodes=2, cluster_managers=2, coordinating=2)
+ # Should not raise.
+ await wait_for_opensearch(client, max_retries=1, base_delay=0.0, max_delay=0.0)
+
+
+async def test_raises_when_data_nodes_short(monkeypatch):
+ _patch_settings(monkeypatch)
+ client = _fake_os_client(data_nodes=1, cluster_managers=2, coordinating=2)
+ with pytest.raises(OpenSearchNotReadyError):
+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)
+
+
+async def test_raises_when_cluster_manager_count_short(monkeypatch):
+ _patch_settings(monkeypatch)
+ client = _fake_os_client(data_nodes=2, cluster_managers=1, coordinating=2)
+ with pytest.raises(OpenSearchNotReadyError):
+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)
+
+
+async def test_raises_when_coordinating_count_short(monkeypatch):
+ _patch_settings(monkeypatch)
+ client = _fake_os_client(data_nodes=2, cluster_managers=2, coordinating=1)
+ with pytest.raises(OpenSearchNotReadyError):
+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)
+
+
+async def test_returns_when_flag_disabled(monkeypatch):
+ _patch_settings(monkeypatch, enabled=False)
+ client = _fake_os_client(data_nodes=1, cluster_managers=1, coordinating=1)
+ # Flag off -> counts ignored, returns despite short counts.
+ await wait_for_opensearch(client, max_retries=1, base_delay=0.0, max_delay=0.0)