OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
chore: add node check and manager count check for opensearch health (#1913) (#1959)
Edwin Jose86bb09655593
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex d041cdbe2..84dcb8c73 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -66,6 +66,8 @@ jobs:AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}+ # Single-node E2E cluster never reaches the multi-node readiness counts+ OPENSEARCH_NODE_COUNT_CHECK: "false"LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.ymlindex 360e1aa1d..a42eee86c 100644--- a/.github/workflows/test-integration.yml+++ b/.github/workflows/test-integration.yml@@ -164,6 +164,8 @@ jobs:GOOGLE_OAUTH_CLIENT_SECRET: ""# Disable startup ingest noise unless a test enables itDISABLE_STARTUP_INGEST: "true"+ # Single-node test cluster never reaches the multi-node readiness counts+ OPENSEARCH_NODE_COUNT_CHECK: "false"run: |OPENRAG_VERSION="${CI_IMAGE_TAG}" TEST_SUITE="${{ matrix.suite }}" make test-ci-suiteecho "Keys directory after tests:"diff --git a/docker-compose.yml b/docker-compose.ymlindex 7c63f8a15..412d5f5ed 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -73,6 +73,9 @@ services:- LANGFLOW_OPENSEARCH_PORT=${LANGFLOW_OPENSEARCH_PORT:-${OPENSEARCH_PORT:-9200}}- OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME:-admin}- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}+ # Single-node compose clusters don't meet the multi-node readiness counts,+ # so default the node-count check off here (override to re-enable).+ - OPENSEARCH_NODE_COUNT_CHECK=${OPENSEARCH_NODE_COUNT_CHECK:-false}- OPENAI_API_KEY=${OPENAI_API_KEY}- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}- WATSONX_API_KEY=${WATSONX_API_KEY}diff --git a/src/app/lifespan.py b/src/app/lifespan.pyindex 42d45e426..846580685 100644--- a/src/app/lifespan.py+++ b/src/app/lifespan.py@@ -17,6 +17,7 @@ from config.settings import (JWT_CLAIMS_CACHE_MAX_SIZE,JWT_CLAIMS_CACHE_TTL_SECONDS,OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,+ OPENSEARCH_WAIT_MAX_RETRIES,RBAC_CACHE_BACKEND,RBAC_PERMISSION_CACHE_TTL_SECONDS,UVICORN_WORKER_COUNT,@@ -292,7 +293,7 @@ async def run_startup(app: FastAPI):)try:logger.info("Verifying OpenSearch readiness before bootstrap")- await wait_for_opensearch(opensearch_client)+ await wait_for_opensearch(opensearch_client, max_retries=OPENSEARCH_WAIT_MAX_RETRIES)logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)await setup_opensearch_security(opensearch_client, admin_username=admin_username)logger.info("OpenSearch security bootstrap completed", admin_username=admin_username)diff --git a/src/config/settings.py b/src/config/settings.pyindex bfc52280b..239cf27fc 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -39,6 +39,27 @@ LANGFLOW_OPENSEARCH_PORT = get_env_int("LANGFLOW_OPENSEARCH_PORT", OPENSEARCH_POOPENSEARCH_USERNAME = os.getenv("OPENSEARCH_USERNAME", "admin")OPENSEARCH_PASSWORD = os.getenv("OPENSEARCH_PASSWORD")+# Gate the OpenSearch node-count readiness check ("OS node_count" flag).+# Enabled by default; set to false on small/single-node clusters.+OPENSEARCH_NODE_COUNT_CHECK_ENABLED = os.getenv(+ "OPENSEARCH_NODE_COUNT_CHECK", "true"+).strip().lower() in ("true", "1", "yes")++# Expected cluster size, used only when the node-count check is enabled.+OPENSEARCH_EXPECTED_DATA_NODE_COUNT = get_env_int("OPENSEARCH_EXPECTED_DATA_NODE_COUNT", 3)+# Minimum reachable cluster-manager (master) nodes, gated by the same flag.+OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT = get_env_int(+ "OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT", 3+)+# Minimum reachable coordinating-only nodes, gated by the same flag.+OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT = get_env_int(+ "OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT", 3+)+# Max readiness-probe attempts for the lifespan startup bootstrap (exponential+# backoff between tries). Higher than other callers because a large cluster can+# take longer to fully form; raise further for very large clusters.+OPENSEARCH_WAIT_MAX_RETRIES = get_env_int("OPENSEARCH_WAIT_MAX_RETRIES", 100)+def get_opensearch_username() -> str:"""OpenSearch admin/basic-auth username, read per-call so runtime/testdiff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.pyindex a87bd1630..e794503e2 100644--- a/src/utils/opensearch_init.py+++ b/src/utils/opensearch_init.py@@ -102,7 +102,7 @@ async def _ensure_field_mappings()-async def wait_for_opensearch(opensearch_client=None):+async def wait_for_opensearch(opensearch_client=None, max_retries: int = 30):"""Wait for OpenSearch to be ready, delegating to the shared utility."""from utils.opensearch_utils import (OpenSearchNotReadyError,@@ -112,7 +112,9 @@ async def wait_for_opensearch(opensearch_client=None):)try:- await _wait_for_opensearch(opensearch_client or clients.opensearch)+ logger.info("Waiting for OpenSearch to be ready...", max_retries=max_retries)+ await _wait_for_opensearch(opensearch_client or clients.opensearch, max_retries=max_retries)+ logger.info("OpenSearch is ready!")await TelemetryClient.send_event(Category.OPENSEARCH_SETUP, MessageId.ORB_OS_CONN_ESTABLISHED)diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.pyindex d01ac780e..1c5c5e20f 100644--- a/src/utils/opensearch_utils.py+++ b/src/utils/opensearch_utils.py@@ -132,7 +132,7 @@ def opensearch_error_fields(error: Exception) -> dict[str, Any]:async def wait_for_opensearch(opensearch_client: AsyncOpenSearch,- max_retries: int = 15,+ max_retries: int = 30,base_delay: float = 2.0,max_delay: float = 30.0,) -> None:@@ -163,12 +163,60 @@ async def wait_for_opensearch(health = await opensearch_client.cluster.health()status = health.get("status")if status in ["green", "yellow"]:- logger.info(- "Successfully verified that OpenSearch is ready.",- attempt=display_attempt,- status=status,+ from config.settings import (+ OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,+ OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,+ OPENSEARCH_EXPECTED_DATA_NODE_COUNT,+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED,)- return++ if OPENSEARCH_NODE_COUNT_CHECK_ENABLED:+ data_node_count = health.get("number_of_data_nodes", 0)+ # Reachable cluster-manager (master) nodes.+ cm_info = await opensearch_client.transport.perform_request(+ "GET", "/_nodes/cluster_manager:true/process,transport"+ )+ cluster_manager_count = cm_info.get("_nodes", {}).get("successful", 0)+ # Reachable coordinating-only nodes.+ coord_info = await opensearch_client.transport.perform_request(+ "GET", "/_nodes/coordinating_only:true/process,transport"+ )+ coordinating_count = coord_info.get("_nodes", {}).get("successful", 0)++ if (+ data_node_count < OPENSEARCH_EXPECTED_DATA_NODE_COUNT+ or cluster_manager_count < OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT+ or coordinating_count < OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT+ ):+ logger.warning(+ "OpenSearch healthy but cluster has not reached expected node count.",+ attempt=display_attempt,+ status=status,+ number_of_data_nodes=data_node_count,+ cluster_manager_nodes=cluster_manager_count,+ coordinating_nodes=coordinating_count,+ expected_data_nodes=OPENSEARCH_EXPECTED_DATA_NODE_COUNT,+ expected_cluster_managers=OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,+ expected_coordinating=OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,+ )+ # Fall through to the retry/backoff below until nodes join.+ else:+ logger.info(+ "Successfully verified that OpenSearch is ready.",+ attempt=display_attempt,+ status=status,+ number_of_data_nodes=data_node_count,+ cluster_manager_nodes=cluster_manager_count,+ coordinating_nodes=coordinating_count,+ )+ return+ else:+ logger.info(+ "Successfully verified that OpenSearch is ready.",+ attempt=display_attempt,+ status=status,+ )+ returnelse:logger.warning("OpenSearch is up but cluster health is red.",diff --git a/tests/unit/test_opensearch_init_wait_retries.py b/tests/unit/test_opensearch_init_wait_retries.pynew file mode 100644index 000000000..8d28d51df--- /dev/null+++ b/tests/unit/test_opensearch_init_wait_retries.py@@ -0,0 +1,46 @@+"""The opensearch_init.wait_for_opensearch wrapper forwards max_retries to the+underlying readiness util.++ * No max_retries arg -> wrapper default of 30 (used by startup_orchestrator /+ init_index).+ * Explicit max_retries -> forwarded verbatim (lifespan passes the configured 100).+"""++import sys+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from utils import opensearch_init # noqa: E402+++async def test_wrapper_defaults_to_30(monkeypatch):+ inner = AsyncMock()+ monkeypatch.setattr(opensearch_init, "TelemetryClient", MagicMock(send_event=AsyncMock()))+ monkeypatch.setattr("utils.opensearch_utils.wait_for_opensearch", inner)++ fake_client = MagicMock()+ await opensearch_init.wait_for_opensearch(fake_client)++ inner.assert_awaited_once()+ assert inner.await_args.kwargs["max_retries"] == 30+++async def test_wrapper_forwards_explicit_value(monkeypatch):+ inner = AsyncMock()+ monkeypatch.setattr(opensearch_init, "TelemetryClient", MagicMock(send_event=AsyncMock()))+ monkeypatch.setattr("utils.opensearch_utils.wait_for_opensearch", inner)++ fake_client = MagicMock()+ await opensearch_init.wait_for_opensearch(fake_client, max_retries=100)++ inner.assert_awaited_once()+ assert inner.await_args.kwargs["max_retries"] == 100diff --git a/tests/unit/test_opensearch_wait_node_count.py b/tests/unit/test_opensearch_wait_node_count.pynew file mode 100644index 000000000..c5efbdc0e--- /dev/null+++ b/tests/unit/test_opensearch_wait_node_count.py@@ -0,0 +1,94 @@+"""The OpenSearch readiness probe (wait_for_opensearch) gates on data-node,+cluster-manager, and coordinating-node counts behind the OPENSEARCH_NODE_COUNT_CHECK+flag.++Cases:+ * Flag on, all counts met (>= expected): returns without raising.+ * Flag on, data-node count short: raises OpenSearchNotReadyError.+ * Flag on, cluster-manager count short: raises OpenSearchNotReadyError.+ * Flag on, coordinating count short: raises OpenSearchNotReadyError.+ * Flag off: returns regardless of counts.+"""++import sys+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from utils.opensearch_utils import ( # noqa: E402+ OpenSearchNotReadyError,+ wait_for_opensearch,+)+++def _fake_os_client(+ data_nodes: int = 2, cluster_managers: int = 2, coordinating: int = 2+) -> MagicMock:+ client = MagicMock()+ client.ping = AsyncMock(return_value=True)+ client.cluster.health = AsyncMock(+ return_value={"status": "green", "number_of_data_nodes": data_nodes}+ )++ async def _perform_request(method, path, *args, **kwargs):+ if "cluster_manager:true" in path:+ return {"_nodes": {"successful": cluster_managers}}+ if "coordinating_only:true" in path:+ return {"_nodes": {"successful": coordinating}}+ raise AssertionError(f"unexpected request path: {path}")++ client.transport.perform_request = AsyncMock(side_effect=_perform_request)+ return client+++def _patch_settings(monkeypatch, *, enabled=True, data=2, managers=2, coordinating=2):+ monkeypatch.setattr("config.settings.OPENSEARCH_NODE_COUNT_CHECK_ENABLED", enabled)+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_DATA_NODE_COUNT", data)+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT", managers)+ monkeypatch.setattr("config.settings.OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT", coordinating)+++async def test_returns_when_all_counts_met(monkeypatch):+ _patch_settings(monkeypatch)+ client = _fake_os_client(data_nodes=2, cluster_managers=2, coordinating=2)+ # Should not raise.+ await wait_for_opensearch(client, max_retries=1, base_delay=0.0, max_delay=0.0)+++async def test_raises_when_data_nodes_short(monkeypatch):+ _patch_settings(monkeypatch)+ client = _fake_os_client(data_nodes=1, cluster_managers=2, coordinating=2)+ with pytest.raises(OpenSearchNotReadyError):+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)+++async def test_raises_when_cluster_manager_count_short(monkeypatch):+ _patch_settings(monkeypatch)+ client = _fake_os_client(data_nodes=2, cluster_managers=1, coordinating=2)+ with pytest.raises(OpenSearchNotReadyError):+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)+++async def test_raises_when_coordinating_count_short(monkeypatch):+ _patch_settings(monkeypatch)+ client = _fake_os_client(data_nodes=2, cluster_managers=2, coordinating=1)+ with pytest.raises(OpenSearchNotReadyError):+ await wait_for_opensearch(client, max_retries=2, base_delay=0.0, max_delay=0.0)+++async def test_returns_when_flag_disabled(monkeypatch):+ _patch_settings(monkeypatch, enabled=False)+ client = _fake_os_client(data_nodes=1, cluster_managers=1, coordinating=1)+ # Flag off -> counts ignored, returns despite short counts.+ await wait_for_opensearch(client, max_retries=1, base_delay=0.0, max_delay=0.0)