OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: configurable OpenSearch shards/replicas + startup replica reconciliation (#1951) (#1964)
Edwin Jose27fdb1364167
diff --git a/.env.example b/.env.exampleindex 276418e31..01e404ed9 100644--- a/.env.example+++ b/.env.example@@ -119,6 +119,18 @@ OPENSEARCH_USERNAME=admin# Change this if you want to use a different index name or avoid conflictsOPENSEARCH_INDEX_NAME=documents+# OpenSearch index layout for newly-created OpenRAG indices+# Shard count cannot be changed on an existing index without reindexing.+# Local/dev OpenSearch is single-node, so default to 1 shard / 0 replicas here+# (production deployments default to 2/2 in code; override as needed).+OPENRAG_OPENSEARCH_NUMBER_OF_SHARDS=1+OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS=0++# Reconcile existing OpenRAG indices' replica counts to+# OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS on startup. Defaults to true in code+# (for multi-node production); disabled here since local/dev is single-node.+OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP=false+# IBM AMS Authentication (IBM Watsonx Data embedded mode)# Set IBM_AUTH_ENABLED=true to authenticate via the ibm-openrag-session cookie# instead of Google OAuth. The raw IBM JWT is also passed directly to OpenSearch.diff --git a/docker-compose.yml b/docker-compose.ymlindex 412d5f5ed..4d15fb04d 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -98,6 +98,9 @@ services:- IBM_COS_HMAC_SECRET_ACCESS_KEY=${IBM_COS_HMAC_SECRET_ACCESS_KEY}- IBM_COS_AUTH_ENDPOINT=${IBM_COS_AUTH_ENDPOINT}- OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}+ - OPENRAG_OPENSEARCH_NUMBER_OF_SHARDS=${OPENRAG_OPENSEARCH_NUMBER_OF_SHARDS:-1}+ - OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS=${OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS:-0}+ - OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP=${OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP:-false}- LANGFLOW_KEY=${LANGFLOW_KEY}- SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY:-}- ENVIRONMENT=${ENVIRONMENT:-production}diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex d7b6e3c8f..7b05b897a 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -152,6 +152,8 @@ Configure OpenSearch database authentication.| `OPENSEARCH_HOST` | `localhost` | OpenSearch service host. || `OPENSEARCH_PORT` | `9200` | OpenSearch service port. || `OPENSEARCH_USERNAME` | `admin` | OpenSearch administrator username. |+| `OPENRAG_OPENSEARCH_NUMBER_OF_SHARDS` | `2` | Primary shard count for newly-created OpenRAG indices. Existing indices must be reindexed to change shard count. |+| `OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS` | `2` | Replica shard count for OpenRAG indices. OpenRAG reconciles existing managed indices to this value at startup. || `LANGFLOW_OPENSEARCH_HOST` | Not set | By default, OpenRAG passes the `OPENSEARCH_HOST` value to Langflow. Use the `LANGFLOW_OPENSEARCH_*` variables to set a different OpenSearch endpoint for Langflow specifically. OpenRAG itself still uses the `OPENSEARCH_HOST` value. || `LANGFLOW_OPENSEARCH_PORT` | Not set | By default, OpenRAG passes the `OPENSEARCH_PORT` value to Langflow. Use the `LANGFLOW_OPENSEARCH_*` variables to set a different OpenSearch endpoint for Langflow specifically. OpenRAG itself still uses the `OPENSEARCH_PORT` value. |diff --git a/src/app/lifespan.py b/src/app/lifespan.pyindex 846580685..50f0603aa 100644--- a/src/app/lifespan.py+++ b/src/app/lifespan.py@@ -17,6 +17,7 @@ from config.settings import (JWT_CLAIMS_CACHE_MAX_SIZE,JWT_CLAIMS_CACHE_TTL_SECONDS,OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,+ OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP,OPENSEARCH_WAIT_MAX_RETRIES,RBAC_CACHE_BACKEND,RBAC_PERMISSION_CACHE_TTL_SECONDS,@@ -243,13 +244,20 @@ async def run_startup(app: FastAPI):log_bootstrap_env(logger, "startup")- # One-shot OpenSearch security bootstrap driven by the platform's- # service JWT. Runs synchronously (before startup_tasks) so the- # admin role mapping is in place before any other startup work- # talks to OpenSearch. The corresponding call inside startup_tasks- # is suppressed when this flag is on.- if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:- logger.info("OpenSearch security bootstrap enabled - starting")+ # One-shot OpenSearch security bootstrap (driven by the platform service+ # JWT) and/or startup replica reconciliation. Runs synchronously (before+ # startup_tasks) so the admin role mapping is in place before any other+ # startup work talks to OpenSearch. The block is entered when EITHER flag is+ # set so both steps can reuse the same per-run-mode authenticated client;+ # each step is then gated on its OWN flag. The setup_opensearch_security call+ # inside startup_tasks is suppressed when OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP+ # is on.+ if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP or OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP:+ logger.info(+ "OpenSearch startup tasks enabled",+ security_bootstrap=OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,+ reconcile_replicas=OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP,+ )from utils.opensearch_init import wait_for_opensearchfrom utils.opensearch_utils import setup_opensearch_securityfrom utils.run_mode_utils import (@@ -257,50 +265,86 @@ async def run_startup(app: FastAPI):is_run_mode_oss,)- # Choose the bootstrap OpenSearch client + admin identity, by run mode:+ # Choose the OpenSearch client + admin identity, by run mode:# saas -> platform service token (JWT); admin from its claim# on_prem -> OpenSearch basic auth; OpenSearch username as admin# oss -> OpenSearch basic auth; OpenSearch username as admin+ admin_username = None+ opensearch_client = Noneif is_run_mode_on_prem() or is_run_mode_oss():admin_username = get_opensearch_username()opensearch_client = clients.create_basic_opensearch_client(admin_username, get_opensearch_password())logger.info(- "OpenSearch security bootstrap: %s mode, using OpenSearch basic auth"+ "OpenSearch startup client: %s mode, using OpenSearch basic auth"% ("on_prem" if is_run_mode_on_prem() else "oss"),admin_username=admin_username,)else:service_token = get_openrag_service_token()if not service_token:- raise RuntimeError(- "OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP is enabled but "- "OPENRAG_SERVICE_TOKEN is not set"+ # A missing service token is fatal only when security bootstrap+ # was explicitly requested; replica reconciliation is best-effort,+ # so skip it rather than aborting startup.+ if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:+ raise RuntimeError(+ "OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP is enabled but "+ "OPENRAG_SERVICE_TOKEN is not set"+ )+ logger.warning(+ "Skipping startup OpenSearch replica reconciliation: saas mode "+ "requires OPENRAG_SERVICE_TOKEN, which is not set"+ )+ else:+ from auth.ibm_auth import admin_username_from_service_jwt++ admin_username = admin_username_from_service_jwt(service_token)+ if not admin_username and OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:+ raise RuntimeError(+ "OPENRAG_SERVICE_TOKEN has no 'username' or 'sub' claim; "+ "cannot bootstrap OpenSearch security"+ )+ opensearch_client = clients.create_opensearch_client_from_jwt(service_token)+ logger.info(+ "OpenSearch startup client: saas mode, using platform service token",+ admin_username=admin_username,)- from auth.ibm_auth import admin_username_from_service_jwt- admin_username = admin_username_from_service_jwt(service_token)- if not admin_username:- raise RuntimeError(- "OPENRAG_SERVICE_TOKEN has no 'username' or 'sub' claim; "- "cannot bootstrap OpenSearch security"+ if opensearch_client is not None:+ try:+ # Readiness is needed for either operation.+ logger.info("Verifying OpenSearch readiness")+ await wait_for_opensearch(+ opensearch_client, max_retries=OPENSEARCH_WAIT_MAX_RETRIES)- opensearch_client = clients.create_opensearch_client_from_jwt(service_token)- logger.info(- "OpenSearch security bootstrap: saas mode, using platform service token",- admin_username=admin_username,- )- try:- logger.info("Verifying OpenSearch readiness before bootstrap")- await wait_for_opensearch(opensearch_client, max_retries=OPENSEARCH_WAIT_MAX_RETRIES)- logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)- await setup_opensearch_security(opensearch_client, admin_username=admin_username)- logger.info("OpenSearch security bootstrap completed", admin_username=admin_username)- finally:- await opensearch_client.close()++ if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:+ logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)+ await setup_opensearch_security(+ opensearch_client, admin_username=admin_username+ )+ logger.info(+ "OpenSearch security bootstrap completed", admin_username=admin_username+ )++ # Reconcile replica counts on existing OpenRAG indices using the+ # authenticated (admin / service-token) client, before serving+ # traffic. Best-effort: failures are logged, not fatal.+ if OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP:+ try:+ from utils.opensearch_init import ensure_openrag_index_replicas++ logger.info("Reconciling index replicas at startup")+ await ensure_openrag_index_replicas(opensearch_client)+ except Exception as e:+ logger.warning(+ "Index replica reconciliation at startup failed", error=str(e)+ )+ finally:+ await opensearch_client.close()else:- logger.info("OpenSearch security bootstrap disabled - skipping (handled in startup_tasks)")+ logger.info("OpenSearch startup security/replica tasks disabled - skipping")# Start index initialization in background to avoid blocking OIDC endpointst1 = asyncio.create_task(startup_tasks(services))diff --git a/src/config/settings.py b/src/config/settings.pyindex 239cf27fc..8c411414c 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -380,6 +380,13 @@ OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP = os.getenv("OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP", "false").lower() in ("true", "1", "yes")+# Reconcile replica counts on existing OpenRAG indices at startup so they match+# OPENSEARCH_NUMBER_OF_REPLICAS. Defaults to true for production (multi-node)+# deployments; single-node dev (docker-compose) overrides it to false.+OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP = os.getenv(+ "OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP", "true"+).lower() in ("true", "1", "yes")+# Enable FastAPI's `debug` mode (verbose tracebacks in HTTP error responses# on the FastAPI app instance). Named explicitly so it isn't confused with# logging-level "debug" or other unrelated debug flags.@@ -594,16 +601,24 @@ WEBHOOK_RENEWAL_THRESHOLD_SECONDS = max(60, _raw_webhook_renewal_threshold)# actual frontend origin that is carried in the OAuth state parameter.OAUTH_BROKER_URL = os.getenv("OAUTH_BROKER_URL")++def _get_min_env_int(key: str, default: int, minimum: int) -> int:+ """Read an integer env var, clamped to a minimum valid value."""+ return max(get_env_int(key, default), minimum)++# OpenSearch configurationVECTOR_DIM = 1536KNN_EF_CONSTRUCTION = 100KNN_M = 16+OPENSEARCH_NUMBER_OF_SHARDS = _get_min_env_int("OPENRAG_OPENSEARCH_NUMBER_OF_SHARDS", 2, 1)+OPENSEARCH_NUMBER_OF_REPLICAS = _get_min_env_int("OPENRAG_OPENSEARCH_NUMBER_OF_REPLICAS", 2, 0)INDEX_BODY = {"settings": {"index": {"knn": True},- "number_of_shards": 1,- "number_of_replicas": 0,+ "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,+ "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,},"mappings": {"properties": {@@ -639,7 +654,10 @@ INDEX_BODY = {DLS_PRINCIPAL_INDEX_NAME = "openrag_dls_principals"DLS_PRINCIPAL_INDEX_BODY: dict[str, Any] = {"settings": {- "index": {"number_of_replicas": 0, "number_of_shards": 1},+ "index": {+ "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,+ "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,+ },},"mappings": {"properties": {@@ -658,8 +676,8 @@ DLS_PRINCIPAL_INDEX_BODY: dict[str, Any] = {API_KEYS_INDEX_NAME = "api_keys"API_KEYS_INDEX_BODY = {"settings": {- "number_of_shards": 1,- "number_of_replicas": 0,+ "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,+ "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,},"mappings": {"properties": {diff --git a/src/utils/embeddings.py b/src/utils/embeddings.pyindex 4226324d8..01b6c5086 100644--- a/src/utils/embeddings.py+++ b/src/utils/embeddings.py@@ -16,6 +16,8 @@ async def create_index_body(from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODELfrom config.settings import (ACL_PRINCIPAL_LABELS_MAPPING,+ OPENSEARCH_NUMBER_OF_REPLICAS,+ OPENSEARCH_NUMBER_OF_SHARDS,VECTOR_DIM,get_openrag_config,)@@ -59,6 +61,10 @@ async def create_index_body()return {- "settings": {"index": {"knn": True}, "number_of_shards": 1, "number_of_replicas": 0},+ "settings": {+ "index": {"knn": True},+ "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,+ "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,+ },"mappings": {"properties": properties},}diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.pyindex e794503e2..fc34954f5 100644--- a/src/utils/opensearch_init.py+++ b/src/utils/opensearch_init.py@@ -14,6 +14,8 @@ from config.settings import (IBM_AUTH_ENABLED,INDEX_BODY,OPENRAG_SKIP_OS_SECURITY_SETUP,+ OPENSEARCH_NUMBER_OF_REPLICAS,+ OPENSEARCH_NUMBER_OF_SHARDS,PLATFORM_AUTH_DEV_MODE,clients,get_index_name,@@ -102,6 +104,63 @@ async def _ensure_field_mappings()+async def _ensure_index_replicas(os_client, index_name: str) -> None:+ """Set an existing index's replica count to the configured value."""+ if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:+ return++ try:+ current = await os_client.indices.get_settings(index=index_name)+ current_replicas = int(+ current[index_name]["settings"]["index"].get(+ "number_of_replicas", OPENSEARCH_NUMBER_OF_REPLICAS+ )+ )+ if current_replicas != OPENSEARCH_NUMBER_OF_REPLICAS:+ await os_client.indices.put_settings(+ index=index_name,+ body={"index": {"number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS}},+ )+ logger.info(+ "Updated OpenSearch index replica setting",+ index_name=index_name,+ number_of_replicas=OPENSEARCH_NUMBER_OF_REPLICAS,+ )+ except Exception as e:+ logger.warning(+ "Failed to check or update index replicas",+ index_name=index_name,+ desired_replicas=OPENSEARCH_NUMBER_OF_REPLICAS,+ error=str(e),+ )+++async def ensure_openrag_index_replicas(os_client=None) -> None:+ """Reconcile replica counts for all known OpenRAG indices at startup.++ Aligns each existing OpenRAG index's number_of_replicas with+ OPENSEARCH_NUMBER_OF_REPLICAS. Indices that do not yet exist are skipped —+ init_index creates them with the correct replica count.+ """+ os_client = os_client or clients.opensearch+ index_names = [+ get_index_name(),+ "knowledge_filters",+ API_KEYS_INDEX_NAME,+ DLS_PRINCIPAL_INDEX_NAME,+ ]+ for index_name in index_names:+ try:+ if await os_client.indices.exists(index=index_name):+ await _ensure_index_replicas(os_client, index_name)+ except Exception as e:+ logger.warning(+ "Failed to ensure replicas for index",+ index_name=index_name,+ error=str(e),+ )++async def wait_for_opensearch(opensearch_client=None, max_retries: int = 30):"""Wait for OpenSearch to be ready, delegating to the shared utility."""from utils.opensearch_utils import (@@ -164,6 +223,7 @@ async def _ensure_opensearch_index():index_name,{"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},)+ await _ensure_index_replicas(clients.opensearch, index_name)returnawait clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)@@ -244,25 +304,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):index_name,{"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},)- if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):- # Set number of replicas to 0 to not create unused nodes in OpenSearch, in case it was created with more replicas- try:- current = await os_client.indices.get_settings(index=index_name)- current_replicas = int(- current[index_name]["settings"]["index"].get("number_of_replicas", 1)- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info("Updated documents index settings")- except Exception as e:- logger.warning(- "Failed to check or update index replicas",- index_name=index_name,- error=str(e),- )+ await _ensure_index_replicas(os_client, index_name)await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS)@@ -270,7 +312,10 @@ async def init_index(opensearch_client=None, admin_username: str = None):knowledge_filter_index_name = "knowledge_filters"knowledge_filter_index_body = {"settings": {- "index": {"number_of_replicas": 0, "number_of_shards": 1},+ "index": {+ "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,+ "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,+ },},"mappings": {"properties": {@@ -311,29 +356,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):knowledge_filter_index_name,["allowed_users", "allowed_groups", "allowed_principals"],)-- if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):- try:- current = await os_client.indices.get_settings(- index=knowledge_filter_index_name- )- current_replicas = int(- current[knowledge_filter_index_name]["settings"]["index"].get(- "number_of_replicas", 1- )- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=knowledge_filter_index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info("Updated knowledge filters index settings")- except Exception as e:- logger.warning(- "Failed to check or update knowledge filter index replicas",- index_name=knowledge_filter_index_name,- error=str(e),- )+ await _ensure_index_replicas(os_client, knowledge_filter_index_name)step = f"api_keys_index:{API_KEYS_INDEX_NAME}"if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):@@ -344,6 +367,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):"API keys index already exists, skipping creation",index_name=API_KEYS_INDEX_NAME,)+ await _ensure_index_replicas(os_client, API_KEYS_INDEX_NAME)step = f"dls_principal_index:{DLS_PRINCIPAL_INDEX_NAME}"if not await os_client.indices.exists(index=DLS_PRINCIPAL_INDEX_NAME):@@ -367,6 +391,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):DLS_PRINCIPAL_INDEX_NAME,{"principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},)+ await _ensure_index_replicas(os_client, DLS_PRINCIPAL_INDEX_NAME)step = "configure_alerting_security"await configure_alerting_security()diff --git a/tests/unit/test_embedding_fields.py b/tests/unit/test_embedding_fields.pyindex 1fa1acbbb..3d56e89cf 100644--- a/tests/unit/test_embedding_fields.py+++ b/tests/unit/test_embedding_fields.py@@ -124,3 +124,17 @@ class TestBuildKnnVectorFieldCallSitesMatch:assert properties[embedding_field] == build_knn_vector_field(3072)assert properties["owner_email"] == {"type": "keyword"}++ @pytest.mark.asyncio+ async def test_create_index_body_uses_configured_shards_and_replicas(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr("config.settings.OPENSEARCH_NUMBER_OF_SHARDS", 3)+ monkeypatch.setattr("config.settings.OPENSEARCH_NUMBER_OF_REPLICAS", 2)++ from utils.embeddings import create_index_body++ body = await create_index_body("text-embedding-3-small", 1536)++ assert body["settings"]["number_of_shards"] == 3+ assert body["settings"]["number_of_replicas"] == 2diff --git a/tests/unit/test_ensure_index_replicas.py b/tests/unit/test_ensure_index_replicas.pynew file mode 100644index 000000000..a2a0ddf43--- /dev/null+++ b/tests/unit/test_ensure_index_replicas.py@@ -0,0 +1,79 @@+"""Unit tests for startup replica reconciliation.++Covers ``ensure_openrag_index_replicas`` (and the underlying+``_ensure_index_replicas`` helper) in ``utils.opensearch_init``: it should align+each existing OpenRAG index's ``number_of_replicas`` with the configured+``OPENSEARCH_NUMBER_OF_REPLICAS`` and skip indices that do not exist.+"""++from typing import Any+from unittest.mock import AsyncMock++import pytest++import utils.opensearch_init as osi+from utils.opensearch_init import ensure_openrag_index_replicas++EXPECTED_INDEX_NAMES = [+ "documents",+ "knowledge_filters",+ "api_keys",+ "openrag_dls_principals",+]+++def _make_os_client(*, exists: bool, current_replicas: int) -> Any:+ """Build a fake async OpenSearch client with the indices methods used."""+ client = AsyncMock()+ client.indices.exists = AsyncMock(return_value=exists)+ client.indices.get_settings = AsyncMock(+ side_effect=lambda index: {+ index: {"settings": {"index": {"number_of_replicas": str(current_replicas)}}}+ }+ )+ client.indices.put_settings = AsyncMock()+ return client++[email protected](autouse=True)+def _stable_index_env(monkeypatch: pytest.MonkeyPatch) -> None:+ # Pin the configured target and the dynamic documents index name, and make+ # sure the IBM dev-mode short-circuit in _ensure_index_replicas stays off.+ monkeypatch.setattr(osi, "OPENSEARCH_NUMBER_OF_REPLICAS", 2)+ monkeypatch.setattr(osi, "get_index_name", lambda: "documents")+ monkeypatch.setattr(osi, "IBM_AUTH_ENABLED", False)+ monkeypatch.setattr(osi, "PLATFORM_AUTH_DEV_MODE", False)+ monkeypatch.setattr(osi, "API_KEYS_INDEX_NAME", "api_keys")+ monkeypatch.setattr(osi, "DLS_PRINCIPAL_INDEX_NAME", "openrag_dls_principals")+++async def test_corrects_replicas_for_all_known_indices() -> None:+ client = _make_os_client(exists=True, current_replicas=0)++ await ensure_openrag_index_replicas(client)++ assert client.indices.put_settings.await_count == len(EXPECTED_INDEX_NAMES)+ corrected = {call.kwargs["index"] for call in client.indices.put_settings.await_args_list}+ assert corrected == set(EXPECTED_INDEX_NAMES)+ for call in client.indices.put_settings.await_args_list:+ assert call.kwargs["body"] == {"index": {"number_of_replicas": 2}}+++async def test_noop_when_replicas_already_match() -> None:+ client = _make_os_client(exists=True, current_replicas=2)++ await ensure_openrag_index_replicas(client)++ client.indices.put_settings.assert_not_awaited()+++async def test_skips_missing_indices() -> None:+ client = _make_os_client(exists=False, current_replicas=0)++ await ensure_openrag_index_replicas(client)++ client.indices.get_settings.assert_not_awaited()+ client.indices.put_settings.assert_not_awaited()