Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
16.5 KB
Raw Blame
"""OpenSearch index/security initialization at startup.

Pure functions that create or update OpenSearch indices and apply security
configuration. Imported by app startup orchestration and by api/* modules
that need to (re)create the documents index after onboarding completes.
"""

from config.settings import (
    ACL_PRINCIPAL_LABELS_MAPPING,
    API_KEYS_INDEX_BODY,
    API_KEYS_INDEX_NAME,
    DLS_PRINCIPAL_INDEX_BODY,
    DLS_PRINCIPAL_INDEX_NAME,
    IBM_AUTH_ENABLED,
    INDEX_BODY,
    OPENRAG_SKIP_OS_SECURITY_SETUP,
    OPENSEARCH_NUMBER_OF_REPLICAS,
    OPENSEARCH_NUMBER_OF_SHARDS,
    PLATFORM_AUTH_DEV_MODE,
    clients,
    get_index_name,
    get_openrag_config,
)
from utils.embeddings import create_index_body
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient

logger = get_logger(__name__)


async def _ensure_keyword_mappings(os_client, index_name: str, field_names: list[str]) -> None:
    """Add missing keyword mappings to an existing index.

    New ACL fields must be explicit keyword fields before documents are indexed;
    otherwise OpenSearch dynamic mapping can create analyzed text fields that do
    not work for exact-match DLS terms queries.
    """
    try:
        current_mapping = await os_client.indices.get_mapping(index=index_name)
        properties = current_mapping.get(index_name, {}).get("mappings", {}).get("properties", {})
        missing: dict[str, dict[str, str]] = {}
        for field_name in field_names:
            existing = properties.get(field_name)
            if existing is None:
                missing[field_name] = {"type": "keyword"}
            elif existing.get("type") != "keyword":
                logger.warning(
                    "OpenSearch field has incompatible mapping for DLS exact match",
                    index_name=index_name,
                    field_name=field_name,
                    mapping=existing,
                )

        if missing:
            await os_client.indices.put_mapping(
                index=index_name,
                body={"properties": missing},
            )
            logger.info(
                "Updated OpenSearch keyword mappings",
                index_name=index_name,
                fields=list(missing),
            )
    except Exception as e:
        logger.warning(
            "Failed to ensure OpenSearch keyword mappings",
            index_name=index_name,
            fields=field_names,
            error=str(e),
        )


async def _ensure_field_mappings(
    os_client,
    index_name: str,
    field_mappings: dict[str, dict],
) -> None:
    """Add missing explicit mappings to an existing index."""
    try:
        current_mapping = await os_client.indices.get_mapping(index=index_name)
        properties = current_mapping.get(index_name, {}).get("mappings", {}).get("properties", {})
        missing = {
            field_name: mapping
            for field_name, mapping in field_mappings.items()
            if properties.get(field_name) is None
        }

        if missing:
            await os_client.indices.put_mapping(
                index=index_name,
                body={"properties": missing},
            )
            logger.info(
                "Updated OpenSearch field mappings",
                index_name=index_name,
                fields=list(missing),
            )
    except Exception as e:
        logger.warning(
            "Failed to ensure OpenSearch field mappings",
            index_name=index_name,
            fields=list(field_mappings),
            error=str(e),
        )


async def _ensure_index_replicas(os_client, index_name: str) -> None:
    """Set an existing index's replica count to the configured value."""
    if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
        return

    try:
        current = await os_client.indices.get_settings(index=index_name)
        current_replicas = int(
            current[index_name]["settings"]["index"].get(
                "number_of_replicas", OPENSEARCH_NUMBER_OF_REPLICAS
            )
        )
        if current_replicas != OPENSEARCH_NUMBER_OF_REPLICAS:
            await os_client.indices.put_settings(
                index=index_name,
                body={"index": {"number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS}},
            )
            logger.info(
                "Updated OpenSearch index replica setting",
                index_name=index_name,
                number_of_replicas=OPENSEARCH_NUMBER_OF_REPLICAS,
            )
    except Exception as e:
        logger.warning(
            "Failed to check or update index replicas",
            index_name=index_name,
            desired_replicas=OPENSEARCH_NUMBER_OF_REPLICAS,
            error=str(e),
        )


async def ensure_openrag_index_replicas(os_client=None) -> None:
    """Reconcile replica counts for all known OpenRAG indices at startup.

    Aligns each existing OpenRAG index's number_of_replicas with
    OPENSEARCH_NUMBER_OF_REPLICAS. Indices that do not yet exist are skipped —
    init_index creates them with the correct replica count.
    """
    os_client = os_client or clients.opensearch
    index_names = [
        get_index_name(),
        "knowledge_filters",
        API_KEYS_INDEX_NAME,
        DLS_PRINCIPAL_INDEX_NAME,
    ]
    for index_name in index_names:
        try:
            if await os_client.indices.exists(index=index_name):
                await _ensure_index_replicas(os_client, index_name)
        except Exception as e:
            logger.warning(
                "Failed to ensure replicas for index",
                index_name=index_name,
                error=str(e),
            )


async def wait_for_opensearch(opensearch_client=None, max_retries: int = 30):
    """Wait for OpenSearch to be ready, delegating to the shared utility."""
    from utils.opensearch_utils import (
        OpenSearchNotReadyError,
    )
    from utils.opensearch_utils import (
        wait_for_opensearch as _wait_for_opensearch,
    )

    try:
        logger.info("Waiting for OpenSearch to be ready...", max_retries=max_retries)
        await _wait_for_opensearch(opensearch_client or clients.opensearch, max_retries=max_retries)
        logger.info("OpenSearch is ready!")
        await TelemetryClient.send_event(
            Category.OPENSEARCH_SETUP, MessageId.ORB_OS_CONN_ESTABLISHED
        )
    except OpenSearchNotReadyError:
        await TelemetryClient.send_event(Category.OPENSEARCH_SETUP, MessageId.ORB_OS_TIMEOUT)
        raise


async def configure_alerting_security():
    """Configure OpenSearch alerting plugin security settings"""
    if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
        logger.info("Skipping alerting security configuration in IBM dev mode.")
        return

    try:
        alerting_settings = {
            "persistent": {
                "plugins.alerting.filter_by_backend_roles": "false",
                "opendistro.alerting.filter_by_backend_roles": "false",
                "opensearch.notifications.general.filter_by_backend_roles": "false",
            }
        }

        response = await clients.opensearch.cluster.put_settings(body=alerting_settings)
        logger.info("Alerting security settings configured successfully", response=response)
    except Exception as e:
        logger.error("Failed to configure alerting security settings", error=str(e))


async def _ensure_opensearch_index():
    """Ensure OpenSearch index exists when using traditional connector service."""
    if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
        logger.info("Skipping OpenSearch index creation in IBM dev mode.")
        return

    try:
        index_name = get_index_name()
        if await clients.opensearch.indices.exists(index=index_name):
            logger.info("[OPENSEARCH] Index already exists", index_name=index_name)
            await _ensure_keyword_mappings(
                clients.opensearch,
                index_name,
                [
                    "allowed_users",
                    "allowed_groups",
                    "allowed_principals",
                    "ingest_run_id",
                    "connector_file_id",
                ],
            )
            await _ensure_field_mappings(
                clients.opensearch,
                index_name,
                {"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
            )
            await _ensure_index_replicas(clients.opensearch, index_name)
            return

        await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)
        logger.info(
            "Created OpenSearch index for traditional connector service",
            index_name=index_name,
            vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"]["dimension"],
        )
        await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED)

    except Exception as e:
        logger.error(
            "Failed to initialize OpenSearch index for traditional connector service",
            error=str(e),
            index_name=get_index_name(),
        )
        await TelemetryClient.send_event(
            Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL
        )


async def init_index(opensearch_client=None, admin_username: str = None):
    """Initialize OpenSearch index and security roles"""
    os_client = opensearch_client or clients.opensearch
    # Tracks the most recent OpenSearch operation so the except block can name
    # the failing step (TransportError stringifies to just "TransportError(500, '')").
    step = "wait_for_opensearch"
    try:
        await wait_for_opensearch(opensearch_client)

        # Skip security setup when the platform manages it externally
        # (SaaS / CPD). Index creation below still runs — SaaS / CPD
        # deployments still need indices, they just don't want OpenRAG
        # touching roles or role mappings.
        if OPENRAG_SKIP_OS_SECURITY_SETUP:
            logger.info(
                "Skipping OpenSearch security setup during init_index "
                "(OPENRAG_SKIP_OS_SECURITY_SETUP=true)",
                admin_username=admin_username,
            )
        else:
            from utils.opensearch_utils import setup_opensearch_security

            step = "setup_opensearch_security"
            await setup_opensearch_security(os_client, admin_username=admin_username)

        step = "build_index_body"
        config = get_openrag_config()
        embedding_model = config.knowledge.embedding_model

        index_body = await create_index_body()

        index_name = get_index_name()
        step = f"documents_index:{index_name}"
        if not await os_client.indices.exists(index=index_name):
            await os_client.indices.create(index=index_name, body=index_body)
            logger.info(
                "Created OpenSearch index",
                index_name=index_name,
                embedding_model=embedding_model,
            )
            await TelemetryClient.send_event(
                Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED
            )
        else:
            logger.info(
                "Index already exists, skipping creation",
                index_name=index_name,
                embedding_model=embedding_model,
            )
            await _ensure_keyword_mappings(
                os_client,
                index_name,
                [
                    "allowed_users",
                    "allowed_groups",
                    "allowed_principals",
                    "ingest_run_id",
                    "connector_file_id",
                ],
            )
            await _ensure_field_mappings(
                os_client,
                index_name,
                {"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
            )
            await _ensure_index_replicas(os_client, index_name)
            await TelemetryClient.send_event(
                Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS
            )

        knowledge_filter_index_name = "knowledge_filters"
        knowledge_filter_index_body = {
            "settings": {
                "index": {
                    "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,
                    "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,
                },
            },
            "mappings": {
                "properties": {
                    "id": {"type": "keyword"},
                    "name": {"type": "text", "analyzer": "standard"},
                    "description": {"type": "text", "analyzer": "standard"},
                    "query_data": {"type": "text"},
                    "owner": {"type": "keyword"},
                    "allowed_users": {"type": "keyword"},
                    "allowed_groups": {"type": "keyword"},
                    "allowed_principals": {"type": "keyword"},
                    "subscriptions": {"type": "object"},
                    "created_at": {"type": "date"},
                    "updated_at": {"type": "date"},
                }
            },
        }

        step = f"knowledge_filters_index:{knowledge_filter_index_name}"
        if not await os_client.indices.exists(index=knowledge_filter_index_name):
            await os_client.indices.create(
                index=knowledge_filter_index_name, body=knowledge_filter_index_body
            )
            logger.info(
                "Created knowledge filters index",
                index_name=knowledge_filter_index_name,
            )
            await TelemetryClient.send_event(
                Category.OPENSEARCH_INDEX, MessageId.ORB_OS_KF_INDEX_CREATED
            )
        else:
            logger.info(
                "Knowledge filters index already exists, skipping creation",
                index_name=knowledge_filter_index_name,
            )
            await _ensure_keyword_mappings(
                os_client,
                knowledge_filter_index_name,
                ["allowed_users", "allowed_groups", "allowed_principals"],
            )
            await _ensure_index_replicas(os_client, knowledge_filter_index_name)

        step = f"api_keys_index:{API_KEYS_INDEX_NAME}"
        if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):
            await os_client.indices.create(index=API_KEYS_INDEX_NAME, body=API_KEYS_INDEX_BODY)
            logger.info("Created API keys index", index_name=API_KEYS_INDEX_NAME)
        else:
            logger.info(
                "API keys index already exists, skipping creation",
                index_name=API_KEYS_INDEX_NAME,
            )
            await _ensure_index_replicas(os_client, API_KEYS_INDEX_NAME)

        step = f"dls_principal_index:{DLS_PRINCIPAL_INDEX_NAME}"
        if not await os_client.indices.exists(index=DLS_PRINCIPAL_INDEX_NAME):
            await os_client.indices.create(
                index=DLS_PRINCIPAL_INDEX_NAME,
                body=DLS_PRINCIPAL_INDEX_BODY,
            )
            logger.info("Created DLS principal index", index_name=DLS_PRINCIPAL_INDEX_NAME)
        else:
            logger.info(
                "DLS principal index already exists, skipping creation",
                index_name=DLS_PRINCIPAL_INDEX_NAME,
            )
            await _ensure_keyword_mappings(
                os_client,
                DLS_PRINCIPAL_INDEX_NAME,
                ["user_name", "auth_user_id", "auth_email", "provider", "principals"],
            )
            await _ensure_field_mappings(
                os_client,
                DLS_PRINCIPAL_INDEX_NAME,
                {"principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
            )
            await _ensure_index_replicas(os_client, DLS_PRINCIPAL_INDEX_NAME)

        step = "configure_alerting_security"
        await configure_alerting_security()

    except Exception as e:
        from utils.opensearch_utils import (
            OpenSearchDiskSpaceError,
            is_disk_space_error,
            opensearch_error_fields,
        )

        # TransportError stringifies to just "TransportError(500, '')" — surface
        # the failing step plus the OpenSearch status/error/body it carries.
        err_fields = {
            "failed_step": step,
            "error": str(e),
            "error_repr": repr(e),
            **opensearch_error_fields(e),
        }
        logger.error("init_index failed", **err_fields)

        if is_disk_space_error(e):
            logger.error("OpenSearch disk space exceeded watermark. Index creation failed.")
            raise OpenSearchDiskSpaceError(
                "OpenSearch disk space is full (watermark exceeded). "
                "Please free up disk space on your Docker volume or host machine to continue."
            ) from e
        raise