Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
26.8 KB
Raw Blame
import asyncio
import copy
import os
import random
from typing import Any

import yaml
from opensearchpy import AsyncOpenSearch

from utils.logging_config import get_logger

logger = get_logger(__name__)

OPENRAG_USER_ROLE = "openrag_user_role"
OPENRAG_USER_ACL_ROLE = "openrag_user_acl_role"
ALL_ACCESS_ROLE = "all_access"
ROLE_MAPPING_PRINCIPAL_FIELDS = {"users", "hosts", "backend_roles", "and_backend_roles"}

DISK_SPACE_ERROR_MESSAGE = (
    "OpenSearch has run out of available disk space. "
    "Search and indexing operations are blocked. "
    "Please free up disk space to restore OpenRAG functionality."
)

# Error strings emitted by OpenSearch when disk watermark thresholds are breached
_DISK_SPACE_INDICATORS = [
    "disk watermark",
    "flood_stage",
    "flood stage",
    "disk usage exceeded",
    "index read-only",
    "no space left on device",
    "cluster_block_exception",
    "forbidden/12",
    "too_many_requests/12",
]


class OpenSearchNotReadyError(Exception):
    """Raised when OpenSearch fails to become ready within the retry limit."""


class OpenSearchDiskSpaceError(Exception):
    """Raised when OpenSearch operations fail due to insufficient disk space."""


# Error strings emitted by OpenSearch when the presented credential is rejected.
# This is an AUTHENTICATION failure (bad/expired/over-ridden credential), not an
# authorization one — callers must surface it as 401, never 403 "insufficient
# permissions", so the real cause isn't masked.
_AUTH_ERROR_INDICATORS = [
    "authenticationexception",
    "unauthorized",
    "authentication failed",
]

AUTH_ERROR_MESSAGE = (
    "Authentication failed: OpenSearch rejected the credential. Please sign in again."
)

_SECURITY_SETUP_SKIP_INDICATORS = [
    "401",
    "403",
    "404",
    "security_exception",
    "not_found",
    "unauthorized",
    "forbidden",
]


def _is_security_setup_skip_error(error: Exception) -> bool:
    """True when the security API is unavailable before any mutation is attempted."""
    error_str = str(error).lower()
    return any(indicator in error_str for indicator in _SECURITY_SETUP_SKIP_INDICATORS)


def _is_not_found_error(error: Exception) -> bool:
    """True when OpenSearch reports a missing security object."""
    error_str = str(error).lower()
    return "404" in error_str or "not_found" in error_str


def _dedupe_preserving_order(values: list[Any]) -> list[Any]:
    """Return unique values while preserving their first-seen order."""
    result: list[Any] = []
    for value in values:
        if value not in result:
            result.append(value)
    return result


def _merge_role_mappings(*mappings: dict[str, Any] | None) -> dict[str, Any]:
    """Merge OpenSearch role-mapping principals without dropping custom entries."""
    merged: dict[str, Any] = {}
    for mapping in mappings:
        if not mapping:
            continue
        for key, value in mapping.items():
            if key not in ROLE_MAPPING_PRINCIPAL_FIELDS:
                continue
            merged[key] = _dedupe_preserving_order((merged.get(key) or []) + (value or []))
    return merged


def is_opensearch_auth_error(error: Exception | str) -> bool:
    """Whether *error* is an OpenSearch authentication failure (401).

    Accepts an exception or an already-stringified error message. Distinct from
    authorization ("only the owner can …"): this means the credential OpenRAG
    presented was not accepted at all. Callers should map it to HTTP 401
    (re-authenticate), not 403.
    """
    error_str = str(error).lower()
    return any(indicator in error_str for indicator in _AUTH_ERROR_INDICATORS)


def is_disk_space_error(error: Exception) -> bool:
    """Check whether an exception is caused by OpenSearch disk space constraints.

    OpenSearch blocks write and search operations when disk usage crosses
    the high-watermark or flood-stage watermark thresholds.
    This function detects those error signatures.

    Args:
        error: The exception to inspect.

    Returns:
        True if the error is disk-space related, False otherwise.
    """
    error_str = str(error).lower()
    return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)


def opensearch_error_reason(error: Exception) -> str | None:
    """Return the most specific OpenSearch failure reason carried by an exception.

    OpenSearch transport errors stringify to just ``TransportError(500, '')``; the
    real cause (and, on mapping/parse failures, the offending field) lives in the
    ``info`` body under ``error.root_cause[0].reason`` or ``error.reason``. Returns
    ``None`` when the exception carries no such structured reason.
    """
    info = getattr(error, "info", None)
    if not isinstance(info, dict):
        return None
    err = info.get("error")
    if not isinstance(err, dict):
        return None
    root_cause = err.get("root_cause")
    if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):
        if root_cause[0].get("reason"):
            return root_cause[0]["reason"]
    return err.get("reason")


def opensearch_error_fields(error: Exception) -> dict[str, Any]:
    """Extract structured OpenSearch context from an exception for logging.

    Surfaces ``opensearch_status``/``opensearch_error``/``opensearch_info`` (and
    ``opensearch_root_cause`` when present) so every OpenSearch-touching call site
    logs the same schema instead of an opaque ``str(e)``. Non-OpenSearch
    exceptions yield an empty dict.
    """
    fields: dict[str, Any] = {}
    status = getattr(error, "status_code", None)
    if status is not None:
        fields["opensearch_status"] = status
    os_error = getattr(error, "error", None)
    if os_error is not None:
        fields["opensearch_error"] = os_error
    info = getattr(error, "info", None)
    if info is not None:
        fields["opensearch_info"] = info
        if isinstance(info, dict):
            err = info.get("error")
            if isinstance(err, dict):
                root_cause = err.get("root_cause")
                if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):
                    fields["opensearch_root_cause"] = root_cause[0]
    return fields


async def wait_for_opensearch(
    opensearch_client: AsyncOpenSearch,
    max_retries: int = 30,
    base_delay: float = 2.0,
    max_delay: float = 30.0,
) -> None:
    """Wait for OpenSearch to be ready with exponential backoff and jitter.

    Args:
        opensearch_client: The OpenSearch client to use for health checks.
        max_retries: Maximum number of retry attempts.
        base_delay: Initial delay in seconds before the first retry.
        max_delay: Upper bound in seconds for the retry delay.

    Raises:
        OpenSearchNotReadyError: If OpenSearch fails to become ready within the retry limit.
    """
    for attempt in range(max_retries):
        display_attempt: int = attempt + 1

        logger.info(
            "Verifying whether OpenSearch is ready...",
            attempt=display_attempt,
            max_retries=max_retries,
        )

        try:
            # Simple ping to check connection
            if await opensearch_client.ping():
                # Also check cluster health
                health = await opensearch_client.cluster.health()
                status = health.get("status")
                if status in ["green", "yellow"]:
                    from config.settings import (
                        OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,
                        OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,
                        OPENSEARCH_EXPECTED_DATA_NODE_COUNT,
                        OPENSEARCH_NODE_COUNT_CHECK_ENABLED,
                    )

                    if OPENSEARCH_NODE_COUNT_CHECK_ENABLED:
                        data_node_count = health.get("number_of_data_nodes", 0)
                        # Reachable cluster-manager (master) nodes.
                        cm_info = await opensearch_client.transport.perform_request(
                            "GET", "/_nodes/cluster_manager:true/process,transport"
                        )
                        cluster_manager_count = cm_info.get("_nodes", {}).get("successful", 0)
                        # Reachable coordinating-only nodes.
                        coord_info = await opensearch_client.transport.perform_request(
                            "GET", "/_nodes/coordinating_only:true/process,transport"
                        )
                        coordinating_count = coord_info.get("_nodes", {}).get("successful", 0)

                        if (
                            data_node_count < OPENSEARCH_EXPECTED_DATA_NODE_COUNT
                            or cluster_manager_count < OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT
                            or coordinating_count < OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT
                        ):
                            logger.warning(
                                "OpenSearch healthy but cluster has not reached expected node count.",
                                attempt=display_attempt,
                                status=status,
                                number_of_data_nodes=data_node_count,
                                cluster_manager_nodes=cluster_manager_count,
                                coordinating_nodes=coordinating_count,
                                expected_data_nodes=OPENSEARCH_EXPECTED_DATA_NODE_COUNT,
                                expected_cluster_managers=OPENSEARCH_EXPECTED_CLUSTER_MANAGER_COUNT,
                                expected_coordinating=OPENSEARCH_EXPECTED_COORDINATING_NODE_COUNT,
                            )
                            # Fall through to the retry/backoff below until nodes join.
                        else:
                            logger.info(
                                "Successfully verified that OpenSearch is ready.",
                                attempt=display_attempt,
                                status=status,
                                number_of_data_nodes=data_node_count,
                                cluster_manager_nodes=cluster_manager_count,
                                coordinating_nodes=coordinating_count,
                            )
                            return
                    else:
                        logger.info(
                            "Successfully verified that OpenSearch is ready.",
                            attempt=display_attempt,
                            status=status,
                        )
                        return
                else:
                    logger.warning(
                        "OpenSearch is up but cluster health is red.",
                        attempt=display_attempt,
                        status=status,
                    )
            else:
                logger.warning(
                    "OpenSearch ping failed.",
                    attempt=display_attempt,
                )
        except Exception as e:
            logger.warning(
                "OpenSearch is not ready.",
                attempt=display_attempt,
                error=str(e),
            )

        if attempt < max_retries - 1:
            delay = min(base_delay * (2**attempt), max_delay)
            delay = random.uniform(delay / 2, delay)

            logger.debug(
                "Retry OpenSearch readiness check after a delay (seconds).",
                attempt=display_attempt,
                delay=delay,
            )

            await asyncio.sleep(delay)

    message: str = "Failed to verify whether OpenSearch is ready."
    logger.error(message)
    raise OpenSearchNotReadyError(message)


async def graceful_opensearch_shutdown(opensearch_client: AsyncOpenSearch) -> None:
    """Gracefully shutdown OpenSearch client connection.

    This ensures that all pending operations are completed and connections
    are properly closed before the application exits.

    Args:
        opensearch_client: The OpenSearch client to shutdown.
    """
    if opensearch_client is None:
        logger.debug("OpenSearch client is None, skipping graceful shutdown")
        return

    try:
        logger.info("Initiating graceful OpenSearch shutdown...")

        # Flush any pending operations by checking cluster health one last time
        try:
            await asyncio.wait_for(opensearch_client.cluster.health(), timeout=10.0)
            logger.debug("Final cluster health check completed")
        except TimeoutError:
            logger.warning("Timeout during final cluster health check")
        except Exception as e:
            logger.debug("[OPENSEARCH] Final cluster health check skipped", reason=str(e))

        # Close the client connection
        await opensearch_client.close()
        logger.info("OpenSearch client connection closed gracefully")

    except Exception as e:
        logger.error("Error during graceful OpenSearch shutdown", error=str(e))


async def setup_opensearch_security(
    opensearch_client: AsyncOpenSearch,
    admin_username: str | None = None,
) -> None:
    """Setup OpenSearch roles and roles mapping.

    The setup involves:
    1. GET /_plugins/_security/api/rolesmapping (check existing)
    2. GET /_cluster/health
    3. PUT and verify additive roles/mappings, including openrag_user_acl_role.
    4. PUT narrowed legacy roles/mappings, preserving existing mapping principals.
    5. PUT /_plugins/_security/api/rolesmapping/all_access (merge admin mapping).
    6. Verify updated legacy roles/mappings with final GETs.

    Args:
        opensearch_client: Authenticated OpenSearch client.
        admin_username: OpenSearch username of the onboarding user (IBM mode).
            When provided, this user is pinned into the all_access role mapping's
            ``users`` list so they retain admin access after ``backend_roles``
            are modified for DLS.

    This should be called during initial setup after OpenSearch is ready.
    """
    from config.settings import IBM_AUTH_ENABLED, PLATFORM_AUTH_DEV_MODE

    if IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE:
        logger.info("Skipping OpenSearch security configuration in IBM dev mode.")
        return

    logger.info("Initializing OpenSearch security configuration...", ibm_auth=IBM_AUTH_ENABLED)

    base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    if IBM_AUTH_ENABLED:
        security_config_dir = os.path.join(base_dir, "cloud_securityconfig")
    else:
        security_config_dir = os.path.join(base_dir, "securityconfig")

    logger.info("[OPENSEARCH] Using config directory", config_dir=security_config_dir)

    roles_file = os.path.join(security_config_dir, "roles.yml")
    roles_mapping_file = os.path.join(security_config_dir, "roles_mapping.yml")

    logger.info(
        "[OPENSEARCH] Configuration paths",
        base_dir=base_dir,
        security_config_dir=security_config_dir,
        roles_file=roles_file,
        roles_mapping_file=roles_mapping_file,
    )

    try:
        # 1. & 2. Readiness checks
        logger.info("[OPENSEARCH] Performing readiness checks...")

        rolesmapping_response: dict[str, Any] = {}
        try:
            rolesmapping_response = await opensearch_client.transport.perform_request(
                "GET", "/_plugins/_security/api/rolesmapping"
            )
            logger.info(
                "[OPENSEARCH] Current rolesmapping retrieved",
                count=len(rolesmapping_response)
                if isinstance(rolesmapping_response, dict)
                else "unknown",
            )
        except Exception as e:
            if _is_security_setup_skip_error(e):
                logger.warning(
                    "Skipping OpenSearch security configuration: "
                    "The cluster may not have the security plugin enabled or "
                    "the provided credentials do not have administrative permissions."
                )
                return
            logger.error("[OPENSEARCH] Failed to get current rolesmapping", error=str(e))
            raise

        cluster_health = await opensearch_client.cluster.health()
        logger.info("[OPENSEARCH] Cluster health check passed", status=cluster_health.get("status"))

        # Load role definitions from YAML
        if not os.path.exists(roles_file):
            logger.error(f"[OPENSEARCH] Roles configuration file not found: {roles_file}")
            raise FileNotFoundError(f"Roles configuration file not found: {roles_file}")

        with open(roles_file) as f:
            roles_config = yaml.safe_load(f)

        logger.info(
            "[OPENSEARCH] Loaded roles configuration",
            roles=list(roles_config.keys()) if roles_config else [],
        )

        # Load roles mapping from YAML
        if not os.path.exists(roles_mapping_file):
            logger.error(f"[OPENSEARCH] Roles mapping file not found: {roles_mapping_file}")
            raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")

        with open(roles_mapping_file) as f:
            mapping_config = yaml.safe_load(f)

        logger.info(
            "[OPENSEARCH] Loaded roles mapping configuration",
            mappings=list(mapping_config.keys()) if mapping_config else [],
        )

        role_names = [role_name for role_name in roles_config if not role_name.startswith("_")]
        pre_legacy_role_names = [
            role_name for role_name in role_names if role_name != OPENRAG_USER_ROLE
        ]
        ordered_role_names = pre_legacy_role_names + [
            role_name for role_name in role_names if role_name == OPENRAG_USER_ROLE
        ]

        mapping_names = [
            mapping_name
            for mapping_name in mapping_config
            if not mapping_name.startswith("_") and mapping_name != ALL_ACCESS_ROLE
        ]
        pre_legacy_mapping_names = [
            mapping_name for mapping_name in mapping_names if mapping_name != OPENRAG_USER_ROLE
        ]
        ordered_mapping_names = pre_legacy_mapping_names + [
            mapping_name for mapping_name in mapping_names if mapping_name == OPENRAG_USER_ROLE
        ]

        def merged_mapping_body(mapping_name: str) -> dict[str, Any]:
            configured = copy.deepcopy(mapping_config[mapping_name])
            existing_same = rolesmapping_response.get(mapping_name, {})
            if mapping_name == OPENRAG_USER_ACL_ROLE:
                existing_legacy = rolesmapping_response.get(OPENRAG_USER_ROLE, {})
                return _merge_role_mappings(configured, existing_legacy, existing_same)
            if mapping_name == OPENRAG_USER_ROLE:
                return _merge_role_mappings(configured, existing_same)
            return _merge_role_mappings(configured, existing_same)

        async def put_role(role_name: str) -> None:
            role_body = roles_config[role_name]
            logger.info(
                f"[OPENSEARCH] Creating/updating '{role_name}' role",
                patterns=role_body["index_permissions"][0]["index_patterns"]
                if "index_permissions" in role_body
                else "default",
                allowed_actions=role_body["index_permissions"][0].get("allowed_actions", [])
                if "index_permissions" in role_body
                else [],
            )

            resp = await opensearch_client.transport.perform_request(
                "PUT",
                f"/_plugins/_security/api/roles/{role_name}",
                body=role_body,
                headers={"Content-Type": "application/json"},
            )
            logger.info(f"[OPENSEARCH] Role '{role_name}' creation response", response=resp)

        async def verify_role(role_name: str) -> None:
            role_verify = await opensearch_client.transport.perform_request(
                "GET", f"/_plugins/_security/api/roles/{role_name}"
            )
            logger.info("[OPENSEARCH] Role verification", role_name=role_name, role=role_verify)

        async def put_mapping(mapping_name: str) -> None:
            mapping_body = merged_mapping_body(mapping_name)
            logger.info(
                f"[OPENSEARCH] Creating/updating '{mapping_name}' mapping",
                backend_roles=mapping_body.get("backend_roles", []),
                users=mapping_body.get("users", []),
                hosts=mapping_body.get("hosts", []),
            )
            resp = await opensearch_client.transport.perform_request(
                "PUT",
                f"/_plugins/_security/api/rolesmapping/{mapping_name}",
                body=mapping_body,
                headers={"Content-Type": "application/json"},
            )
            logger.info(
                f"[OPENSEARCH] Role mapping '{mapping_name}' update response", response=resp
            )

        async def verify_mapping(mapping_name: str) -> None:
            mapping_verify = await opensearch_client.transport.perform_request(
                "GET", f"/_plugins/_security/api/rolesmapping/{mapping_name}"
            )
            logger.info(
                "[OPENSEARCH] Role mapping verification",
                mapping_name=mapping_name,
                mapping=mapping_verify,
            )

        # Apply and verify the additive ACL role/mapping before narrowing the
        # legacy role. This avoids losing shared-document access if a later
        # security API call fails during upgrade.
        for role_name in pre_legacy_role_names:
            await put_role(role_name)
        for mapping_name in pre_legacy_mapping_names:
            await put_mapping(mapping_name)
        for role_name in pre_legacy_role_names:
            await verify_role(role_name)
        for mapping_name in pre_legacy_mapping_names:
            await verify_mapping(mapping_name)

        for role_name in ordered_role_names:
            if role_name in pre_legacy_role_names:
                continue
            await put_role(role_name)
        for mapping_name in ordered_mapping_names:
            if mapping_name in pre_legacy_mapping_names:
                continue
            await put_mapping(mapping_name)

        # 5. Update all_access mapping — merge with existing to preserve
        # IBM-managed entries, but ensure backend_roles never contains
        # "all_access" (which would give IBM API key users the super-admin
        # role and bypass DLS).
        if ALL_ACCESS_ROLE in mapping_config:
            all_access_body = copy.deepcopy(mapping_config[ALL_ACCESS_ROLE])

            if "backend_roles" not in all_access_body:
                all_access_body["backend_roles"] = ["admin"]
            if "description" not in all_access_body:
                all_access_body["description"] = "Maps admin to all_access"

            # Always fetch existing mapping first so we never lose previous admins
            # in multi-tenant deployments where each tenant onboards independently.
            existing_users: list = []
            existing_hosts: list = []
            existing_backend_roles: list = []
            try:
                existing = await opensearch_client.transport.perform_request(
                    "GET", "/_plugins/_security/api/rolesmapping/all_access"
                )
                existing_mapping = existing.get(ALL_ACCESS_ROLE, {})
                existing_users = existing_mapping.get("users", []) or []
                existing_hosts = existing_mapping.get("hosts", []) or []
                existing_backend_roles = existing_mapping.get("backend_roles", []) or []
            except Exception as e:
                if not _is_not_found_error(e):
                    raise
                logger.debug("[OPENSEARCH] No existing all_access mapping found, creating fresh")

            # Build merged users: source file + cluster + new admin (bare + ibmlhapikey_ variant).
            # Adding both variants ensures the user can authenticate via JWT *and* via IBM
            # Basic-Auth (ibmlhapikey_<username>), which are treated as separate principals
            # by OpenSearch's security plugin.
            new_admin_users: list = []
            if IBM_AUTH_ENABLED and admin_username:
                new_admin_users = [admin_username, f"ibmlhapikey_{admin_username}"]
                logger.info(
                    "[OPENSEARCH] Pinning onboarding user as admin (both variants)",
                    users=new_admin_users,
                )

            merged_users = _dedupe_preserving_order(
                all_access_body.get("users", []) + existing_users + new_admin_users
            )
            all_access_body["users"] = merged_users
            logger.debug("[OPENSEARCH] Merged all_access users", users=merged_users)

            if existing_hosts:
                merged_hosts = _dedupe_preserving_order(
                    all_access_body.get("hosts", []) + existing_hosts
                )
                all_access_body["hosts"] = merged_hosts
                logger.debug(
                    "[OPENSEARCH] Preserved existing all_access hosts",
                    hosts=merged_hosts,
                )

            if existing_backend_roles:
                safe_existing_backend_roles = [
                    r for r in existing_backend_roles if r != "all_access"
                ]
                merged_backend_roles = _dedupe_preserving_order(
                    all_access_body.get("backend_roles", []) + safe_existing_backend_roles
                )
                all_access_body["backend_roles"] = merged_backend_roles
                logger.debug(
                    "[OPENSEARCH] Preserved existing all_access backend_roles",
                    backend_roles=merged_backend_roles,
                )

            if "all_access" in all_access_body.get("backend_roles", []):
                all_access_body["backend_roles"] = [
                    r for r in all_access_body["backend_roles"] if r != "all_access"
                ]
                logger.info(
                    "[OPENSEARCH] Removed 'all_access' from all_access backend_roles to preserve DLS",
                    final_backend_roles=all_access_body["backend_roles"],
                )

            logger.info("[OPENSEARCH] Updating 'all_access' mapping...", body=all_access_body)
            resp = await opensearch_client.transport.perform_request(
                "PUT",
                "/_plugins/_security/api/rolesmapping/all_access",
                body=all_access_body,
                headers={"Content-Type": "application/json"},
            )
            logger.info("[OPENSEARCH] All access mapping update response", response=resp)

        # 6. Final verification
        logger.info("[OPENSEARCH] Verifying security configuration...")
        for role_name in ordered_role_names:
            if role_name in pre_legacy_role_names:
                continue
            await verify_role(role_name)
        for mapping_name in ordered_mapping_names:
            if mapping_name in pre_legacy_mapping_names:
                continue
            await verify_mapping(mapping_name)

        logger.info("Successfully completed OpenSearch security configuration.")

    except Exception as e:
        logger.error("Failed to setup OpenSearch security configuration", error=str(e))
        raise