Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
18.4 KB
Raw Blame
"""Consolidated startup + shutdown lifecycle for the OpenRAG backend.

`run_startup(app)` and `run_shutdown(app)` collapse what previously lived
as three @app.on_event("startup") handlers and three @app.on_event("shutdown")
handlers into one helper each. The factory registers these as on_event
handlers so they fire under FastAPI's lifespan-from-on_event flow, both in
production (the ASGI server drives it) and in tests (which enter/exit
`app.router.lifespan_context(app)` directly, since Starlette 1.x removed the
`app.router.startup()` / `app.router.shutdown()` helpers).
"""

import asyncio

from fastapi import FastAPI

from config.settings import (
    JWT_CLAIMS_CACHE_MAX_SIZE,
    JWT_CLAIMS_CACHE_TTL_SECONDS,
    OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,
    OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP,
    OPENSEARCH_WAIT_MAX_RETRIES,
    RBAC_CACHE_BACKEND,
    RBAC_PERMISSION_CACHE_TTL_SECONDS,
    UVICORN_WORKER_COUNT,
    clients,
    get_openrag_config,
    get_openrag_service_token,
    get_opensearch_password,
    get_opensearch_username,
)
from services.startup_orchestrator import startup_tasks
from utils.logging_config import get_logger, log_bootstrap_env
from utils.telemetry import Category, MessageId, TelemetryClient

logger = get_logger(__name__)


async def cleanup_subscriptions_proper(services):
    """Cancel all active webhook subscriptions"""
    logger.info("Cancelling active webhook subscriptions")

    try:
        connector_service = services["connector_service"]
        await connector_service.connection_manager.load_connections()

        all_connections = await connector_service.connection_manager.list_connections()
        active_connections = [
            c for c in all_connections if c.is_active and c.config.get("webhook_channel_id")
        ]

        for connection in active_connections:
            try:
                logger.info(
                    "Cancelling subscription for connection",
                    connection_id=connection.connection_id,
                )
                connector = await connector_service.get_connector(connection.connection_id)
                if connector:
                    subscription_id = connection.config.get("webhook_channel_id")
                    await connector.cleanup_subscription(subscription_id)
                    logger.info("Cancelled subscription", subscription_id=subscription_id)
            except Exception as e:
                logger.error(
                    "Failed to cancel subscription",
                    connection_id=connection.connection_id,
                    error=str(e),
                )

        logger.info(
            "Finished cancelling subscriptions",
            subscription_count=len(active_connections),
        )

    except Exception as e:
        logger.error("Failed to cleanup subscriptions", error=str(e))


async def _periodic_backup(services):
    """Run flow backups every 5 minutes once onboarding is complete."""
    while True:
        try:
            await asyncio.sleep(5 * 60)

            config = get_openrag_config()
            if not config.edited:
                logger.debug("Onboarding not completed yet, skipping periodic backup")
                continue

            flows_service = services.get("flows_service")
            if flows_service:
                logger.info("Running periodic flow backup")
                backup_results = await flows_service.backup_all_flows(only_if_changed=True)
                if backup_results["backed_up"]:
                    logger.info(
                        "Periodic backup completed",
                        backed_up=len(backup_results["backed_up"]),
                        skipped=len(backup_results["skipped"]),
                    )
                else:
                    logger.debug(
                        "Periodic backup: no flows changed",
                        skipped=len(backup_results["skipped"]),
                    )
        except asyncio.CancelledError:
            logger.info("Periodic backup task cancelled")
            break
        except Exception as e:
            logger.error(f"Error in periodic backup task: {str(e)}")


async def _periodic_webhook_renewal(services):
    """Renew connector webhook subscriptions before they expire.

    Checks immediately at startup (subscriptions may have lapsed while the
    app was down), then on a fixed interval. Provider subscriptions are
    short-lived (Google Drive ~24h, Microsoft Graph 3 days) and go silent
    without renewal.
    """
    from config.settings import (
        WEBHOOK_RENEWAL_INTERVAL_SECONDS,
        WEBHOOK_RENEWAL_THRESHOLD_SECONDS,
    )

    # Let startup_tasks finish loading connections before the first pass
    await asyncio.sleep(60)

    while True:
        try:
            connector_service = services.get("connector_service")
            if connector_service:
                stats = await connector_service.connection_manager.renew_expiring_subscriptions(
                    WEBHOOK_RENEWAL_THRESHOLD_SECONDS
                )
                if stats["renewed"] or stats["failed"]:
                    logger.info("Webhook subscription renewal pass completed", **stats)
                else:
                    logger.debug("Webhook subscription renewal pass completed", **stats)
        except asyncio.CancelledError:
            logger.info("Webhook renewal task cancelled")
            break
        except Exception as e:
            logger.error(f"Error in webhook renewal task: {str(e)}")
        await asyncio.sleep(WEBHOOK_RENEWAL_INTERVAL_SECONDS)


async def run_startup(app: FastAPI):
    """Single source of truth for startup work.

    Expects `app.state.services`, `app.state.mcp_lifespan_ctx`, and
    `app.state.background_tasks` to be set by the factory.
    """
    services = app.state.services
    mcp_lifespan_ctx = getattr(app.state, "mcp_lifespan_ctx", None)

    # Hard-fail if the operator has configured multiple workers. The
    # RBAC permission cache and the OAuth-subject→DB-id cache are
    # both per-process; a second worker silently sees stale
    # permissions for up to OPENRAG_PERM_CACHE_TTL seconds after
    # role mutations. Until the cache moves to a shared backend
    # (Redis), this constraint is real and must be enforced.
    if UVICORN_WORKER_COUNT > 1:
        logger.error(
            "Multi-worker deployment unsupported until cache is "
            "shared across processes. Set UVICORN_WORKERS=1.",
            requested_workers=UVICORN_WORKER_COUNT,
        )
        raise RuntimeError("UVICORN_WORKERS>1 is not supported")
    if RBAC_CACHE_BACKEND not in ("memory",):
        logger.error(
            "Unsupported CACHE_BACKEND. Only 'memory' is wired.",
            requested=RBAC_CACHE_BACKEND,
        )
        raise RuntimeError(f"unsupported CACHE_BACKEND={RBAC_CACHE_BACKEND!r}")
    logger.info(
        "Permission cache configured",
        backend=RBAC_CACHE_BACKEND,
        workers=UVICORN_WORKER_COUNT,
        perm_cache_ttl_s=RBAC_PERMISSION_CACHE_TTL_SECONDS,
    )
    logger.info(
        "JWT claims cache configured",
        backend="memory",
        ttl_s=JWT_CLAIMS_CACHE_TTL_SECONDS,
        maxsize=JWT_CLAIMS_CACHE_MAX_SIZE,
    )

    # RBAC kill-switch visibility. OPENRAG_RBAC_ENFORCE=false makes
    # every authenticated user effectively admin — log loudly so
    # operators see it on every boot. Available in all run modes;
    # the operator owns the trade-off.
    from services.rbac_service import is_rbac_enforced
    from utils.run_mode_utils import get_run_mode

    if is_rbac_enforced():
        logger.info("RBAC enforcement is ON", run_mode=get_run_mode())
    else:
        logger.warning(
            "RBAC enforcement is DISABLED — every authenticated "
            "user has full access via the OPENRAG_RBAC_ENFORCE=false "
            "kill switch.",
            run_mode=get_run_mode(),
        )

    # Backend LLM tracing to Langfuse (OPENRAG_TRACING_ENABLED, default off).
    # Registered before any gateway call can run; misconfiguration warns and
    # leaves the backend untraced rather than failing startup.
    from observability.tracing import configure_tracing

    configure_tracing()

    # Open the SQL engine on uvicorn's live loop (NOT the one used by
    # asyncio.run(create_app()) which closed already). All RBAC code
    # uses RBACService's lazy factory that reads db.engine.SessionLocal
    # at call time, so it will pick up the binding we set here.
    try:
        from db.engine import init_engine

        init_engine()
    except Exception as e:
        logger.error("DB engine init failed at startup", error=str(e))
        raise

    # One-shot JSON->DB migration + test-fixture cleanup. Runs once
    # per install, idempotent via migration_status rows.
    try:
        from db.engine import SessionLocal as _SL
        from db.migrations_runtime import run as run_runtime_migration

        if _SL is not None:
            async with _SL() as _session:
                await run_runtime_migration(_session)
                await _session.commit()
    except Exception as e:
        logger.error("Runtime DB migration failed", error=str(e))
        raise

    try:
        wcs = services.get("workspace_config_service")
        if wcs is not None:
            await wcs.hydrate_on_startup()
    except Exception as e:
        logger.error("Workspace config hydration failed at startup", error=str(e))

    await TelemetryClient.send_event(Category.APPLICATION_STARTUP, MessageId.ORB_APP_STARTED)

    # FastMCP requires its own lifespan to be entered before requests
    # arrive so the StreamableHTTPSessionManager task group exists.
    if mcp_lifespan_ctx:
        await mcp_lifespan_ctx.__aenter__()
        logger.info("FastMCP lifespan started")

    log_bootstrap_env(logger, "startup")

    # One-shot OpenSearch security bootstrap (driven by the platform service
    # JWT) and/or startup replica reconciliation. Runs synchronously (before
    # startup_tasks) so the admin role mapping is in place before any other
    # startup work talks to OpenSearch. The block is entered when EITHER flag is
    # set so both steps can reuse the same per-run-mode authenticated client;
    # each step is then gated on its OWN flag. The setup_opensearch_security call
    # inside startup_tasks is suppressed when OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP
    # is on.
    if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP or OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP:
        logger.info(
            "OpenSearch startup tasks enabled",
            security_bootstrap=OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,
            reconcile_replicas=OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP,
        )
        from utils.opensearch_init import wait_for_opensearch
        from utils.opensearch_utils import setup_opensearch_security
        from utils.run_mode_utils import (
            get_run_mode,
            is_run_mode_on_prem,
            is_run_mode_saas,
        )

        # Choose the OpenSearch client + admin identity, by run mode:
        #   saas/on_prem -> platform service token (JWT); admin from its claim
        #   oss          -> OpenSearch basic auth; OpenSearch username as admin
        admin_username = None
        opensearch_client = None
        if is_run_mode_saas() or is_run_mode_on_prem():
            service_token = get_openrag_service_token()
            if not service_token:
                # A missing service token is fatal only when security bootstrap
                # was explicitly requested; replica reconciliation is best-effort,
                # so skip it rather than aborting startup.
                if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:
                    raise RuntimeError(
                        "OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP is enabled but "
                        "OPENRAG_SERVICE_TOKEN is not set"
                    )
                logger.warning(
                    "Skipping startup OpenSearch replica reconciliation: "
                    f"{get_run_mode()} mode requires OPENRAG_SERVICE_TOKEN, "
                    "which is not set"
                )
            else:
                from auth.ibm_auth import admin_username_from_service_jwt

                admin_username = admin_username_from_service_jwt(service_token)
                if not admin_username and OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:
                    raise RuntimeError(
                        "OPENRAG_SERVICE_TOKEN has no 'username' or 'sub' claim; "
                        "cannot bootstrap OpenSearch security"
                    )
                opensearch_client = clients.create_opensearch_client_from_jwt(service_token)
                logger.info(
                    f"OpenSearch startup client: {get_run_mode()} mode, "
                    "using platform service token",
                    admin_username=admin_username,
                )
        else:
            admin_username = get_opensearch_username()
            opensearch_client = clients.create_basic_opensearch_client(
                admin_username, get_opensearch_password()
            )
            logger.info(
                "OpenSearch startup client: oss mode, using OpenSearch basic auth",
                admin_username=admin_username,
            )

        if opensearch_client is not None:
            try:
                # Readiness is needed for either operation.
                logger.info("Verifying OpenSearch readiness")
                await wait_for_opensearch(
                    opensearch_client, max_retries=OPENSEARCH_WAIT_MAX_RETRIES
                )

                if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:
                    logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)
                    await setup_opensearch_security(
                        opensearch_client, admin_username=admin_username
                    )
                    logger.info(
                        "OpenSearch security bootstrap completed", admin_username=admin_username
                    )

                # Reconcile replica counts on existing OpenRAG indices using the
                # authenticated (admin / service-token) client, before serving
                # traffic. Best-effort: failures are logged, not fatal.
                if OPENRAG_ENSURE_INDEX_REPLICAS_ON_STARTUP:
                    try:
                        from utils.opensearch_init import ensure_openrag_index_replicas

                        logger.info("Reconciling index replicas at startup")
                        await ensure_openrag_index_replicas(opensearch_client)
                    except Exception as e:
                        logger.warning(
                            "Index replica reconciliation at startup failed", error=str(e)
                        )
            finally:
                await opensearch_client.close()
    else:
        logger.info("OpenSearch startup security/replica tasks disabled - skipping")

    # Start index initialization in background to avoid blocking OIDC endpoints
    t1 = asyncio.create_task(startup_tasks(services))
    app.state.background_tasks.add(t1)
    t1.add_done_callback(app.state.background_tasks.discard)

    # Start periodic task cleanup scheduler
    services["task_service"].start_cleanup_scheduler()

    # Start periodic flow backup task (every 5 minutes)
    backup_task = asyncio.create_task(_periodic_backup(services))
    app.state.background_tasks.add(backup_task)
    backup_task.add_done_callback(app.state.background_tasks.discard)

    # Start periodic webhook subscription renewal task
    renewal_task = asyncio.create_task(_periodic_webhook_renewal(services))
    app.state.background_tasks.add(renewal_task)
    renewal_task.add_done_callback(app.state.background_tasks.discard)


async def run_shutdown(app: FastAPI):
    """Single source of truth for shutdown work."""
    services = app.state.services
    mcp_lifespan_ctx = getattr(app.state, "mcp_lifespan_ctx", None)

    logger.info("Application shutdown initiated")

    await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)

    # Cancel and await our long-lived background tasks (startup_tasks,
    # _periodic_backup) before tearing down the resources they touch
    # (clients, task_service, db engine). Without this, a periodic
    # backup can wake up mid-shutdown and crash on a closed connection
    # or, worse, write partial state to a half-disposed engine.
    background_tasks = list(getattr(app.state, "background_tasks", set()))
    for task in background_tasks:
        task.cancel()
    if background_tasks:
        await asyncio.gather(*background_tasks, return_exceptions=True)
        logger.info("Background tasks cancelled at shutdown", count=len(background_tasks))

    # Drain any pending workspace_config DB-mirror tasks before we
    # tear down the engine. Without this, a save_config triggered
    # right before shutdown can be cancelled mid-write, leaving
    # yaml and DB out of sync.
    try:
        wcs = services.get("workspace_config_service") if isinstance(services, dict) else None
        if wcs is not None:
            await wcs.await_pending_mirrors()
    except Exception as e:  # noqa: BLE001
        logger.error("Error awaiting pending DB mirrors", error=str(e))

    # Stop FastMCP lifespan
    if mcp_lifespan_ctx:
        try:
            await mcp_lifespan_ctx.__aexit__(None, None, None)
            logger.info("FastMCP lifespan stopped")
        except Exception as e:
            logger.error("Error stopping FastMCP lifespan", error=str(e))

    # Gracefully shutdown OpenSearch connection
    try:
        from utils.opensearch_utils import graceful_opensearch_shutdown

        await graceful_opensearch_shutdown(clients.opensearch)
    except Exception as e:
        logger.error("Error during graceful OpenSearch shutdown", error=str(e))

    await cleanup_subscriptions_proper(services)
    # Cleanup task service (cancels background tasks and process pool)
    await services["task_service"].shutdown()
    # Cleanup async clients (this will also close OpenSearch client if not already closed)
    await clients.cleanup()

    # Dispose the SQL engine cleanly. Previously only the (dead) lifespan
    # path did this; the live shutdown_event leaked the engine.
    try:
        from db.engine import dispose_engine

        await dispose_engine()
    except Exception as e:
        logger.error("Error disposing DB engine", error=str(e))

    # Cleanup telemetry client
    from utils.telemetry.client import cleanup_telemetry_client

    await cleanup_telemetry_client()
    logger.info("Application shutdown completed")