Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
28.9 KB
Raw Blame
"""Provider health check endpoint."""

import asyncio
from typing import Any

import httpx
from fastapi import Depends
from fastapi.responses import JSONResponse

from api.provider_validation import (
    ProbeResult,
    sanitize_provider_error_content,
    validate_provider_setup,
)
from config.settings import get_openrag_config
from dependencies import require_permission
from services import provider_error_log
from services.model_catalog import is_known_provider
from session_manager import User
from utils import provider_health_cache
from utils.logging_config import get_logger

logger = get_logger(__name__)

#: `warnings[].code` for indexed vectors whose model the provider no longer serves.
STALE_EMBEDDING_SPACE = "stale_embedding_space"


#: Ceiling on re-listing a provider's models from inside a health check. The
#: listing is only an input to a diagnostic, so a slow cluster costs this at
#: most; the probes report an unreachable one in their own words.
_MODEL_REFRESH_TIMEOUT_SECONDS = 5.0


async def _refresh_live_models(provider: str) -> None:
    """Re-list what `provider` serves, if it can say, best effort.

    Only the provider being checked: the listing of any other would not be
    read, and waiting on its cluster would only slow this one's verdict. A
    provider with no enhancement, or one that cannot list its own models, has
    nothing to refresh, and is skipped without a call.

    TTL-guarded inside each enhancement, so this is a network call once every
    few minutes rather than once per poll — while the cluster answers. A failed
    listing is not cached, so an unreachable cluster is asked again on every
    poll; hence the timeout. A failure or timeout leaves the previous answer in
    place and must never fail the health check.
    """
    try:
        from enhancements.providers.registry import get as get_enhancement
        from services.model_catalog import refresh_live_models

        enhancement = get_enhancement(provider)
        if enhancement is None or not hasattr(enhancement, "fetch_models"):
            return
        await asyncio.wait_for(
            refresh_live_models(provider), timeout=_MODEL_REFRESH_TIMEOUT_SECONDS
        )
    except TimeoutError:
        logger.debug(
            "Timed out refreshing live model listing",
            provider=provider,
            timeout_seconds=_MODEL_REFRESH_TIMEOUT_SECONDS,
        )
    except Exception:
        logger.debug("Could not refresh live model listings", provider=provider, exc_info=True)


#: Spaces per aggregation page — the same page size retrieval discovers them
#: with. A corpus has a handful: one per embedding model it was indexed with.
_EMBEDDING_SPACE_PAGE_SIZE = 100

#: Pages read before giving up. Far past any real corpus; it bounds what one
#: health poll can cost if the index is somehow full of distinct spaces.
_EMBEDDING_SPACE_MAX_PAGES = 10


async def _indexed_spaces(provider: str) -> tuple[str, ...] | None:
    """Embedding models this corpus holds vectors for, under `provider`.

    None means unknown — OpenSearch unreachable, nothing indexed, or an answer
    that may be incomplete. A partial list is never returned as if it were the
    whole: a space missing from it is a stale space that goes unreported. So
    every page of the composite aggregation is read, shard failures raise
    rather than quietly dropping buckets, and running out of pages before the
    cursor does is treated as not knowing.

    Read with the admin client, not a user-scoped one, because this is a
    property of the corpus rather than of whoever is looking; DLS would
    silently shrink it.
    """
    try:
        from config.settings import clients, get_index_name
        from utils.embedding_fields import (
            build_embedding_space_aggregation,
            embedding_space_after_keys,
            embedding_spaces_from_aggregation,
            split_embedding_space_id,
        )

        client = clients.opensearch
        if client is None:
            return None
        space_ids: list[str] = []
        after: dict[str, Any] | None = None
        for _ in range(_EMBEDDING_SPACE_MAX_PAGES):
            result = await client.search(
                index=get_index_name(),
                body={
                    "size": 0,
                    # Qualified spaces only. A legacy space records no provider,
                    # so it cannot be attributed to this one, and it is governed
                    # by OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP rather than by what
                    # the cluster serves today.
                    "aggs": build_embedding_space_aggregation(
                        size=_EMBEDDING_SPACE_PAGE_SIZE,
                        qualified_after=after,
                        include_legacy=False,
                    ),
                },
                # A failed shard must fail the read, not shrink its buckets.
                params={"allow_partial_search_results": "false"},
            )
            if result.get("timed_out"):
                logger.debug("Reading indexed embedding spaces timed out")
                return None
            space_ids.extend(space.space_id for space in embedding_spaces_from_aggregation(result))
            # Raw buckets, not parsed spaces: the parser skips blank ids, and a
            # page it thinned out is not a short one.
            buckets = (
                result.get("aggregations", {}).get("embedding_spaces", {}).get("buckets") or []
            )
            next_after, _ = embedding_space_after_keys(result)
            # A short page is the last one; a composite aggregation still hands
            # back an `after_key` for it, and following it costs an empty read.
            if len(buckets) < _EMBEDDING_SPACE_PAGE_SIZE or not next_after or next_after == after:
                break
            after = next_after
        else:
            logger.debug(
                "Indexed embedding spaces exceed the read limit",
                pages=_EMBEDDING_SPACE_MAX_PAGES,
                page_size=_EMBEDDING_SPACE_PAGE_SIZE,
            )
            return None
    except Exception:
        logger.debug("Could not read indexed embedding spaces", exc_info=True)
        return None

    key = (provider or "").strip().lower()
    models = []
    for space_id in space_ids:
        space_provider, model = split_embedding_space_id(space_id)
        if space_provider == key and model:
            models.append(model)
    return tuple(dict.fromkeys(models)) or None


async def _stale_embedding_spaces(provider: str) -> dict[str, Any] | None:
    """Indexed vector spaces whose model the provider has stopped serving.

    Nothing re-checks a corpus once it is indexed. A chunk records the space it
    was embedded in (`provider:model`), retrieval embeds the query once per
    space it finds there, and an `InferenceService` redeployed under a new
    `--served-model-name` leaves every earlier chunk pointing at a model that
    is gone. No save can be rejected over it and no probe reaches it — the
    *configured* model is fine. Retrieval skips a space it cannot embed for and
    carries on, so nothing fails outright: those documents quietly stop being
    found by meaning, and the only trace is a 502 quoting a model name that
    appears nowhere in Settings.

    A warning, never an error: the provider is serving and nothing in its setup
    is wrong. The remedy is in the corpus — re-ingest or delete — so reporting
    it as a provider failure would send the operator to the one screen that
    cannot fix it.

    Silent unless both halves are known: what the provider serves, and what the
    corpus holds. Neither absence is evidence of the other. "Known" is
    `served is not None`: an empty listing is a provider that answered and
    serves no embedding model, which makes every indexed space stale.
    """
    try:
        from enhancements.providers.registry import live_models_for

        served = live_models_for(provider, "embedding")
    except Exception:
        logger.debug("Could not read live models for %s", provider, exc_info=True)
        return None
    if served is None:
        return None

    indexed = await _indexed_spaces(provider)
    if not indexed:
        return None
    stale = [model for model in indexed if model not in served]
    if not stale:
        return None
    serving = f"it serves: {', '.join(served)}" if served else "it lists no embedding models at all"
    # Debug, not warning: the banner polls this every 30s per open tab, and the
    # condition lasts until someone re-ingests. The response carries it to the
    # user; the log is for tracing why.
    logger.debug(
        "Indexed embedding spaces are no longer served by the provider",
        provider=provider,
        stale=stale,
        served=list(served),
    )
    return {
        "code": STALE_EMBEDDING_SPACE,
        "provider": provider,
        "models": stale,
        "served": list(served),
        "message": (
            f"Documents are indexed with {', '.join(repr(model) for model in stale)}, which "
            f"this endpoint no longer serves ({serving}). Search skips "
            "those vectors, so the documents are found by keyword only until they are "
            "re-ingested or deleted."
        ),
    }


async def check_provider_health(
    provider: str | None = None,
    test_completion: bool = False,
    model: str | None = None,
    embedding_model_override: str | None = None,
    user: User = Depends(require_permission("providers:read")),
):
    """
    Check if the configured provider is healthy and properly validated.

    Query parameters:
        provider (optional): Provider to check ('openai', 'ollama', 'watsonx', 'anthropic').
                           If not provided, checks the currently configured provider.
        test_completion (optional): If true, performs full validation with completion/embedding tests.
        model (optional): Validate against this chat model instead of the configured one.
                          Generic LiteLLM providers are validated by issuing a real call, which
                          needs a model name; a provider that is not the selected LLM/embedding
                          provider has none, and one whose model names are deployment-specific
                          (Azure, Bedrock, SageMaker) cannot be validated against the catalogue's
                          generic names. Only meaningful together with ``provider``.
        embedding_model_override (optional): Same, for validating an embedding model instead.

    Returns:
        200: Provider is healthy and validated
        400: Invalid provider specified
        503: Provider validation failed
    """
    check_provider = provider
    _health_leader_key: str | None = None  # set when this coroutine wins leader election
    try:
        # Get current config
        current_config = get_openrag_config()

        # Determine which provider to check
        if check_provider:
            provider = check_provider.lower()
        else:
            # Default to checking LLM provider
            provider = current_config.agent.llm_provider

        # Validate provider name
        if not is_known_provider(provider):
            return JSONResponse(
                {
                    "status": "error",
                    "message": f"Unknown LiteLLM provider: {provider}",
                    "provider": provider,
                },
                status_code=400,
            )

        # Get provider configuration
        if check_provider:
            # If checking a specific provider, use its configuration
            try:
                provider_config = current_config.providers.get_provider_config(provider)
                api_key = getattr(provider_config, "api_key", None)
                endpoint = getattr(provider_config, "endpoint", None)
                project_id = getattr(provider_config, "project_id", None)

                # Check if this provider is used for LLM or embedding
                llm_model = (
                    current_config.agent.llm_model
                    if provider == current_config.agent.llm_provider
                    else None
                )
                embedding_model = (
                    current_config.knowledge.embedding_model
                    if provider == current_config.knowledge.embedding_provider
                    else None
                )

                # An explicit model wins over whatever the provider happens to be
                # selected for. Setting one clears the other so the validator
                # tests exactly what the caller asked for rather than falling
                # back to an embedding call for a chat model (or vice versa).
                if model or embedding_model_override:
                    llm_model = model or None
                    embedding_model = embedding_model_override or None

                # One credential set per role, because the endpoint a provider
                # is checked against depends on which kind of call it is being
                # checked for (Red Hat OpenShift AI serves the two from
                # different endpoints). Each role is probed on its own below.
                role_credentials = {
                    kind: current_config.providers.credential_values(provider, kind=kind)
                    for kind in ("chat", "embedding")
                }
                # The untranslated form as well: a provider enhancement's
                # lightweight check needs every endpoint the operator entered,
                # and each entry above has been narrowed to one of them.
                stored_credentials = current_config.providers.stored_credentials(provider)
            except ValueError:
                # Provider not found in configuration
                return JSONResponse(
                    {
                        "status": "error",
                        "message": f"Cannot validate {provider} - not currently configured. Please configure it first.",
                        "provider": provider,
                    },
                    status_code=400,
                )
        else:
            # Check both LLM and embedding providers
            embedding_provider = current_config.knowledge.embedding_provider

            llm_provider_config = current_config.get_llm_provider_config()
            embedding_provider_config = current_config.get_embedding_provider_config()

            api_key = getattr(llm_provider_config, "api_key", None)
            endpoint = getattr(llm_provider_config, "endpoint", None)
            project_id = getattr(llm_provider_config, "project_id", None)
            llm_model = current_config.agent.llm_model

            embedding_api_key = getattr(embedding_provider_config, "api_key", None)
            embedding_endpoint = getattr(embedding_provider_config, "endpoint", None)
            embedding_project_id = getattr(embedding_provider_config, "project_id", None)
            embedding_model = current_config.knowledge.embedding_model
            credentials = current_config.providers.credential_values(provider, kind="chat")
            embedding_credentials = current_config.providers.credential_values(
                embedding_provider, kind="embedding"
            )
            stored_credentials = current_config.providers.stored_credentials(provider)
            embedding_stored_credentials = current_config.providers.stored_credentials(
                embedding_provider
            )

            # Short-circuit identical concurrent polls from the provider-health
            # banner so we don't fan out N watsonx round-trips per poll cycle.
            # Only the polled (no `check_provider`) success path is cached; the
            # 503 branch and the specific-provider branch always re-validate.
            health_cache_key = provider_health_cache.cache_key(
                provider=provider,
                embedding_provider=embedding_provider,
                test_completion=test_completion,
                credentials=credentials,
                llm_model=llm_model,
                embedding_model=embedding_model,
                endpoint=endpoint,
                project_id=project_id,
                api_key=api_key,
                embedding_api_key=embedding_api_key,
                embedding_endpoint=embedding_endpoint,
                embedding_project_id=embedding_project_id,
                embedding_credentials=embedding_credentials,
            )
            # A cached *healthy* verdict must not outlive a real failure. The
            # cache exists to coalesce identical probes, and a recorded failure
            # means traffic is failing right now regardless of what the last
            # probe concluded — so fall through and let the response below
            # report it.
            has_real_failure = bool(
                provider_error_log.latest_failure(provider, "chat")
                or provider_error_log.latest_failure(embedding_provider, "embedding")
            )
            cached_payload = provider_health_cache.get(health_cache_key)
            if cached_payload is not None and not has_real_failure:
                logger.debug("Returning cached provider-health response")
                return JSONResponse(cached_payload, status_code=200)

            # Singleflight: if another coroutine is already validating this
            # exact config, wait for it to finish rather than issuing a
            # redundant upstream call.
            while True:
                is_leader = await provider_health_cache.acquire(health_cache_key)
                if is_leader:
                    _health_leader_key = health_cache_key
                    break
                # Woke up after an in-flight validation completed.
                cached_payload = provider_health_cache.get(health_cache_key)
                if cached_payload is not None and not has_real_failure:
                    logger.debug("Returning cached provider-health response (waited for in-flight)")
                    return JSONResponse(cached_payload, status_code=200)
                # Leader's validation failed; retry leader election rather than
                # all waiters fanning out to validate simultaneously.

        logger.info(f"Checking health for provider: {provider}")

        # Validate provider setup
        if check_provider:
            # Validate specific provider
            # Generic LiteLLM providers keep their secrets in ``credentials``
            # rather than the dedicated api_key/endpoint/project_id fields, so
            # this must be forwarded or validating one from the providers page
            # runs with no credentials at all.
            #
            # One probe per role the provider is selected for. The validator
            # tests a single model per call, so a provider that is both the
            # LLM and the embedding provider needs two calls, each with the
            # credentials for that role — otherwise the chat model is never
            # checked and the response below claims it was. With no model at
            # all, one lightweight check runs.
            probes = [
                (kind, model_name)
                for kind, model_name in (("chat", llm_model), ("embedding", embedding_model))
                if model_name
            ] or [("chat", None)]
            for index, (kind, model_name) in enumerate(probes):
                # Same spacing the polled branch applies between the two
                # watsonx tests, so back-to-back calls don't trip its rate limit.
                if index and test_completion and provider == "watsonx":
                    logger.info("Waiting 2 seconds before WatsonX embedding test")
                    await asyncio.sleep(2)
                await validate_provider_setup(
                    provider=provider,
                    api_key=api_key,
                    embedding_model=model_name if kind == "embedding" else None,
                    llm_model=model_name if kind == "chat" else None,
                    endpoint=endpoint,
                    project_id=project_id,
                    test_completion=test_completion,
                    credentials=role_credentials[kind],
                    stored_credentials=stored_credentials,
                )

            return JSONResponse(
                {
                    "status": "healthy",
                    "message": "Properly configured and validated",
                    "provider": provider,
                    "details": {
                        "llm_model": llm_model,
                        "embedding_model": embedding_model,
                        "endpoint": endpoint if provider in ["ollama", "watsonx"] else None,
                    },
                },
                status_code=200,
            )
        else:
            # Validate both LLM and embedding providers
            # Note: For Ollama, we use lightweight checks that don't require model inference.
            # This prevents false-positive errors when Ollama is busy processing other requests.
            llm_error = None
            embedding_error = None

            # Validate LLM provider
            try:
                llm_probe: ProbeResult = await validate_provider_setup(
                    provider=provider,
                    api_key=api_key,
                    llm_model=llm_model,
                    endpoint=endpoint,
                    project_id=project_id,
                    test_completion=test_completion,
                    credentials=credentials,
                    stored_credentials=stored_credentials,
                )
            except httpx.TimeoutException as e:
                # Timeout means provider is busy, not misconfigured
                if provider == "ollama":
                    llm_error = None  # Don't treat as error
                    logger.info(f"LLM provider ({provider}) appears busy: {str(e)}")
                else:
                    llm_error = sanitize_provider_error_content(e)
                    logger.error(f"LLM provider ({provider}) validation timed out: {llm_error}")
            except Exception as e:
                llm_error = sanitize_provider_error_content(e)
                logger.error(f"LLM provider ({provider}) validation failed: {llm_error}")
            else:
                # Without this the banner can only be cleared by chat traffic:
                # one failed turn latches it, the frontend then polls every 5s
                # with `test_completion` while it stays latched, and a provider
                # that has recovered keeps being reported broken until the entry
                # goes stale 15 minutes later.
                #
                # But a probe cannot reproduce the request that failed (see
                # `provider_error_log`), so only one that sent the same shape
                # of request may speak for it. A passing validation can mean a
                # deployment listing (Azure), no call at all (no model set), or
                # a tool-less completion (the LiteLLM probe) — none of which
                # reaches the tool-calling failures agent traffic actually hits.
                # Clearing on those would drop the banner while chat is still
                # broken, only for the next turn to raise it again.
                if test_completion and llm_probe.model_probed and llm_probe.tools_exercised:
                    provider_error_log.record_success(provider, "chat")

            # Validate embedding provider
            # For WatsonX with test_completion=True, wait 2 seconds between completion and embedding tests
            if (
                test_completion
                and provider == "watsonx"
                and embedding_provider == "watsonx"
                and llm_error is None
            ):
                logger.info(
                    "Waiting 2 seconds before WatsonX embedding test (after completion test)"
                )
                await asyncio.sleep(2)

            try:
                embedding_probe: ProbeResult = await validate_provider_setup(
                    provider=embedding_provider,
                    api_key=embedding_api_key,
                    embedding_model=embedding_model,
                    endpoint=embedding_endpoint,
                    project_id=embedding_project_id,
                    test_completion=test_completion,
                    credentials=embedding_credentials,
                    stored_credentials=embedding_stored_credentials,
                )
            except httpx.TimeoutException as e:
                # Timeout means provider is busy, not misconfigured
                if embedding_provider == "ollama":
                    embedding_error = None  # Don't treat as error
                    logger.info(f"Embedding provider ({embedding_provider}) appears busy: {str(e)}")
                else:
                    embedding_error = sanitize_provider_error_content(e)
                    logger.error(
                        f"Embedding provider ({embedding_provider}) validation timed out: {embedding_error}"
                    )
            except Exception as e:
                embedding_error = sanitize_provider_error_content(e)
                logger.error(
                    f"Embedding provider ({embedding_provider}) validation failed: {embedding_error}"
                )
            else:
                # An embedding request has no shape beyond model and input, so a
                # real call to the configured model is the same request traffic
                # makes. Anything short of one proves nothing about it.
                if test_completion and embedding_probe.model_probed:
                    provider_error_log.record_success(embedding_provider, "embedding")

            # A real call beats a probe. The probe sends its own request, so it
            # hits its own failure: OpenAI checks request shape before billing,
            # which is how a probe can report "no credits remaining" while the
            # agent's own call reports a 400 about its parameters. Both are
            # true; the actionable one is the one the user's traffic produced.
            # An entry only exists while calls are still failing — the gateway
            # erases it on the next success.
            llm_error = provider_error_log.latest_failure(provider, "chat") or llm_error
            embedding_error = (
                provider_error_log.latest_failure(embedding_provider, "embedding")
                or embedding_error
            )

            # Nothing above looks at the corpus, and a vector space outlives
            # the model that made it. Reported beside the verdict, never in it:
            # a stale space degrades search but says nothing about whether the
            # provider is serving, so it must not turn the status code.
            await _refresh_live_models(embedding_provider)
            stale_spaces = await _stale_embedding_spaces(embedding_provider)
            warnings = [stale_spaces] if stale_spaces else []

            # Return combined status
            if llm_error or embedding_error:
                errors = []
                if llm_error:
                    errors.append(f"LLM ({provider}): {llm_error}")
                if embedding_error:
                    errors.append(f"Embedding ({embedding_provider}): {embedding_error}")

                if _health_leader_key:
                    provider_health_cache.release_error(_health_leader_key)
                    _health_leader_key = None
                return JSONResponse(
                    {
                        "status": "unhealthy",
                        "message": "; ".join(errors),
                        "llm_provider": provider,
                        "embedding_provider": embedding_provider,
                        "llm_error": llm_error,
                        "embedding_error": embedding_error,
                        "warnings": warnings,
                    },
                    status_code=503,
                )

            healthy_payload = {
                "status": "healthy",
                "message": "Both providers properly configured and validated",
                "llm_provider": provider,
                "embedding_provider": embedding_provider,
                "details": {
                    "llm_model": llm_model,
                    "embedding_model": embedding_model,
                },
                "warnings": warnings,
            }
            provider_health_cache.set_and_release(health_cache_key, healthy_payload)
            _health_leader_key = None
            return JSONResponse(healthy_payload, status_code=200)

    except asyncio.CancelledError:
        if _health_leader_key:
            provider_health_cache.release_error(_health_leader_key)
        raise
    except Exception as e:
        if _health_leader_key:
            provider_health_cache.release_error(_health_leader_key)
        error_message = sanitize_provider_error_content(e)
        logger.error(f"Provider health check failed for {provider}: {error_message}")

        return JSONResponse(
            {
                "status": "unhealthy",
                "message": error_message,
                "provider": provider,
            },
            status_code=503,
        )