Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
87.5 KB
Raw Blame
"""OpenAI-compatible LLM gateway backed by the LiteLLM SDK.

Langflow and other OpenAI clients call `/v1/chat/completions` and
`/v1/embeddings`. This module owns provider secrets (from OpenRAG config) and
routes by model prefix / configured provider. Callers never see upstream keys.
"""

from __future__ import annotations

import asyncio
import contextlib
import json
import math
import re
from collections import OrderedDict, deque
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
from itertools import pairwise
from typing import Any, Literal, NamedTuple
from uuid import uuid4

from observability.tracing import gateway_trace_kwargs, trace_context
from services import provider_error_log
from services.model_catalog import is_known_provider, litellm_provider_key
from utils.logging_config import get_logger

logger = get_logger(__name__)

_LITELLM_FORWARDED_PARAMS = (
    "tools",
    "tool_choice",
    "temperature",
    "top_p",
    "max_tokens",
    "max_completion_tokens",
    "stop",
    "presence_penalty",
    "frequency_penalty",
    "seed",
    "response_format",
    "parallel_tool_calls",
    "n",
    "user",
    "logit_bias",
    "stream_options",
)


class LlmGatewayError(Exception):
    """Gateway failure. `status_code` is an HTTP status.

    `message` is safe to return to the caller. `detail` carries the internal
    text (upstream exception type and body) and is for logs only — returning it
    is stack-trace exposure (CodeQL py/stack-trace-exposure). `code` is the
    OpenAI `error.code` for the failure, when it has been classified.
    """

    def __init__(
        self,
        message: str,
        status_code: int = 400,
        detail: str | None = None,
        code: str | None = None,
    ):
        """Store caller-safe text, HTTP status, internal details, and an optional code."""
        super().__init__(message)
        self.message = message
        self.status_code = status_code
        self.detail = detail or message
        self.code = code


CONTEXT_LENGTH_ERROR_CODE = "context_length_exceeded"


class LlmContextLengthError(LlmGatewayError):
    """The input does not fit the model's context window.

    The provider is serving and a shorter input succeeds, so a caller that can
    shorten or split what it sent catches this instead of the base class.
    `max_tokens` and `requested_tokens` are None when the provider's error did
    not state them.
    """

    def __init__(
        self,
        message: str,
        detail: str | None = None,
        *,
        max_tokens: int | None = None,
        requested_tokens: int | None = None,
    ):
        """Create a context-length 400 with the token counts the provider supplied."""
        super().__init__(message, 400, detail, code=CONTEXT_LENGTH_ERROR_CODE)
        self.max_tokens = max_tokens
        self.requested_tokens = requested_tokens


def _get_config():
    from config.settings import get_openrag_config

    return get_openrag_config()


#: Canonical separator between an OpenRAG provider tag and the model id.
#: `/` cannot serve as one: watsonx hosts `openai/gpt-oss-120b`, whose own name
#: begins with a provider key, so a `/`-split routes it to OpenAI and LiteLLM
#: then rejects the bare `gpt-oss-120b`. No catalogue id has a provider-shaped
#: prefix before a colon, so `provider:model` stays unambiguous even for ids
#: that use colons themselves (`ollama:gpt-oss:120b-cloud`, `openai:ft:gpt-4`).
PROVIDER_SEPARATOR = ":"
LEGACY_EMBEDDING_MODEL_PREFIX = "legacy:"
INDEXED_EMBEDDING_SPACE_PREFIX = "space:"
_BUILTIN_LEGACY_EMBEDDING_PROVIDERS = {
    "text-embedding-3-small": "openai",
}


def split_model_id(model: str) -> tuple[str | None, str]:
    """Split an OpenRAG provider tag off a model id.

    `provider:model` is the canonical form and is checked first. `provider/`
    is still accepted so ids stored before the switch keep resolving, but only
    when the remainder does not look like a provider-qualified name itself.
    """
    raw = (model or "").strip()

    prefix, sep, rest = raw.partition(PROVIDER_SEPARATOR)
    if sep and rest:
        prefix_lower = prefix.lower()
        if is_known_provider(prefix_lower):
            return prefix_lower, rest

    if "/" not in raw:
        return None, raw

    # A slash is ambiguous: it separates a legacy tag from its model, but it is
    # also part of vendor-qualified names. When the whole string names exactly
    # one catalogue model, that provider owns it — `openai/gpt-oss-120b` is
    # watsonx's model, not OpenAI's `gpt-oss-120b`.
    from services.model_catalog import catalog_owner, catalog_owners

    owner = catalog_owner(raw)
    if owner:
        return owner, raw

    if catalog_owners(raw):
        # More than one provider serves this exact id, so the text before the
        # slash is part of the model's own name and not a provider tag. A
        # deployment offering both watsonx.ai on IBM Cloud and on a cluster has
        # `openai/gpt-oss-120b` twice; splitting it would hand the id to OpenAI
        # — with OpenAI's key, for a model OpenAI does not serve. Leaving it
        # untagged sends it to the configured default provider whole, which is
        # right whenever that provider is one of the two and a clean "no such
        # model" when it is not.
        return None, raw

    prefix, rest = raw.split("/", 1)
    prefix_lower = prefix.lower()
    if is_known_provider(prefix_lower):
        return prefix_lower, rest
    return None, raw


def qualified_model_id(provider: str | None, model: str) -> str:
    """`provider:model` for an internal caller that already knows the provider.

    The inverse of `model_catalog.public_model_id`, which *strips* the tag for
    OpenAI because that is what an OpenAI-compatible client expects to see on
    `/v1/models`. An internal caller needs the opposite guarantee: an untagged
    id is resolved against whichever provider happens to be configured as the
    default (`resolve_call` -> `default_provider`), so ingesting with a model
    whose provider is *not* the default would silently call the wrong one.

    An explicit `provider` always wins, including over a slash-shaped prefix in
    `model` itself: watsonx serves `openai/gpt-oss-120b`, and leaving that
    untagged resolves it to OpenAI — with OpenAI's credentials, for a model
    OpenAI does not serve. `watsonx:openai/gpt-oss-120b` says which is meant.

    An id already tagged for this provider is returned as it is, in either
    form. The legacy `provider/` spelling matters: ids stored before the switch
    to `provider:` are still in saved configuration, and re-tagging one gives
    `azure:azure/prod-embed`, which `split_model_id` reduces to the model name
    `azure/prod-embed` — so LiteLLM is asked for `azure/azure/prod-embed`.

    Only the explicit provider's own prefix is removed. A slash elsewhere in
    the name is part of the name: watsonx serves `openai/gpt-oss-120b`, and
    `ibm/slate-125m` is one model, not a tag and a model.

    With no `provider`, the id is returned unchanged so the gateway falls back
    to the configured default, which is what an untagged call has always done.
    """
    name = (model or "").strip()
    key = (provider or "").strip().lower()
    if not name or not key:
        return name
    canonical = f"{key}{PROVIDER_SEPARATOR}"
    if name.lower().startswith(canonical):
        return name
    legacy = f"{key}/"
    if name.lower().startswith(legacy):
        name = name[len(legacy) :]
    return f"{canonical}{name}"


def default_provider(kind: Literal["chat", "embedding"], config=None) -> str:
    cfg = config or _get_config()
    if kind == "embedding":
        return (cfg.knowledge.embedding_provider or "openai").lower()
    return (cfg.agent.llm_provider or "openai").lower()


def default_model(kind: Literal["chat", "embedding"], config=None) -> str:
    cfg = config or _get_config()
    if kind == "embedding":
        return cfg.knowledge.embedding_model or ""
    return cfg.agent.llm_model or ""


def legacy_embedding_provider(model: str, config: Any | None = None) -> str | None:
    """Resolve provenance for a model-only embedding space from before provider tracking."""
    name = (model or "").strip()
    if name in _BUILTIN_LEGACY_EMBEDDING_PROVIDERS:
        return _BUILTIN_LEGACY_EMBEDDING_PROVIDERS[name]

    cfg = config or _get_config()
    mapping = getattr(cfg.knowledge, "legacy_embedding_provider_map", {}) or {}
    if not isinstance(mapping, Mapping):
        return None
    provider = str(mapping.get(name) or "").strip().lower()
    return provider or None


def provider_credentials(
    provider: str, config=None, *, kind: Literal["chat", "embedding"] = "chat"
) -> dict[str, Any]:
    """LiteLLM kwargs for any configured OpenRAG provider. Never logs secrets."""
    cfg = config or _get_config()
    key = (provider or "").strip().lower()
    try:
        prov = cfg.providers
    except Exception as exc:
        raise LlmGatewayError("LLM providers are not configured", 400) from exc

    if hasattr(prov, "credential_values"):
        credentials = prov.credential_values(key, kind=kind)
    else:
        provider_config = getattr(prov, key, None)
        if provider_config is None:
            credentials = {}
        elif key == "watsonx":
            credentials = {
                name: value
                for name, value in {
                    "api_key": getattr(provider_config, "api_key", None),
                    "api_base": getattr(provider_config, "endpoint", None),
                    "project_id": getattr(provider_config, "project_id", None),
                }.items()
                if value
            }
        elif key == "ollama":
            endpoint = getattr(provider_config, "resolved_endpoint", None) or getattr(
                provider_config, "endpoint", None
            )
            credentials = {"api_base": endpoint} if endpoint else {}
        else:
            api_key = getattr(provider_config, "api_key", None)
            credentials = {"api_key": api_key} if api_key else {}
    if key == "ollama" and credentials.get("api_base"):
        from utils.container_utils import transform_localhost_url

        credentials["api_base"] = transform_localhost_url(str(credentials["api_base"]))
    custom = getattr(prov, "custom", {})
    custom_config = custom.get(key) if isinstance(custom, dict) else None
    configured = bool(getattr(custom_config, "configured", False))
    if not credentials and not configured:
        raise LlmGatewayError(
            f"Provider {key!r} is not configured in OpenRAG. "
            "Configure it in Settings or pick a model from a configured provider.",
            400,
        )
    return credentials


def _provider_runtime_kwargs(provider: str, config=None) -> dict[str, Any]:
    """Non-serializable transport kwargs for the immediate LiteLLM invocation."""
    cfg = config or _get_config()
    prov = cfg.providers
    if not hasattr(prov, "credential_values"):
        return {}

    from enhancements.providers.registry import get, runtime_kwargs_for

    key = (provider or "").strip().lower()
    enhancement = get(key)
    if enhancement is None:
        return {}
    stored = (
        prov.stored_credentials(key)
        if hasattr(prov, "stored_credentials")
        else prov.credential_values(key)
    )
    return runtime_kwargs_for(enhancement, stored)


#: Lanes of a provider's embedding limit. `interactive` is query embedding —
#: search and chat retrieval, where a person is waiting; `bulk` is everything
#: else, chiefly ingestion. One slot of the limit is reserved for queries, so a
#: folder upload cannot queue them behind hundreds of chunks.
_BULK_LANE = "bulk"
_INTERACTIVE_LANE = "interactive"


def _lane_limit(limit: int, interactive: bool) -> tuple[str, int]:
    """The lane a call waits in and that lane's size, out of `limit` in total.

    The lanes split the operator's limit rather than add to it, so the endpoint
    never sees more than `limit` calls at once: `limit - 1` for bulk traffic
    and one for queries. A limit of 1 cannot be split without exceeding it, so
    both kinds of call then share the single slot.
    """
    if limit < 2:
        return _BULK_LANE, limit
    if interactive:
        return _INTERACTIVE_LANE, 1
    return _BULK_LANE, limit - 1


class _EmbeddingLimiter:
    """Bounds one provider's in-flight embedding calls under a limit that can change.

    A pair of per-lane semaphores cannot be resized: replacing them when the
    operator changes the limit forgets the calls still holding the old ones, so
    the endpoint briefly sees more than the new limit. This keeps one count per
    lane for the provider's lifetime and admits a call only while the total is
    under the current limit *and* its lane is under its share (`_lane_limit`).

    A freed slot is handed straight to the next waiter (as `asyncio.Semaphore`
    does), queries first, so neither a new arrival nor a bulk chunk can take a
    slot a waiting query is entitled to. `limit is None` means unbounded: calls
    are still counted, so turning a limit back on measures it against them.

    Not thread-safe; it lives on one event loop (see `_embedding_limiters`).
    """

    def __init__(self, provider: str, limit: int | None, loop: asyncio.AbstractEventLoop):
        self.provider = provider
        self.limit = limit
        self.loop = loop
        self._active = {_BULK_LANE: 0, _INTERACTIVE_LANE: 0}
        self._waiters: dict[str, deque[asyncio.Future[None]]] = {
            _BULK_LANE: deque(),
            _INTERACTIVE_LANE: deque(),
        }

    @property
    def in_flight(self) -> int:
        return sum(self._active.values())

    def _admits(self, lane: str) -> bool:
        if self.limit is None:
            return True
        if self.in_flight >= self.limit:
            return False
        _, lane_limit = _lane_limit(self.limit, lane == _INTERACTIVE_LANE)
        return self._active[lane] < lane_limit

    def _wake(self) -> None:
        """Grant free slots to waiters, queries before bulk, FIFO within a lane."""
        for lane in (_INTERACTIVE_LANE, _BULK_LANE):
            waiters = self._waiters[lane]
            while waiters and self._admits(lane):
                waiter = waiters.popleft()
                if waiter.done():  # cancelled while queued
                    continue
                self._active[lane] += 1
                waiter.set_result(None)

    async def acquire(self, lane: str, model: str = "") -> None:
        if not self._waiters[lane] and self._admits(lane):
            self._active[lane] += 1
            return
        waiter: asyncio.Future[None] = self.loop.create_future()
        self._waiters[lane].append(waiter)
        logger.debug(
            "Embedding call waiting for a free provider slot",
            provider=self.provider,
            model=model,
            lane=lane,
            max_concurrency=self.limit,
            in_flight=self.in_flight,
        )
        try:
            await waiter
        except asyncio.CancelledError:
            if waiter.done() and not waiter.cancelled():
                # Granted a slot in the same tick the caller was cancelled:
                # hand it on, or it is lost for the provider's lifetime.
                self.release(lane)
            else:
                with contextlib.suppress(ValueError):
                    self._waiters[lane].remove(waiter)
            raise

    def release(self, lane: str) -> None:
        if self._active[lane] <= 0:
            logger.error(
                "Embedding slot released more often than acquired; ignoring",
                provider=self.provider,
                lane=lane,
            )
            self._active[lane] = 0
        else:
            self._active[lane] -= 1
        self._wake()

    def resize(self, limit: int | None) -> None:
        """Apply a limit changed in Settings, counting every call still in flight."""
        if limit == self.limit:
            return
        previous = self.limit
        self.limit = limit
        logger.info(
            "Resizing embedding concurrency limit for provider",
            provider=self.provider,
            previous_max_concurrency=previous,
            max_concurrency=limit,
            bulk_in_flight=self._active[_BULK_LANE],
            interactive_in_flight=self._active[_INTERACTIVE_LANE],
        )
        if limit is not None and self.in_flight > limit:
            logger.info(
                "More embedding calls in flight than the new limit allows; "
                "new calls wait until they finish",
                provider=self.provider,
                max_concurrency=limit,
                in_flight=self.in_flight,
            )
        self._wake()

    @contextlib.asynccontextmanager
    async def slot(self, lane: str, model: str = "") -> AsyncIterator[None]:
        """Hold one of the provider's embedding slots for the duration of a call."""
        await self.acquire(lane, model)
        try:
            yield
        finally:
            self.release(lane)


#: Per-provider embedding bulkheads, keyed by provider key. The backend runs a
#: single worker (enforced in `app/lifespan.py`), so a process-local limiter
#: bounds the provider's traffic globally. Each limiter records its loop, and a
#: different running loop gets a new one, because asyncio futures must not be
#: shared across loops.
_embedding_limiters: dict[str, _EmbeddingLimiter] = {}


def _embedding_limiter(provider: str, config=None) -> _EmbeddingLimiter | None:
    """The limiter bounding embedding calls to `provider`, if any.

    The limit comes from the provider enhancement (`embedding_max_concurrency`),
    so providers without one — and every provider whose enhancement sets no
    limit — are unaffected. Excess callers wait here instead of queueing at the
    upstream, where a slow model server lets a fronting proxy time them out
    (RHOAI's kube-rbac-proxy answers 502 after 30s). Interactive calls wait in
    their own lane (`_lane_limit`). A limit changed in Settings takes effect on
    the next call and is measured against every call still in flight, so a
    lowered limit admits nothing new until enough of them finish.
    """
    cfg = config or _get_config()
    prov = getattr(cfg, "providers", None)
    if prov is None or not hasattr(prov, "credential_values"):
        return None

    from enhancements.providers.registry import embedding_concurrency_for, get

    key = (provider or "").strip().lower()
    if get(key) is None:
        return None
    stored = (
        prov.stored_credentials(key)
        if hasattr(prov, "stored_credentials")
        else prov.credential_values(key)
    )
    limit = embedding_concurrency_for(key, stored)
    loop = asyncio.get_running_loop()
    cached = _embedding_limiters.get(key)
    if cached is not None and cached.loop is loop:
        # Resized in place (also to "no limit"), so the calls it is already
        # counting stay counted against whatever limit applies next.
        cached.resize(limit)
        return cached
    if limit is None:
        _embedding_limiters.pop(key, None)
        return None

    limiter = _EmbeddingLimiter(key, limit, loop)
    _embedding_limiters[key] = limiter
    logger.info(
        "Bounding concurrent embedding calls for provider",
        provider=key,
        max_concurrency=limit,
        bulk_max_concurrency=_lane_limit(limit, False)[1],
        interactive_max_concurrency=_lane_limit(limit, True)[1],
    )
    return limiter


@contextlib.asynccontextmanager
async def _embedding_slot(
    limiter: _EmbeddingLimiter | None, provider: str, model: str, lane: str = _BULK_LANE
) -> AsyncIterator[None]:
    """Hold one of the provider's embedding slots for the duration of a call."""
    if limiter is None:
        yield
        return
    async with limiter.slot(lane, model):
        yield


def resolve_call(
    model: str | None,
    *,
    kind: Literal["chat", "embedding"],
    config=None,
) -> tuple[str, str, dict[str, Any]]:
    """Return `(litellm_model, provider, credentials)` for a request model id."""
    cfg = config or _get_config()
    requested = (model or "").strip() or default_model(kind, cfg)
    if not requested:
        raise LlmGatewayError("model is required", 400)

    if kind == "embedding" and requested.startswith(INDEXED_EMBEDDING_SPACE_PREFIX):
        space_id = requested[len(INDEXED_EMBEDDING_SPACE_PREFIX) :].strip()
        provider, separator, name = space_id.partition(PROVIDER_SEPARATOR)
        provider = provider.strip().lower()
        name = name.strip()
        if not separator or not provider or not name:
            raise LlmGatewayError(
                f"Indexed embedding space {space_id!r} has no exact provider route.",
                400,
            )
    elif kind == "embedding" and requested.startswith(LEGACY_EMBEDDING_MODEL_PREFIX):
        name = requested[len(LEGACY_EMBEDDING_MODEL_PREFIX) :].strip()
        provider = legacy_embedding_provider(name, cfg)
        if not provider:
            raise LlmGatewayError(
                f"Legacy embedding model {name!r} has no provider mapping. "
                "Set knowledge.legacy_embedding_provider_map for this model.",
                400,
            )
    else:
        provider, name = split_model_id(requested)
    if provider is None:
        provider = default_provider(kind, cfg)
        name = requested
    credentials = provider_credentials(provider, cfg, kind=kind)
    # An OpenRAG provider that LiteLLM does not know by that name is routed
    # under the key it aliases (`watsonx_onprem` -> `watsonx`). The OpenRAG key
    # is still what the caller sees and what credentials are stored under.
    #
    # The stored (untranslated) form is passed because one provider's transport
    # depends on how it was configured: Azure AI Foundry routes as `azure_ai`
    # on its legacy `/models` endpoint and as an OpenAI-compatible transport on
    # `/openai/v1`, which LiteLLM's `azure_ai` handler cannot address.
    try:
        route = litellm_provider_key(provider, _stored_credentials(provider, cfg))
    except ValueError as exc:
        # The provider knows its own endpoint shapes and this is not one of
        # them. Surfaced here rather than routed on a guess, which would reach
        # the wrong handler and fail as an opaque upstream error.
        raise LlmGatewayError(str(exc), 400) from exc
    litellm_model = f"{route}/{name}" if route != "openai" else name
    _warn_on_unexpected_route(provider, litellm_model, route)
    return litellm_model, provider, credentials


def _stored_credentials(provider: str, config) -> dict[str, Any]:
    """A provider's credentials as the operator entered them, or `{}`.

    Only routing reads this here; the call itself uses the translated form.

    `{}` means "this configuration does not keep the untranslated form", which
    is true of some test doubles and of any config object predating it. It does
    *not* swallow a failure to read: an empty mapping sends
    `litellm_provider_key` to the static alias, and for a provider whose
    transport depends on its endpoint that is a silent wrong route — the opaque
    upstream 404 this module exists to prevent. A real error belongs in the
    caller's face instead.
    """
    providers = getattr(config, "providers", None)
    reader = getattr(providers, "stored_credentials", None)
    if reader is None:
        return {}
    return dict(reader(provider))


#: Model ids `_warn_on_unexpected_route` has already checked, so the resolver
#: runs once per distinct id rather than on every request, and a mismatch is
#: logged once rather than repeatedly.
#:
#: Bounded, and an LRU rather than a set: the key contains the model id, which
#: on `/v1/chat/completions` comes straight from the request body, so an
#: unbounded memo is a cache an unauthenticated caller can grow. Evicting the
#: oldest entry costs at worst a repeated warning for a model nobody has used
#: in the last `_ROUTE_CHECK_MEMO_MAX` distinct ids.
_ROUTE_CHECK_MEMO_MAX = 256
_CHECKED_ROUTES: OrderedDict[tuple[str, str], None] = OrderedDict()


def _warn_on_unexpected_route(provider: str, litellm_model: str, intended_route: str) -> None:
    """Warn when LiteLLM resolves a model to a different provider than intended.

    Not a check that the route differs from the OpenRAG provider key — for
    Foundry on `/openai/v1` it deliberately does, and comparing against
    `azure_ai` would fire on every healthy request. The comparison is against
    the route this gateway *asked* for, so what it catches is LiteLLM
    re-deciding underneath us.

    That is not hypothetical: litellm 1.84.0 silently rerouted
    `azure_ai/gpt-4o` and other OpenAI-family names to the `azure` provider, so
    a Foundry hostname was called with the Azure OpenAI handler and failed as
    an opaque 404. The floor this repo pins no longer does it; this makes a
    return visible instead of mysterious.
    """
    if not litellm_model.startswith(f"{intended_route}/"):
        # Nothing was asserted, so there is nothing to check. This is the bare
        # name form that `openai` alone uses: the id goes out unprefixed, so
        # LiteLLM resolves it from the name and is entitled to its own answer.
        # A model whose *name* carries a vendor segment — `meta-llama/...` on
        # an OpenAI-compatible server — lands here too, and would otherwise
        # warn on every healthy request. A prefixed id is still checked even
        # when the route equals the provider key: `azure_ai/gpt-4o` silently
        # becoming `azure` is the regression this guard was written for.
        return
    key = (provider, litellm_model)
    if key in _CHECKED_ROUTES:
        _CHECKED_ROUTES.move_to_end(key)
        return
    _CHECKED_ROUTES[key] = None
    while len(_CHECKED_ROUTES) > _ROUTE_CHECK_MEMO_MAX:
        _CHECKED_ROUTES.popitem(last=False)
    try:
        from litellm import get_llm_provider

        _model, resolved, _key, _base = get_llm_provider(model=litellm_model)
    except Exception:
        return
    if resolved == intended_route:
        return
    logger.warning(
        "LiteLLM resolved this model to a different provider than the gateway "
        "selected; the request will be built by that provider's handler",
        provider=provider,
        model=litellm_model,
        intended_route=intended_route,
        resolved_route=resolved,
    )


def forget_route_mismatches() -> None:
    """Clear the once-per-route warning memo. For tests."""
    _CHECKED_ROUTES.clear()


_UPSTREAM_FAILURE_MESSAGE = "The model provider could not be reached. Please try again."
_UPSTREAM_CREDENTIAL_MESSAGE = (
    "The configured API key is invalid or has been revoked. Update it in Settings and retry."
)
#: A TLS trust failure reads as an outage otherwise — the request never left the
#: process, so there is no provider error to quote and the text collapses to
#: "could not be reached", which sends an operator hunting for a network fault.
#: Nothing in Settings fixes this, so the message says who has to act.
_UPSTREAM_TLS_MESSAGE = (
    "The provider's TLS certificate is not trusted by this deployment. An operator needs to "
    "add the provider's CA certificate to OpenRAG's trust store."
)
#: An upstream 502/504 with no body to quote — typically a proxy in front of
#: the model server (RHOAI's kube-rbac-proxy) giving up while the server is
#: still busy. LiteLLM then reports only its own wrapper names
#: ("BadGatewayError: Hosted_vllmException -"), which reads like a bug.
_UPSTREAM_GATEWAY_MESSAGE = (
    "The model server did not respond in time or is unavailable (upstream 502/504). It may "
    "be overloaded: check its capacity and any proxy timeout in front of it, or lower the "
    "provider's max concurrent requests."
)
#: The connection closed before the response finished. A proxy or load balancer
#: in front of the model server timing out a long response, not the model
#: rejecting anything — see `is_provider_truncated_response_error`.
_UPSTREAM_TRUNCATED_MESSAGE = (
    "The connection to the model server closed before the response finished. A proxy or "
    "load balancer in front of it likely timed out a long response: raise that timeout, "
    "or point the provider at an endpoint that bypasses it."
)
_GATEWAY_FAILURE_STATUSES = frozenset({502, 504})
#: LiteLLM exception class names that wrap an upstream error, e.g.
#: "litellm.BadGatewayError: BadGatewayError: Hosted_vllmException -".
_EXCEPTION_WRAPPER_RE = re.compile(r"(?:litellm\.)?[A-Za-z_]+(?:Error|Exception)\b\s*[:\-]?\s*")


def _is_opaque_gateway_failure(upstream: str, exc: BaseException | None) -> bool:
    """True for an upstream 502/504 whose text is nothing but exception wrappers."""
    status = getattr(exc, "status_code", None)
    if not isinstance(status, int) or status not in _GATEWAY_FAILURE_STATUSES:
        return False
    remainder = _EXCEPTION_WRAPPER_RE.sub("", upstream or "").strip(" -:.")
    return not remainder


#: Longest upstream explanation we will echo. Provider bodies can embed a whole
#: model catalogue; past this the message stops being readable in a chat bubble.
_MAX_UPSTREAM_MESSAGE_CHARS = 600

#: Lines LiteLLM appends to its own exceptions that say nothing about the failure.
_UPSTREAM_NOISE_MARKERS = (
    "Give Feedback / Get Help",
    "LiteLLM.Info:",
    "For more information check:",
    "For further information visit",
    "During handling of the above exception",
)

#: Everything from here on is interpreter state, not provider explanation.
_TRACEBACK_MARKERS = ("Traceback (most recent call last)", 'File "')

_FILE_PATH_PATTERN = re.compile(r"(?:/[\w.\-]+)+\.py")
#: Loopback and RFC1918 addresses — deployment topology, not provider explanation.
_PRIVATE_HOST_PATTERN = re.compile(
    r"\b(?:127(?:\.\d{1,3}){3}|10(?:\.\d{1,3}){3}|192\.168(?:\.\d{1,3}){2}"
    r"|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|localhost)\b"
)
_ANSI_PATTERN = re.compile(r"\x1b\[[0-9;]*m")


def _label_route(provider: str) -> str:
    """The transport `provider` resolves to, for display only.

    Never raises. This feeds error messages, and a provider misconfigured badly
    enough to fail here is already on its way to an error the caller needs to
    read — losing the label's prefix-stripping is a cosmetic degradation, while
    raising would replace the real failure with this one.
    """
    try:
        return litellm_provider_key(provider, _stored_credentials(provider, _get_config()))
    except Exception:
        try:
            return litellm_provider_key(provider)
        except Exception:
            return ""


def _call_label(provider: str, model: str, route: str = "") -> str:
    """`provider/model` for humans, without repeating a prefix LiteLLM already added.

    The model here is the routed id, so for an aliased provider it carries the
    *LiteLLM* prefix rather than the OpenRAG one — naively joining them gives
    "watsonx_onprem/watsonx/openai/gpt-oss-120b", which names two providers and
    reads like a bug in the error it appears in.
    """
    # `route` is passed by callers that already have it. The rest resolve it
    # the same way `resolve_call` did, from the stored configuration: the
    # static alias answers `azure_ai` for a Foundry endpoint routed over
    # `hosted_vllm`, which leaves the transport prefix on and produces exactly
    # the two-provider label this function exists to avoid.
    route = route or (_label_route(provider) if provider else "")
    if route and route != provider and model.startswith(f"{route}/"):
        model = model[len(route) + 1 :]
    if model and provider and not model.startswith(f"{provider}/"):
        return f"{provider}/{model}"
    return model or provider or "provider"


def _json_bodies(text: str):
    """Yield every JSON object embedded in `text`, outermost first."""
    decoder = json.JSONDecoder()
    index = text.find("{")
    while index != -1:
        try:
            body, end = decoder.raw_decode(text, index)
        except ValueError:
            index = text.find("{", index + 1)
            continue
        yield body
        index = text.find("{", end)


def _provider_message(body: Any) -> str | None:
    """The provider's own explanation out of one parsed error body."""
    if not isinstance(body, dict):
        return None
    error = body.get("error")
    if isinstance(error, dict) and isinstance(error.get("message"), str):
        return error["message"]
    if isinstance(error, str) and error.strip():
        return error
    errors = body.get("errors")
    if isinstance(errors, list):
        parts = [
            item["message"]
            for item in errors
            if isinstance(item, dict) and isinstance(item.get("message"), str)
        ]
        if parts:
            return "; ".join(parts)
    for key in ("message", "detail"):
        value = body.get(key)
        if isinstance(value, str) and value.strip():
            return value
    return None


def _is_provider_attributable(exc: BaseException | None) -> bool:
    """True when the failure came from the provider call, not from our own code.

    Anything LiteLLM raises describes the upstream request — it tags its
    exceptions with `llm_provider`, and its transport wrappers live under the
    `litellm` package. A `RuntimeError` from OpenRAG code does not, and its text
    is ours to keep in the logs.
    """
    if exc is None:
        return False
    if getattr(exc, "llm_provider", None):
        return True
    return type(exc).__module__.split(".")[0] == "litellm"


def _sanitise_upstream_detail(detail: str) -> str:
    """Strip interpreter state out of an upstream failure, keep the explanation.

    Traceback frames, local file paths, private addresses and LiteLLM's own
    footer chatter are removed; what is left is collapsed and capped so it fits
    a chat bubble. Credentials are already redacted by `_redact` before this
    runs.
    """
    text = _ANSI_PATTERN.sub("", detail or "")
    for marker in _TRACEBACK_MARKERS:
        cut = text.find(marker)
        if cut != -1:
            text = text[:cut]
    text = "\n".join(
        line
        for line in text.splitlines()
        if not any(marker in line for marker in _UPSTREAM_NOISE_MARKERS)
    )
    text = _FILE_PATH_PATTERN.sub("<path>", text)
    text = _PRIVATE_HOST_PATTERN.sub("<host>", text)
    text = " ".join(text.split())
    if len(text) > _MAX_UPSTREAM_MESSAGE_CHARS:
        text = text[: _MAX_UPSTREAM_MESSAGE_CHARS - 1].rstrip() + "\u2026"
    return text


def _provider_error_text(detail: str, exc: BaseException | None) -> str | None:
    """The provider's own words for a failure, or None when we cannot attribute it.

    Providers explain themselves precisely — "Model 'x' is not supported for
    this environment", "api_key is invalid" — and collapsing that into a fixed
    literal leaves an operator with nothing to act on. But only text we can tie
    to the upstream call earns a trip to the client: a JSON error body the
    provider sent, or a LiteLLM exception, which always describes the request it
    made. Anything else stays in the logs.
    """
    for body in _json_bodies(detail or ""):
        message = _provider_message(body)
        if message:
            return message
    if _is_provider_attributable(exc):
        return detail
    return None


#: Upstream statuses worth passing through: a client can act on these itself.
#: Everything else becomes 502 — an upstream 401/403 must not read as an OpenRAG
#: auth failure, and an upstream 400 must not read as a bad request to us. The
#: one exception is an input that overflows the context window, which *is* a bad
#: request to us; see `_classify_upstream_failure`.
_PASSTHROUGH_UPSTREAM_STATUSES = frozenset({408, 429, 503, 504})


def _upstream_client_message(
    detail: str, provider: str = "", model: str = "", exc: BaseException | None = None
) -> str:
    """Client-facing text for an upstream failure, naming the provider and cause.

    Nothing is forwarded raw. `_provider_error_text` decides whether the failure
    is the provider's to explain, and `_sanitise_upstream_detail` removes
    traceback frames, paths and private addresses from what it returns — so the
    caller sees the provider's own wording and which provider/model produced it,
    and an unattributable failure still collapses to a fixed literal. A
    credential failure keeps its actionable message so onboarding can tell the
    user to fix the key.
    """
    from api.provider_validation import (
        is_generic_upstream_error,
        is_provider_credential_error,
        is_provider_tls_error,
        is_provider_truncated_response_error,
    )

    label = _call_label(provider, model)
    # Checked before the credential branch: a TLS failure often carries
    # "unauthorized"-adjacent wording from the transport wrapper, and telling
    # someone to rotate a working API key wastes the trip.
    if is_provider_tls_error(detail):
        return f"{_UPSTREAM_TLS_MESSAGE} ({label})"
    # Also ahead of the credential branch: aiohttp reports the cut-off with a
    # "400" of its own, and LiteLLM's wrapper chain around it says nothing an
    # operator can act on.
    if is_provider_truncated_response_error(detail):
        return f"{_UPSTREAM_TRUNCATED_MESSAGE} ({label})"
    if is_provider_credential_error(detail):
        return f"{_UPSTREAM_CREDENTIAL_MESSAGE} ({label})"
    upstream = _provider_error_text(detail, exc)
    upstream = _sanitise_upstream_detail(upstream) if upstream else ""
    if _is_opaque_gateway_failure(upstream, exc):
        return f"{_UPSTREAM_GATEWAY_MESSAGE} ({label})"
    if not upstream or is_generic_upstream_error(upstream):
        return f"{_UPSTREAM_FAILURE_MESSAGE} ({label})"
    # lgtm[py/stack-trace-exposure] — provider error text only; traceback frames,
    # file paths and private hosts are removed by _sanitise_upstream_detail.
    return f"{label}: {upstream}"


def _is_request_scoped_failure(detail: str, exc: BaseException | None) -> bool:
    """True when the failure belongs to this request, not to the provider.

    A prompt that overflows the context window fails because of what this one
    call carried; the next, shorter one succeeds. Recording it in
    `provider_error_log` would latch the health banner against a provider that
    is serving fine — and on a provider whose probe cannot clear it, until the
    entry goes stale. The caller still gets the error.
    """
    from api.provider_validation import is_context_window_error

    if exc is not None and any(
        cls.__name__ == "ContextWindowExceededError" for cls in type(exc).__mro__
    ):
        return True
    return is_context_window_error(detail)


#: Exception classes, anywhere in the MRO, that mean the provider was never
#: reached. LiteLLM gives its own APIConnectionError a 500, so on the status
#: alone it would read as the provider failing.
_UNREACHABLE_EXCEPTION_NAMES = frozenset(
    {"APIConnectionError", "ConnectionError", "ConnectError", "ClientConnectionError"}
)


class _UpstreamFailure(NamedTuple):
    """How one failed upstream call is reported to the caller and to the banner."""

    code: str
    status_code: int
    counts_against_provider: bool


def _classify_upstream_failure(detail: str, exc: BaseException | None) -> _UpstreamFailure:
    """Name a failed upstream call: its `error.code`, its HTTP status, its blame.

    The status keeps the passthrough rule above. The code says what the status
    cannot: a 502 covers a rejected key, a model that does not exist and a
    provider that is down, and a client deciding whether to retry needs to know
    which. Only an input that overflows the context window is the caller's own
    doing, so only that becomes a 400 and stays out of the provider's health.
    """
    from api.provider_validation import (
        is_provider_credential_error,
        is_provider_tls_error,
        is_provider_truncated_response_error,
    )

    if _is_request_scoped_failure(detail, exc):
        return _UpstreamFailure(CONTEXT_LENGTH_ERROR_CODE, 400, counts_against_provider=False)

    status = getattr(exc, "status_code", None)
    if not isinstance(status, int):
        status = None
    # Same precedence as `_upstream_client_message`, so the code and the text
    # never describe two different failures.
    if is_provider_tls_error(detail) or is_provider_truncated_response_error(detail):
        code = "upstream_unreachable"
    elif status in (401, 403) or is_provider_credential_error(detail):
        code = "upstream_auth_error"
    elif status == 429:
        code = "rate_limit_exceeded"
    elif status in (408, 504):
        code = "upstream_timeout"
    elif exc is not None and any(
        cls.__name__ in _UNREACHABLE_EXCEPTION_NAMES for cls in type(exc).__mro__
    ):
        code = "upstream_unreachable"
    elif status in (500, 502, 503):
        code = "upstream_unavailable"
    elif status is not None and 400 <= status < 500:
        code = "upstream_invalid_request"
    else:
        code = "upstream_error"
    http_status = status if status in _PASSTHROUGH_UPSTREAM_STATUSES else 502
    return _UpstreamFailure(code, http_status, counts_against_provider=True)


def _upstream_gateway_error(
    message: str, detail: str, failure: _UpstreamFailure
) -> LlmGatewayError:
    """The exception a classified upstream failure is raised as."""
    if failure.code == CONTEXT_LENGTH_ERROR_CODE:
        from api.provider_validation import context_window_token_counts

        max_tokens, requested_tokens = context_window_token_counts(detail)
        return LlmContextLengthError(
            message, detail, max_tokens=max_tokens, requested_tokens=requested_tokens
        )
    return LlmGatewayError(message, failure.status_code, detail=detail, code=failure.code)


def _redact(message: str, credentials: Mapping[str, Any]) -> str:
    redacted = message
    for value in credentials.values():
        if value and isinstance(value, str) and len(value) > 3:
            redacted = redacted.replace(value, "[redacted]")
    return redacted


def _to_openai_dict(response: Any) -> dict[str, Any]:
    if isinstance(response, dict):
        return response
    if hasattr(response, "model_dump"):
        dumped = response.model_dump()
        if isinstance(dumped, dict):
            return dumped
    if hasattr(response, "json"):
        payload = response.json()
        if isinstance(payload, str):
            return json.loads(payload)
        if isinstance(payload, dict):
            return payload
    return dict(response)


def _chunk_payload(chunk: Any) -> str:
    if hasattr(chunk, "model_dump_json"):
        return chunk.model_dump_json()
    if hasattr(chunk, "json"):
        payload = chunk.json()
        return payload if isinstance(payload, str) else json.dumps(payload)
    if isinstance(chunk, dict):
        return json.dumps(chunk)
    return json.dumps({"data": str(chunk)})


#: A payload wrapped more times than this is not a serialisation slip.
_MAX_ARGUMENT_UNWRAPS = 3


def _normalise_tool_arguments(value: Any) -> tuple[Any, bool]:
    """Return `(arguments, repaired)`, unwrapping arguments serialised twice.

    OpenAI's contract is that `function.arguments` is a string holding a JSON
    *object*. Some models serialise the object and then serialise that string
    again, sending `'"{\\"query\\": \\"x\\"}"'` where `'{"query": "x"}'` was
    meant (watsonx `ibm/granite-4-h-small` does this; its stablemates on the
    same deployment do not). Clients parse `arguments` exactly once, so they get
    a `str` where a mapping is required: langchain-core rejects the tool call,
    the agent sees no callable tool, and the run ends with no content at all.
    Unwrapping is keyed on the payload opening with a quote, which a real
    arguments object never does, so this is a no-op for well-behaved providers.
    """
    if isinstance(value, (dict, list)):
        return json.dumps(value), True
    if not isinstance(value, str) or not value.lstrip().startswith('"'):
        return value, False
    candidate = value
    for _ in range(_MAX_ARGUMENT_UNWRAPS):
        try:
            decoded = json.loads(candidate)
        except ValueError:
            return value, False
        if not isinstance(decoded, str):
            return value, False
        candidate = decoded
        try:
            inner = json.loads(candidate)
        except ValueError:
            continue
        if isinstance(inner, (dict, list)):
            return candidate, True
    return value, False


def _repair_tool_calls(tool_calls: Any, provider: str, model: str) -> int:
    """Normalise `arguments` on every tool call in place. Returns how many changed."""
    repaired = 0
    for call in tool_calls or []:
        if not isinstance(call, dict):
            continue
        function = call.get("function")
        if not isinstance(function, dict):
            continue
        arguments, changed = _normalise_tool_arguments(function.get("arguments"))
        if changed:
            function["arguments"] = arguments
            repaired += 1
    if repaired:
        logger.warning(
            "Repaired double-encoded tool call arguments",
            provider=provider,
            model=model,
            tool_calls=repaired,
        )
    return repaired


def _finalise_tool_call(call: dict[str, Any]) -> bool:
    """Make `call` something a client can execute. False when it cannot be.

    A call with no name names no tool, so nothing can run it and no result can
    ever come back for it; forwarding it only corrupts the assistant message the
    client will replay on its next turn. A missing `id` is recoverable — the id
    is only the handle a tool result is correlated by, so any stable string
    serves — and minting one keeps a real call alive that clients would
    otherwise reject for carrying `id: null`. Blank `arguments` are recoverable
    too: a named call streamed with no arguments fragment is a zero-argument
    call, and the contract is a JSON *object* — `""` does not parse as one.
    """
    function = call.get("function")
    if not isinstance(function, dict) or not function.get("name"):
        return False
    if not isinstance(call.get("id"), str) or not call["id"]:
        call["id"] = f"call_{uuid4().hex}"
    arguments = function.get("arguments")
    if arguments is None or (isinstance(arguments, str) and not arguments.strip()):
        function["arguments"] = "{}"
        logger.debug(
            "Defaulted blank tool call arguments to an empty object", tool=function["name"]
        )
    return True


def _sanitise_messages(messages: Any) -> tuple[list[Any], int]:
    """`messages` with unusable tool calls repaired or dropped, and a repair count.

    The gateway forwards a client's conversation verbatim, and one malformed
    tool call in it is not a one-turn problem. Clients replay their whole
    history on every turn, so the same bad message goes back up again and again,
    and providers reject the *request* rather than the message that spoiled it —
    vLLM answers the lot with "Please ensure `tool_calls` are iterable of tool
    calls". The conversation is then wedged for good, with nothing in the UI
    pointing at the turn that broke it. A proxy sitting between many clients and
    many providers is the right place to stop that.

    Three shapes are handled, all of them things real clients emit: `arguments`
    sent as an object rather than the JSON *string* OpenAI's contract requires;
    a tool call carrying no id or no name, which nothing can execute and no
    result can be matched to; and the tool results left behind by one, which are
    dropped alongside it so no provider is handed a reply to a call that is no
    longer there.

    What survives must still be a sequence providers accept: a `tool` message
    is kept only when it answers a call kept on the assistant message just
    before it, and an assistant message left with neither tool calls nor
    content is dropped, since `content` is only optional beside `tool_calls`.
    """
    cleaned: list[Any] = []
    repairs = 0
    # Ids of the calls kept on the most recent assistant message: the only ones
    # a `tool` message may answer.
    live_ids: set[str] = set()
    for message in messages or []:
        if not isinstance(message, dict):
            cleaned.append(message)
            continue
        if message.get("role") == "tool":
            if message.get("tool_call_id") not in live_ids:
                repairs += 1
                continue
            cleaned.append(message)
            continue
        live_ids = set()
        if "tool_calls" not in message:
            cleaned.append(message)
            continue

        raw = message.get("tool_calls")
        kept: list[dict[str, Any]] = []
        for call in raw if isinstance(raw, list) else []:
            if not isinstance(call, dict):
                repairs += 1
                continue
            # The client controls this value, and `dict()` raises on a string or
            # a number. A non-mapping names no tool, so it becomes an empty one
            # and the call is dropped below like any other nameless call.
            function = call.get("function")
            call = {**call, "function": dict(function) if isinstance(function, dict) else {}}
            arguments, changed = _normalise_tool_arguments(call["function"].get("arguments"))
            if changed:
                call["function"]["arguments"] = arguments
                repairs += 1
            # Unlike the response path, a missing id is not minted here: the
            # result for this call is already somewhere in the history under the
            # id the client used, and inventing a different one would orphan it.
            if isinstance(call.get("id"), str) and call["id"] and call["function"].get("name"):
                kept.append(call)
            else:
                repairs += 1

        message = dict(message)
        if kept:
            message["tool_calls"] = kept
            live_ids = {call["id"] for call in kept}
        else:
            # `tool_calls: null` is itself a shape some providers iterate
            # without a None check, so the key goes rather than emptying.
            message.pop("tool_calls")
            # A list's entries were each counted above; anything else was
            # never iterable in the first place and is one repair on its own.
            repairs += bool(raw) and not isinstance(raw, list)
            if raw and message.get("role") == "assistant" and not message.get("content"):
                # The dropped calls were the whole message; what is left has
                # nothing to say, and no provider accepts it saying nothing.
                continue
        cleaned.append(message)
    return cleaned, repairs


def _repair_completion_payload(payload: dict[str, Any], provider: str, model: str) -> int:
    repaired = 0
    for choice in payload.get("choices") or []:
        if not isinstance(choice, dict):
            continue
        message = choice.get("message")
        if isinstance(message, dict):
            repaired += _repair_tool_calls(message.get("tool_calls"), provider, model)
    return repaired


def _log_completion_shape(payload: dict[str, Any], provider: str, model: str) -> None:
    """Record the shape of a non-streaming completion.

    An empty completion is indistinguishable from a transport failure once it
    reaches Langflow: the agent simply ends its loop and the user sees "The
    server didn't return a response". Only metadata is logged — never message
    content — so this stays safe to leave on.
    """
    try:
        choices = payload.get("choices") or []
        first = choices[0] if choices else {}
        message = first.get("message") or {}
        content = message.get("content") or ""
        tool_calls = message.get("tool_calls") or []
        finish_reason = first.get("finish_reason")
        empty = not str(content).strip() and not tool_calls
        log = logger.warning if empty else logger.info
        log(
            "LLM chat completion returned no content and no tool calls"
            if empty
            else "LLM chat completion",
            provider=provider,
            model=model,
            finish_reason=finish_reason,
            content_chars=len(str(content)),
            tool_calls=len(tool_calls),
            choices=len(choices),
        )
    except Exception:  # diagnostics must never break a response
        logger.debug("Could not summarise completion shape", exc_info=True)


#: Models seen to refuse function tools alongside their own default reasoning
#: effort. OpenAI applies a non-none default to reasoning models and rejects
#: that combination on /v1/chat/completions; the remedy its error names is an
#: explicit "none". Learned from the first failure rather than assumed, so a
#: model that accepts the pair keeps its reasoning.
_TOOLS_NEED_REASONING_OFF: set[str] = set()


def _model_info(model: str, provider: str | None = None) -> dict[str, Any]:
    """LiteLLM's row for `model`: transport id, then logical id, then bare name.

    A provider routed under a transport alias has two identities, and only one
    of them is in LiteLLM's table. Azure AI Foundry on its v1 surface is called
    as `hosted_vllm/<deployment>` — a transport that owns no model rows — while
    its metadata lives under `azure_ai/<deployment>`, which is LiteLLM's own
    key for the provider and the key OpenRAG stores it under.

    So `provider` is tried before the bare name. For a Foundry-exclusive model
    (`Phi-4`) the bare name matches nothing and only the logical id resolves;
    for one that a public vendor also serves (`gpt-6-luna`) the logical id is
    the more accurate of the two rows, since it carries Azure's pricing rather
    than OpenAI's.
    """
    try:
        import litellm

        table = litellm.model_cost
        bare = model.rsplit("/", 1)[-1]
        candidates = [model]
        if provider:
            candidates.append(f"{provider}/{bare}")
        candidates.append(bare)
        for candidate in candidates:
            info = table.get(candidate)
            if isinstance(info, dict):
                return info
        return {}
    except Exception:
        return {}


def _is_reasoning_tool_conflict(detail: str) -> bool:
    """Whether a failure is the tools-plus-reasoning_effort rejection.

    Matched on the provider's own words. There is no capability flag for it —
    `supports_reasoning` and `supports_function_calling` are both true for the
    models that refuse the combination — so the failure itself is the only
    signal available.
    """
    lowered = (detail or "").lower()
    return "reasoning_effort" in lowered and "tool" in lowered


def _reasoning_off_retry(model: str, kwargs: dict[str, Any], provider: str | None = None) -> bool:
    """Set `reasoning_effort="none"` for a retry, if that can help here.

    False when the caller chose an effort itself — overriding a deliberate
    choice would be worse than the error — or when the model cannot take
    "none", in which case only the Responses API can serve tools for it.
    """
    if not kwargs.get("tools") or "reasoning_effort" in kwargs:
        return False
    if not _model_info(model, provider).get("supports_none_reasoning_effort"):
        return False
    kwargs["reasoning_effort"] = "none"
    return True


async def with_stale_connection_retry(
    call: Callable[[], Awaitable[Any]], *, provider: str, model: str, kind: str
) -> Any:
    """Run one LiteLLM call, sending it once more if a dead pooled connection ate it.

    LiteLLM keeps idle connections for 120s; vLLM closes them after 5s. A call
    that picks one up just as the server closes it fails with "Server
    disconnected" before the server reads a byte (it logs nothing), and the
    replay goes out on a fresh connection. Only that failure is retried — see
    `is_provider_stale_connection_error` — so a request the server may have
    acted on is never sent twice. For a stream only the call that opens it is
    wrapped, so nothing has reached the caller yet; a mid-stream failure is
    not retried.
    """
    from api.provider_validation import is_provider_stale_connection_error

    try:
        return await call()
    except Exception as exc:
        if not is_provider_stale_connection_error(exc):
            raise
        logger.warning(
            "Model server closed an idle connection before answering; retrying once",
            provider=provider,
            model=model,
            kind=kind,
        )
        return await call()


async def chat_completions(
    body: Mapping[str, Any], *, config=None
) -> dict[str, Any] | AsyncIterator[str]:
    """OpenAI `POST /v1/chat/completions`. Streams SSE lines when `stream` is true."""
    cfg = config or _get_config()
    litellm_model, provider, credentials = resolve_call(body.get("model"), kind="chat", config=cfg)
    runtime_kwargs = _provider_runtime_kwargs(provider, cfg)
    kwargs = {key: body[key] for key in _LITELLM_FORWARDED_PARAMS if key in body}
    stream = bool(body.get("stream"))
    if litellm_model in _TOOLS_NEED_REASONING_OFF:
        # Already learned about this model; do not spend a round-trip relearning.
        _reasoning_off_retry(litellm_model, kwargs, provider)
    messages, repairs = _sanitise_messages(body.get("messages"))
    if repairs:
        logger.warning(
            "Repaired malformed tool calls in the request's conversation",
            provider=provider,
            model=litellm_model,
            repairs=repairs,
        )

    async def _call() -> Any:
        import litellm

        return await with_stale_connection_retry(
            lambda: litellm.acompletion(
                model=litellm_model,
                messages=messages,
                stream=stream,
                # OpenAI-compatible clients send OpenAI's full parameter set, but
                # providers accept different subsets — watsonx rejects
                # `parallel_tool_calls`, `max_completion_tokens` and `logit_bias`,
                # and LiteLLM raises UnsupportedParamsError rather than ignoring
                # them. A proxy that fans out to many providers must degrade to
                # the provider's capabilities instead of failing the request.
                drop_params=True,
                **credentials,
                **runtime_kwargs,
                **kwargs,
                **gateway_trace_kwargs(
                    "chat",
                    "completion",
                    provider=provider,
                    messages=len(messages) if isinstance(messages, list) else None,
                    tools=len(body["tools"]) if isinstance(body.get("tools"), list) else None,
                    stream=stream,
                ),
            ),
            provider=provider,
            model=litellm_model,
            kind="chat",
        )

    try:
        try:
            result = await _call()
        except Exception as exc:
            # `drop_params` cannot help here: reasoning_effort is a supported
            # parameter, just not one this model accepts beside tools. Retry
            # once with the value the provider's own error asks for.
            if not _is_reasoning_tool_conflict(f"{exc}") or not _reasoning_off_retry(
                litellm_model, kwargs, provider
            ):
                raise
            logger.info(
                "Retrying completion with reasoning_effort=none",
                provider=provider,
                model=litellm_model,
            )
            _TOOLS_NEED_REASONING_OFF.add(litellm_model)
            result = await _call()
    except LlmGatewayError:
        raise
    except Exception as exc:
        detail = _redact(f"{type(exc).__name__}: {exc}", credentials)
        logger.error(
            "LLM chat completions failed", provider=provider, model=litellm_model, error=detail
        )
        message = _upstream_client_message(detail, provider, litellm_model, exc)
        # The health banner otherwise reports whatever its own probe hit, which
        # is a different request and so often a different error. Hand it the
        # text this caller is being shown.
        failure = _classify_upstream_failure(detail, exc)
        if failure.counts_against_provider:
            provider_error_log.record_failure(provider, "chat", message)
        raise _upstream_gateway_error(message, detail, failure) from exc

    if stream:
        # A stream can still fail mid-flight, so _stream_sse clears the record
        # itself once the provider has actually delivered something.
        return _stream_sse(result, provider, litellm_model, credentials)
    provider_error_log.record_success(provider, "chat")
    payload = _to_openai_dict(result)
    _repair_completion_payload(payload, provider, litellm_model)
    _log_completion_shape(payload, provider, litellm_model)
    return payload


class _StreamTally:
    """Running totals for one streamed completion. Metadata only, no content."""

    def __init__(self) -> None:
        self.chunks = 0
        self.content_chars = 0
        self.tool_calls = 0
        self.repaired_tool_calls = 0
        self.finish_reason: str | None = None
        self.error: str | None = None

    def observe(self, payload: str) -> None:
        try:
            chunk = json.loads(payload)
        except Exception:
            return
        self.chunks += 1
        for choice in chunk.get("choices") or []:
            if not isinstance(choice, dict):
                continue
            if choice.get("finish_reason"):
                self.finish_reason = choice["finish_reason"]
            delta = choice.get("delta") or {}
            content = delta.get("content")
            if isinstance(content, str):
                self.content_chars += len(content)
            self.tool_calls += len(delta.get("tool_calls") or [])


class _ToolCallBuffer:
    """Reassembles streamed tool calls so their `arguments` can be repaired.

    A tool call arrives spread over many deltas: the opening one carries `id`
    and `name`, the rest carry `arguments` fragments the client concatenates.
    Whether those fragments concatenate into a usable object is only knowable
    once they are all in hand — the first fragment of a double-encoded payload
    (`'"{\\'`) is indistinguishable from a slow provider. So tool-call deltas are
    held here, joined, normalised, and emitted as one complete tool-call delta
    just before the finishing chunk. Content deltas are never buffered, so token
    streaming is untouched.
    """

    def __init__(self) -> None:
        self._calls: dict[tuple[int, int], dict[str, Any]] = {}
        self._order: list[tuple[int, int]] = []

    def __bool__(self) -> bool:
        return bool(self._order)

    def _open_call(self, choice_index: int) -> dict[str, Any] | None:
        """The call most recently opened on this choice, if there is one."""
        for key in reversed(self._order):
            if key[0] == choice_index:
                return self._calls[key]
        return None

    def _target(self, choice_index: int, index: int, raw: Mapping[str, Any]) -> dict[str, Any]:
        """The call `raw` belongs to: one already open, or a new one.

        A delta that names no call — no `id`, no `function.name` — cannot be the
        start of one, so it continues the call already open on this choice
        rather than opening another. vLLM's tool parsers can attribute a call's
        last `arguments` fragment, often the closing brace by itself, to the
        *next* index; taken at face value that does two kinds of damage at once.
        The real call loses the fragment and its arguments no longer parse, and
        a second call appears with no id and no name that nothing can execute.
        Neither survives the round trip: the client folds both into the
        assistant message it replays on the next turn, and the provider then
        rejects every later request in that conversation rather than the one
        message that is malformed.

        With nothing open to continue, the fragment instead opens a provisional
        call at its own index. Some providers send an `arguments` fragment ahead
        of the delta that carries the call's `id` and `name`; dropping it would
        truncate the arguments once that delta arrives at the same index and
        finds the call. A provisional call that is never named is discarded by
        `drain`, which logs it.
        """
        key = (choice_index, index)
        call = self._calls.get(key)
        if call is not None:
            return call
        function = raw.get("function")
        names_a_call = bool(raw.get("id")) or bool(
            isinstance(function, dict) and function.get("name")
        )
        if not names_a_call:
            open_call = self._open_call(choice_index)
            if open_call is not None:
                return open_call
            logger.debug(
                "Holding an unnamed tool-call fragment until its call is named",
                choice_index=choice_index,
                index=index,
            )
        call = {"index": index, "type": "function", "function": {"name": "", "arguments": ""}}
        self._calls[key] = call
        self._order.append(key)
        return call

    def absorb(self, choice_index: int, tool_calls: Any) -> None:
        for position, raw in enumerate(tool_calls or []):
            if not isinstance(raw, dict):
                continue
            index = raw.get("index")
            if not isinstance(index, int):
                index = position
            call = self._target(choice_index, index, raw)
            if raw.get("id"):
                call["id"] = raw["id"]
            if raw.get("type"):
                call["type"] = raw["type"]
            function = raw.get("function")
            if not isinstance(function, dict):
                continue
            name = function.get("name")
            # Providers split at different granularities: some send the whole
            # name once, some fragment it. Appending handles fragments; skipping
            # an exact repeat handles providers that resend the full name.
            if isinstance(name, str) and name and name != call["function"]["name"]:
                call["function"]["name"] += name
            arguments = function.get("arguments")
            if isinstance(arguments, str):
                call["function"]["arguments"] += arguments

    def drain(self, provider: str, model: str) -> tuple[dict[int, list[dict[str, Any]]], int]:
        """Take everything buffered, grouped by choice index, arguments repaired."""
        grouped: dict[int, list[dict[str, Any]]] = {}
        repaired = 0
        for choice_index, _ in self._order:
            grouped.setdefault(choice_index, [])
        for key in self._order:
            grouped[key[0]].append(self._calls[key])
        for choice_index, calls in grouped.items():
            calls.sort(key=lambda call: call.get("index", 0))
            usable = [call for call in calls if _finalise_tool_call(call)]
            if len(usable) != len(calls):
                logger.warning(
                    "Dropped tool calls that named no tool",
                    provider=provider,
                    model=model,
                    dropped=len(calls) - len(usable),
                )
            grouped[choice_index] = usable
            repaired += _repair_tool_calls(usable, provider, model)
        self._calls.clear()
        self._order.clear()
        return grouped, repaired


def _tool_call_chunks(
    template: Mapping[str, Any], grouped: Mapping[int, list[dict[str, Any]]]
) -> list[dict[str, Any]]:
    """One chunk per choice carrying that choice's complete tool calls."""
    return [
        {
            "id": template.get("id"),
            "object": template.get("object") or "chat.completion.chunk",
            "created": template.get("created"),
            "model": template.get("model"),
            "choices": [
                {
                    "index": choice_index,
                    "delta": {"role": "assistant", "tool_calls": calls},
                    "finish_reason": None,
                }
            ],
        }
        for choice_index, calls in grouped.items()
        if calls
    ]


def _chunk_carries_nothing(chunk: Mapping[str, Any]) -> bool:
    """True when a chunk has no payload left after its tool calls were held back."""
    if chunk.get("usage"):
        return False
    choices = chunk.get("choices") or []
    if not choices:
        return False
    for choice in choices:
        if not isinstance(choice, dict):
            return False
        if choice.get("finish_reason"):
            return False
        delta = choice.get("delta")
        if not isinstance(delta, dict):
            return False
        if any(value not in (None, "", [], {}) for value in delta.values()):
            return False
    return True


def _empty_stream_message(provider: str, model: str, finish_reason: str | None) -> str:
    """Client text for a completion that succeeded and said nothing."""
    return (
        f"{_call_label(provider, model)} accepted the request but returned no content and no "
        f"tool calls (finish_reason: {finish_reason or 'none'}). Check that the model supports "
        "the capabilities this request needs, or select a different model."
    )


def _error_frame(message: str, provider: str, model: str, code: str = "upstream_error") -> str:
    """SSE frame an OpenAI client turns back into a raised APIError."""
    return (
        "data: "
        + json.dumps(
            {
                "error": {
                    "message": message,
                    "type": "api_error",
                    "code": code,
                    "provider": provider,
                    "model": model,
                }
            }
        )
        + "\n\n"
    )


async def _aiter_stream(stream: Any):
    if hasattr(stream, "__aiter__"):
        async for chunk in stream:
            yield chunk
    else:
        for chunk in stream:
            yield chunk


async def _stream_sse(
    stream: Any,
    provider: str = "",
    model: str = "",
    credentials: Mapping[str, Any] | None = None,
) -> AsyncIterator[str]:
    """Yield OpenAI SSE frames, repair tool calls, and report classified failures.

    Context-window failures emit an error frame without affecting provider
    health. Close the upstream stream and finish with a ``[DONE]`` frame.
    """
    tally = _StreamTally()
    buffer = _ToolCallBuffer()
    template: dict[str, Any] = {}
    try:
        async for chunk in _aiter_stream(stream):
            payload = _chunk_payload(chunk)
            try:
                data = json.loads(payload)
            except ValueError:
                tally.observe(payload)
                yield f"data: {payload}\n\n"
                continue
            if not isinstance(data, dict):
                tally.observe(payload)
                yield f"data: {payload}\n\n"
                continue
            template = data

            held = False
            finishing = False
            for choice in data.get("choices") or []:
                if not isinstance(choice, dict):
                    continue
                delta = choice.get("delta")
                if isinstance(delta, dict) and delta.get("tool_calls"):
                    index = choice.get("index")
                    buffer.absorb(index if isinstance(index, int) else 0, delta["tool_calls"])
                    delta["tool_calls"] = None
                    held = True
                if choice.get("finish_reason"):
                    finishing = True

            if finishing and buffer:
                grouped, repaired = buffer.drain(provider, model)
                tally.repaired_tool_calls += repaired
                for flushed in _tool_call_chunks(template, grouped):
                    out = json.dumps(flushed)
                    tally.observe(out)
                    yield f"data: {out}\n\n"

            if held:
                if _chunk_carries_nothing(data):
                    continue
                payload = json.dumps(data)
            tally.observe(payload)
            yield f"data: {payload}\n\n"

        # A provider that ends without a finish_reason still owes us its calls.
        if buffer:
            grouped, repaired = buffer.drain(provider, model)
            tally.repaired_tool_calls += repaired
            for flushed in _tool_call_chunks(template, grouped):
                out = json.dumps(flushed)
                tally.observe(out)
                yield f"data: {out}\n\n"

        # A 200 that carries nothing is indistinguishable from a dead connection
        # by the time it reaches the UI, which can only say "the server didn't
        # return a response". Name the call and its finish_reason instead, so
        # the next person has something to search for.
        if tally.content_chars == 0 and tally.tool_calls == 0:
            tally.error = "empty completion"
            provider_error_log.record_failure(
                provider, "chat", _empty_stream_message(provider, model, tally.finish_reason)
            )
            logger.warning(
                "LLM chat stream produced no content and no tool calls",
                provider=provider,
                model=model,
                finish_reason=tally.finish_reason,
                chunks=tally.chunks,
            )
            yield _error_frame(
                _empty_stream_message(provider, model, tally.finish_reason), provider, model
            )
        else:
            # Content actually reached the client, so whatever the banner was
            # holding against this provider is no longer true.
            provider_error_log.record_success(provider, "chat")
    except Exception as exc:
        # Mid-stream failures used to surface as a truncated stream, which the
        # UI could only report as "the server didn't return a response". Emit an
        # OpenAI error frame instead: the client raises it with the provider's
        # own wording, so the cause reaches the chat and the logs alike.
        detail = _redact(f"{type(exc).__name__}: {exc}", credentials or {})
        tally.error = detail
        logger.error("LLM chat stream failed", provider=provider, model=model, error=detail)
        message = _upstream_client_message(detail, provider, model, exc)
        failure = _classify_upstream_failure(detail, exc)
        if failure.counts_against_provider:
            provider_error_log.record_failure(provider, "chat", message)
        yield _error_frame(message, provider, model, failure.code)
    finally:
        close = getattr(stream, "aclose", None) or getattr(stream, "close", None)
        if close is not None:
            result = close()
            if hasattr(result, "__await__"):
                await result
        _log_stream_shape(tally, provider, model)
    yield "data: [DONE]\n\n"


def _log_stream_shape(tally: _StreamTally, provider: str, model: str) -> None:
    """Same diagnostic as `_log_completion_shape`, for the streaming path."""
    try:
        if tally.error:
            return  # already logged with its cause by the stream's error handler
        empty = tally.content_chars == 0 and tally.tool_calls == 0
        log = logger.warning if empty else logger.info
        log(
            "LLM chat stream produced no content and no tool calls" if empty else "LLM chat stream",
            provider=provider,
            model=model,
            finish_reason=tally.finish_reason,
            content_chars=tally.content_chars,
            tool_calls=tally.tool_calls,
            repaired_tool_calls=tally.repaired_tool_calls,
            chunks=tally.chunks,
        )
    except Exception:  # diagnostics must never break a stream
        logger.debug("Could not summarise stream shape", exc_info=True)


_WATSONX_ONPREM_EMBEDDING_BATCH_SIZE = 32


def _embedding_input(value: Any) -> Any:
    """OpenAI's `input`, in the shape every provider behind us accepts.

    OpenAI takes a bare string as well as an array, so its clients send both —
    and LiteLLM forwards whichever it is given. watsonx.ai takes only an array
    and rejects a string with "Mismatch type []string with value string", so a
    client that sends one gets a failure that reads like a bug in its own
    request. Wrapping is a no-op for a provider that already accepted the
    string, and token-array inputs are left exactly as they are.
    """
    return [value] if isinstance(value, str) else value


def _embedding_trace_sizes(embedding_input: Any) -> dict[str, int | None]:
    """How many inputs and how much text, for the trace; never the text itself."""
    if not isinstance(embedding_input, list) or not embedding_input:
        return {}
    if all(isinstance(item, int) for item in embedding_input):
        return {"inputs": 1}  # one pre-tokenised input
    all_text = all(isinstance(item, str) for item in embedding_input)
    return {
        "inputs": len(embedding_input),
        "input_chars": sum(map(len, embedding_input)) if all_text else None,
    }


#: How many times one input is cut smaller before the gateway gives up on it.
_MAX_OVERFLOW_SPLIT_ROUNDS = 3
#: Pieces are sized under the proportion the provider reported: tokens are not
#: spread evenly through a text, and a cut may land a little short of its mark.
_OVERFLOW_HEADROOM = 0.85


def _is_text_list(value: Any) -> bool:
    """Whether `value` is a batch of strings, the only input that can be cut into pieces."""
    return isinstance(value, list) and bool(value) and all(isinstance(item, str) for item in value)


def _split_into_pieces(
    text: str, max_tokens: int | None, requested_tokens: int | None
) -> list[str]:
    """Cut `text` into equal pieces expected to fit the model, at whitespace where it can.

    The provider's own counts say how far over the input was, so the number of
    pieces follows that proportion; with no counts, the text is halved. Equal
    pieces rather than "as much as fits, then the rest": a short tail would
    carry little context of its own into the average.
    """
    count = 2
    if max_tokens and requested_tokens and requested_tokens > max_tokens:
        count = max(2, math.ceil(requested_tokens / (max_tokens * _OVERFLOW_HEADROOM)))
    size = len(text) / count
    slack = int(size // 10)
    cuts = [0]
    for number in range(1, count):
        ideal = round(number * size)
        earliest = max(ideal - slack, cuts[-1] + 1)
        cuts.append(
            next((at for at in range(ideal, earliest - 1, -1) if text[at - 1].isspace()), ideal)
        )
    cuts.append(len(text))
    return [piece for start, end in pairwise(cuts) if (piece := text[start:end]).strip()]


def _weighted_mean_vector(vectors: list[list[float]], weights: list[int]) -> list[float]:
    """The mean of `vectors` weighted by `weights`, at the length the model's vectors have.

    A plain mean of vectors that point different ways is shorter than any of
    them, which would understate the chunk wherever the index compares raw
    lengths (l2, inner product). Scaling it back to the pieces' weighted mean
    length keeps it comparable to every other vector from the same model: unit
    length when the model normalises its output, its usual magnitude when not.
    """
    total = sum(weights)
    mean = [
        sum(vector[axis] * weight for vector, weight in zip(vectors, weights, strict=True)) / total
        for axis in range(len(vectors[0]))
    ]
    length = math.hypot(*mean)
    if not length:
        return mean
    target = (
        sum(math.hypot(*vector) * weight for vector, weight in zip(vectors, weights, strict=True))
        / total
    )
    return [value * target / length for value in mean]


async def _embed_averaging_overflow(
    send: Callable[[list[str]], Awaitable[dict[str, Any]]],
    texts: list[str],
    overflow: LlmContextLengthError,
    *,
    provider: str,
    model: str,
    max_batch: int | None = None,
) -> dict[str, Any]:
    """Embed a batch the provider rejected as too long, still one vector per input.

    An embeddings call owes its caller exactly one vector per input, so an
    over-length input cannot become several chunks here. It is embedded in
    pieces and the piece vectors are averaged by length instead, so every part
    of the text still contributes to the vector. The provider says only that
    something in the batch was too long, so the batch is halved until the
    offending input is alone. `send` raises `LlmContextLengthError` for a
    rejection of that kind and `LlmGatewayError` for anything else. With
    `max_batch`, no request carries more inputs than that, even when one input
    is cut into more pieces.
    """
    envelope: dict[str, Any] = {}
    usage: dict[str, int | float] = {}

    async def embed(
        batch: list[str], rounds: int, rejected: LlmContextLengthError | None = None
    ) -> list[list[float]]:
        if rejected is None and max_batch and len(batch) > max_batch:
            vectors: list[list[float]] = []
            for start in range(0, len(batch), max_batch):
                vectors += await embed(batch[start : start + max_batch], rounds)
            return vectors
        if rejected is None:
            try:
                payload = await send(batch)
            except LlmContextLengthError as error:
                rejected = error
            else:
                if not envelope:
                    envelope.update(
                        {
                            key: value
                            for key, value in payload.items()
                            if key not in {"data", "usage"}
                        }
                    )
                for key, value in (payload.get("usage") or {}).items():
                    if isinstance(value, (int, float)):
                        usage[key] = usage.get(key, 0) + value
                items = sorted(payload.get("data", []), key=lambda item: item.get("index", 0))
                return [item["embedding"] for item in items]
        if len(batch) > 1:
            middle = len(batch) // 2
            return await embed(batch[:middle], rounds) + await embed(batch[middle:], rounds)
        return [await average(batch[0], rounds, rejected)]

    async def average(text: str, rounds: int, rejected: LlmContextLengthError) -> list[float]:
        pieces = _split_into_pieces(text, rejected.max_tokens, rejected.requested_tokens)
        if rounds >= _MAX_OVERFLOW_SPLIT_ROUNDS or len(pieces) < 2:
            raise rejected
        vectors = await embed(pieces, rounds + 1)
        # Sizes only: the text itself is the customer's document.
        logger.warning(
            "Embedding input exceeded the model's limit; embedded in pieces and averaged",
            provider=provider,
            model=model,
            max_tokens=rejected.max_tokens,
            requested_tokens=rejected.requested_tokens,
            chars=len(text),
            pieces=len(pieces),
        )
        return _weighted_mean_vector(vectors, [len(piece) for piece in pieces])

    vectors = await embed(texts, 0, overflow)
    response = {
        **envelope,
        "data": [
            {"object": "embedding", "index": index, "embedding": vector}
            for index, vector in enumerate(vectors)
        ],
    }
    if usage:
        response["usage"] = usage
    return response


async def embeddings(
    body: Mapping[str, Any], *, config=None, interactive: bool = False
) -> dict[str, Any]:
    """OpenAI `POST /v1/embeddings`.

    `interactive` marks a query embedding someone is waiting on (search, chat
    retrieval). At a provider with a concurrency limit it waits in its own
    lane instead of behind bulk ingestion; see `_embedding_limiter`.

    An input the model rejects as too long is embedded in pieces and averaged
    rather than failed, unless `OPENRAG_EMBEDDING_OVERFLOW_POLICY=error`; see
    `_embed_averaging_overflow`.
    """
    from config.settings import embedding_overflow_policy

    cfg = config or _get_config()
    litellm_model, provider, credentials = resolve_call(
        body.get("model"), kind="embedding", config=cfg
    )
    runtime_kwargs = _provider_runtime_kwargs(provider, cfg)
    limiter = _embedding_limiter(provider, cfg)
    lane = _INTERACTIVE_LANE if interactive else _BULK_LANE
    purpose = "query" if interactive else "ingest"
    embedding_input = _embedding_input(body.get("input"))

    batch_size = _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE if provider == "watsonx_onprem" else None

    async def call_provider(inputs: Any) -> dict[str, Any]:
        import litellm

        async with _embedding_slot(limiter, provider, litellm_model, lane):
            result = await with_stale_connection_retry(
                lambda: litellm.aembedding(
                    model=litellm_model,
                    input=inputs,
                    **credentials,
                    **runtime_kwargs,
                    **gateway_trace_kwargs(
                        "embedding",
                        purpose,
                        provider=provider,
                        **_embedding_trace_sizes(inputs),
                    ),
                ),
                provider=provider,
                model=litellm_model,
                kind="embedding",
            )
        return _to_openai_dict(result)

    async def send(inputs: Any) -> dict[str, Any]:
        try:
            return await call_provider(inputs)
        except LlmGatewayError:
            raise
        except Exception as exc:
            detail = _redact(f"{type(exc).__name__}: {exc}", credentials)
            message = _upstream_client_message(detail, provider, litellm_model, exc)
            # An over-length input is one chunk's problem. Recording it latched the
            # banner against a provider that was embedding every other chunk fine.
            failure = _classify_upstream_failure(detail, exc)
            if failure.counts_against_provider:
                provider_error_log.record_failure(provider, "embedding", message)
            raise _upstream_gateway_error(message, detail, failure) from exc

    async def embed(inputs: Any) -> dict[str, Any]:
        try:
            return await send(inputs)
        except LlmContextLengthError as overflow:
            if embedding_overflow_policy() != "average" or not _is_text_list(inputs):
                raise
            return await _embed_averaging_overflow(
                send,
                inputs,
                overflow,
                provider=provider,
                model=litellm_model,
                max_batch=batch_size,
            )

    should_batch = (
        batch_size is not None
        and isinstance(embedding_input, list)
        and len(embedding_input) > batch_size
        and not all(isinstance(item, int) for item in embedding_input)
    )
    try:
        if not should_batch:
            response = await embed(embedding_input)
        else:
            # Each batch recovers on its own, so an over-length input in a later
            # batch does not send the earlier, already embedded ones again.
            response = {}
            data: list[dict[str, Any]] = []
            usage: dict[str, int | float] = {}
            sub_batches = -(-len(embedding_input) // batch_size)
            for offset in range(0, len(embedding_input), batch_size):
                # Numbered in the trace; an over-length input's pieces keep it.
                with trace_context(
                    attributes={
                        "openrag.embedding.sub_batch": offset // batch_size + 1,
                        "openrag.embedding.sub_batches": sub_batches,
                    }
                ):
                    payload = await embed(embedding_input[offset : offset + batch_size])
                if not response:
                    response = {
                        key: value for key, value in payload.items() if key not in {"data", "usage"}
                    }
                for position, item in enumerate(payload.get("data", [])):
                    merged_item = dict(item)
                    merged_item["index"] = offset + int(item.get("index", position))
                    data.append(merged_item)
                for key, value in payload.get("usage", {}).items():
                    if isinstance(value, (int, float)):
                        usage[key] = usage.get(key, 0) + value
            response["data"] = data
            if usage:
                response["usage"] = usage
    except LlmGatewayError as error:
        logger.error(
            "LLM embeddings failed", provider=provider, model=litellm_model, error=error.detail
        )
        raise
    provider_error_log.record_success(provider, "embedding")
    return response