Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Improve RHOAI model provider error handling and resiliency (#2419)

Mike Pawlowskifa2619db1273
diff --git a/.env.example b/.env.example
index cdbc80318..d02a193a3 100644
--- a/.env.example
+++ b/.env.example
@@ -323,6 +323,16 @@ WATSONX_PROJECT_ID=
# deployed environment.
# RHOAI_TLS_VERIFY=
#
+# Maximum embedding requests OpenRAG sends to the embeddings endpoint at once
+# (default 16, sized for a GPU-served model; 0 = no limit). Extra requests wait
+# inside OpenRAG. Set 4 for a CPU-served model: it embeds well under one chunk
+# per second, and the endpoint's kube-rbac-proxy answers 502 once a request
+# waits upstream past its 30s timeout -- which fails folder uploads (several
+# files x 8 embedding threads). Also editable in Settings. One of these
+# slots is reserved for search and chat queries, so they are not queued behind
+# ingestion (with a limit of 1, both share the single slot).
+# RHOAI_EMBEDDING_MAX_CONCURRENCY=
+#
# Selecting the models. The ids are each InferenceService's --served-model-name,
# not an image tag. OpenRAG lists the real ones from each endpoint's /v1/models.
# LLM_PROVIDER=rhoai
@@ -431,6 +441,14 @@ LOG_FORMAT=
# - Default: INFO
# LOG_LEVEL=DEBUG
+# OPTIONAL: Set the logging level for the Langflow container
+# - Supported values: DEBUG, INFO, WARNING, ERROR, CRITICAL
+# - Default: INFO
+# - DEBUG logs every component's full result, including whole parsed Docling
+# documents (multi-MB lines). Writing them blocks Langflow's single worker
+# during ingestion, so health checks time out. Enable only when debugging a flow.
+# LANGFLOW_LOG_LEVEL=DEBUG
+
# OPTIONAL: Disable colored log output (https://no-color.org)
# - When set (to any value), ANSI color codes are omitted from log output
# NO_COLOR=1
diff --git a/.github/workflows/autofix.ci.yml b/.github/workflows/autofix.ci.yml
index 30fa2056e..4a0960b42 100644
--- a/.github/workflows/autofix.ci.yml
+++ b/.github/workflows/autofix.ci.yml
@@ -78,6 +78,8 @@ jobs:
# the App's commit triggers a fresh CI cycle that passes.
- name: Apply ruff autofix via autofix.ci
if: steps.changed.outputs.files != ''
- uses: autofix-ci/[email protected]
+ # v1.3.3+ calls autofix-api.maximilianhils.com; older pins call
+ # api.autofix.ci, which no longer resolves (ENOTFOUND).
+ uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4
with:
commit-message: "style: ruff autofix (auto)"
diff --git a/Makefile b/Makefile
index ab60a079a..37337061c 100644
--- a/Makefile
+++ b/Makefile
@@ -371,6 +371,7 @@ help_local: ## Show local development commands
@echo " $(CYAN)RHOAI_EMBEDDINGS_ENDPOINT$(NC) - embedding InferenceService URL (vLLM serves one model each)"
@echo " $(CYAN)RHOAI_API_KEY$(NC) - ServiceAccount bearer token for both endpoints"
@echo " $(CYAN)RHOAI_TLS_VERIFY$(NC) - CA bundle path, or 'false' for a port-forward (dev only)"
+ @echo " $(CYAN)RHOAI_EMBEDDING_MAX_CONCURRENCY$(NC) - max in-flight embedding calls (default 16 for GPU; use 4 for CPU; 0 = no limit)"
@echo " In-cluster, use the predictor Service DNS and the projected service CA:"
@echo " $(CYAN)/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt$(NC)"
@echo " From a laptop, port-forward each InferenceService and set RHOAI_TLS_VERIFY=false:"
diff --git a/custom_components/openrag/docling_remote.py b/custom_components/openrag/docling_remote.py
index 522fb10ae..7842c64f3 100644
--- a/custom_components/openrag/docling_remote.py
+++ b/custom_components/openrag/docling_remote.py
@@ -209,6 +209,23 @@ class DoclingRemoteComponent(BaseFileComponent):
# Default behavior
return super().update_build_config(build_config, field_value, field_name)
+ def _raise_if_task_missing(self, response: httpx.Response, task_id: str) -> None:
+ """Raise a clear error when Docling Serve no longer knows the task.
+
+ Docling Serve deletes a finished task some minutes after its result is
+ first fetched, and forgets every task when it restarts. Polling such a
+ task returns 404, which would otherwise surface as a bare HTTP error.
+ """
+ if response.status_code != httpx.codes.NOT_FOUND:
+ return
+ msg = (
+ f"Docling task {task_id} was not found on the Docling Serve server. Its result "
+ "may have expired (Docling Serve deletes finished results after a delay) or the "
+ "server restarted. Re-ingest the file."
+ )
+ self.log(msg)
+ raise RuntimeError(msg)
+
def _poll_and_fetch_result(
self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None
) -> Data | None:
@@ -229,6 +246,7 @@ class DoclingRemoteComponent(BaseFileComponent):
start_wait_time = time.monotonic()
response = client.get(f"{base_url}/status/poll/{task_id}")
+ self._raise_if_task_missing(response, task_id)
response.raise_for_status()
task = response.json()
@@ -255,6 +273,8 @@ class DoclingRemoteComponent(BaseFileComponent):
return None
continue
+ self._raise_if_task_missing(response, task_id)
+ response.raise_for_status()
task = response.json()
result_resp = client.get(f"{base_url}/result/{task_id}")
diff --git a/docker-compose.yml b/docker-compose.yml
index 9a9cd015a..bbf2320a5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -94,6 +94,7 @@ services:
- RHOAI_EMBEDDINGS_ENDPOINT=${RHOAI_EMBEDDINGS_ENDPOINT:-}
- RHOAI_API_KEY=${RHOAI_API_KEY:-}
- RHOAI_TLS_VERIFY=${RHOAI_TLS_VERIFY:-}
+ - RHOAI_EMBEDDING_MAX_CONCURRENCY=${RHOAI_EMBEDDING_MAX_CONCURRENCY:-}
- OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT}
- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}
- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}
@@ -273,7 +274,7 @@ services:
- FILESIZE=0
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-text-embedding-3-small}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_SERVE_VERIFY_SSL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENRAG_LLM_TOKEN,OPENRAG_LLM_BASE_URL,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE
- - LANGFLOW_LOG_LEVEL=DEBUG
+ - LANGFLOW_LOG_LEVEL=${LANGFLOW_LOG_LEVEL:-INFO}
- LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}
- LANGFLOW_RATE_LIMIT_PER_MINUTE=${LANGFLOW_RATE_LIMIT_PER_MINUTE:-100}
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index 36ed10847..88049184e 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -74,6 +74,7 @@ These variables are initially set during the [application onboarding process](/i
| `RHOAI_EMBEDDINGS_ENDPOINT` | Not set | Embeddings endpoint for the Red Hat OpenShift AI model provider, a second `InferenceService`. Leave unset only if one endpoint serves both chat and embeddings. |
| `RHOAI_API_KEY` | Not set | Kubernetes bearer token for the Red Hat OpenShift AI model provider, sent to both endpoints. Its subject needs `get` on `inferenceservices` in the model namespace. |
| `RHOAI_TLS_VERIFY` | Not set | TLS trust for the Red Hat OpenShift AI endpoints. Set to a CA bundle path, such as the service-serving CA OpenShift projects into every pod (`/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`), or `false` to skip verification for a local `oc port-forward` only. Scoped to this provider, so it does not affect other providers. |
+| `RHOAI_EMBEDDING_MAX_CONCURRENCY` | `16` | Maximum number of embedding requests OpenRAG sends to the Red Hat OpenShift AI embeddings endpoint at once; further requests wait inside OpenRAG. The default suits a GPU-served model. For a CPU-served model set `4`: it answers slowly enough that more queued requests outlast the endpoint proxy's upstream timeout, which returns `502` when several files are ingested together. One slot is reserved for search and chat queries so they are not queued behind ingestion; with a limit of `1` both share it. Set `0` for no limit. Also editable in **Settings**. |
Newly indexed chunks keep the model name in `embedding_model` for compatibility and also store
`embedding_provider` plus the unambiguous `embedding_space_id` (`provider:model`). Retrieval queries
diff --git a/enhancements/providers/redhat/openshift_ai.py b/enhancements/providers/redhat/openshift_ai.py
index d153d34d8..e08889c2e 100644
--- a/enhancements/providers/redhat/openshift_ai.py
+++ b/enhancements/providers/redhat/openshift_ai.py
@@ -145,13 +145,41 @@ CREDENTIAL_FIELDS: list[dict[str, Any]] = [
"options": None,
"default_value": None,
},
+ {
+ "key": "embedding_max_concurrency",
+ "label": "Max concurrent embedding requests",
+ "placeholder": "16",
+ "tooltip": "How many embedding requests OpenRAG sends to the embedding endpoint at "
+ "once; the rest wait in OpenRAG. One slot is kept for search and chat queries so "
+ "they never wait behind ingestion. Leave blank for the default of 16, sized for a "
+ "GPU-served model. Lower it to 4 for a CPU-served model: it answers slowly, and "
+ "the endpoint's kube-rbac-proxy returns 502 once a queued request waits past its "
+ "upstream timeout (30s by default), which fails ingestion. 0 means no limit.",
+ "required": False,
+ "field_type": "text",
+ "options": None,
+ "default_value": None,
+ },
]
#: Fields an operator fills in that are not LiteLLM kwargs. The embedding
#: endpoint is selected *into* `api_base` per call; forwarding it as well would
#: land it in the request body, since LiteLLM passes kwargs it does not
-#: recognise straight through.
-_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base"})
+#: recognise straight through. The concurrency limit is enforced by the
+#: gateway (`embedding_max_concurrency`), never sent to the endpoint.
+_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base", "embedding_max_concurrency"})
+
+#: Concurrent embedding calls allowed when the operator sets no limit, sized for
+#: a GPU-served model, the production norm. Ingestion sends one chunk per call,
+#: so vLLM's single API-server process (HTTP, tokenization, JSON) saturates
+#: long before the GPU does: granite-embedding-english-r2 on an H100 served
+#: about 200 calls/s at 16 in flight (p95 under 170ms) and topped out near 430
+#: calls/s at 64-128, with the GPU about 20% busy. 16 leaves that headroom to
+#: other OpenRAG instances sharing the endpoint. A CPU-served model is ~100x
+#: slower (about 2.7s per ~1000-character chunk on 5 cores), so 16 in flight
+#: queues calls past the 30s `kube-rbac-proxy` upstream timeout; CPU
+#: deployments set 4.
+DEFAULT_EMBEDDING_MAX_CONCURRENCY = 16
#: LiteLLM kwargs that must not become request-body fields. `ssl_verify` is
#: read for TLS *and* — on the `hosted_vllm` chat path, verified against litellm
@@ -361,6 +389,35 @@ def litellm_credentials(stored: Mapping[str, Any], *, kind: CallKind = "chat") -
return credentials
+def embedding_max_concurrency(stored: Mapping[str, Any] | None) -> int | None:
+ """How many embedding calls the gateway may have in flight to this endpoint.
+
+ Optional member of the enhancement contract, read through
+ `registry.embedding_concurrency_for`. Returns `None` for "no limit".
+
+ Blank means the default. `0` switches the limit off. A value that is not a
+ non-negative integer falls back to the default with a one-time warning
+ rather than disabling the limit: failing open here is exactly the overload
+ this setting exists to prevent.
+ """
+ raw = _values(stored).get("embedding_max_concurrency", "")
+ if not raw:
+ return DEFAULT_EMBEDDING_MAX_CONCURRENCY
+ try:
+ limit = int(raw)
+ except ValueError:
+ limit = -1
+ if limit < 0:
+ _warn_once(
+ f"concurrency:{raw}",
+ "Ignoring an invalid OpenShift AI embedding concurrency limit; using the default.",
+ value=raw,
+ default=DEFAULT_EMBEDDING_MAX_CONCURRENCY,
+ )
+ return DEFAULT_EMBEDDING_MAX_CONCURRENCY
+ return limit or None
+
+
# --------------------------------------------------------------------------
# Model discovery
# --------------------------------------------------------------------------
@@ -456,8 +513,40 @@ def _auth_headers(api_key: str) -> dict[str, str]:
return {"Authorization": f"Bearer {api_key}", "Accept": "application/json"}
+def _content_type(response: Any) -> str:
+ """The response's media type, lower-cased and without parameters, or ''."""
+ try:
+ headers = getattr(response, "headers", None) or {}
+ value = headers.get("content-type", "") or ""
+ return str(value).split(";", 1)[0].strip().lower()
+ except Exception:
+ return ""
+
+
+def _looks_like_html(response: Any) -> bool:
+ """Whether a response is a web page rather than an API answer.
+
+ An OAuth proxy, SSO gateway or captive portal in front of the route answers
+ with its login page. The content type usually says so; when it is missing,
+ the start of the body does.
+ """
+ if "html" in _content_type(response):
+ return True
+ try:
+ head = str(getattr(response, "text", "") or "").lstrip()[:64].lower()
+ except Exception:
+ return False
+ return head.startswith(("<!doctype html", "<html"))
+
+
def _error_details(response: Any) -> str:
- """The concise error an OpenAI-compatible endpoint returned."""
+ """The concise error an OpenAI-compatible endpoint returned.
+
+ A web page is described rather than quoted: half a kilobyte of login-page
+ markup helps nobody in a Settings error or a log line.
+ """
+ if _looks_like_html(response):
+ return f"The response was an HTML page ({_content_type(response) or 'text/html'})."
try:
body = response.json()
except (ValueError, TypeError):
@@ -525,6 +614,18 @@ def model_ids(body: Any) -> tuple[str, ...]:
return tuple(ids)
+def is_model_listing(body: Any) -> bool:
+ """Whether `body` has the shape of an OpenAI `GET /v1/models` answer.
+
+ Only the shape is checked — an object whose `data` is a list. An empty list
+ still counts: a `vLLM` pod that is still loading its weights is a real
+ endpoint, and the health check deliberately does not depend on a model
+ having been chosen. `model_ids` cannot answer this, because it returns
+ nothing both for a wrong shape and for an empty listing.
+ """
+ return isinstance(body, Mapping) and isinstance(body.get("data"), list)
+
+
async def _list_models(client: Any, api_base: str, api_key: str) -> tuple[str, ...] | None:
"""The models one endpoint serves, or None if it could not be asked."""
url = models_url(api_base)
@@ -682,6 +783,10 @@ async def lightweight_health_check(credentials: Mapping[str, Any]) -> None:
carries `embedding_api_base`; `validate_provider_setup(stored_credentials=...)`
is how it gets here. Given only the LiteLLM form, the one endpoint it was
narrowed to is checked.
+
+ A 200 only passes when its body is an OpenAI-shaped model listing: an
+ OAuth proxy or SSO gateway in front of the route can answer 200 with its
+ login page, which would otherwise read as healthy and fail on first use.
"""
import httpx
@@ -721,6 +826,7 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)
import httpx
url = models_url(api_base)
+ logger.debug("Checking the OpenShift AI endpoint", endpoint=label, url=url)
try:
response = await _http_request_with_retry(
"GET",
@@ -743,8 +849,26 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)
f"Could not reach the OpenShift AI {label} endpoint at {api_base}: {error}"
) from error
+ if 300 <= response.status_code < 400:
+ # httpx does not follow redirects, and should not here: the target is
+ # a login page, and the token must not be carried to it.
+ location = _redirect_location(response)
+ logger.error(
+ "The OpenShift AI endpoint redirected instead of answering",
+ endpoint=label,
+ url=url,
+ status_code=response.status_code,
+ location=location,
+ )
+ raise Exception(
+ f"The OpenShift AI {label} endpoint at {api_base} redirected to "
+ f"{location or 'another page'} instead of answering /v1/models. This is usually "
+ "an OAuth proxy or SSO login page in front of the route; point the URL at the "
+ "model's inference route or predictor Service, which accepts a ServiceAccount token."
+ )
+
if response.status_code == 200:
- logger.debug("OpenShift AI endpoint answered", endpoint=label, url=url)
+ _require_model_listing(response, label, api_base, url)
return
details = _error_details(response)
@@ -769,3 +893,76 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)
raise Exception(
f"The OpenShift AI {label} endpoint returned {response.status_code}: {details}".strip()
)
+
+
+def _redirect_location(response: Any) -> str:
+ """Where a redirect points, without its query string.
+
+ An OAuth redirect's query carries `state`, `client_id` and a nested
+ `redirect_uri`, which make the message unreadable and say nothing the
+ host and path do not.
+ """
+ try:
+ headers = getattr(response, "headers", None) or {}
+ location = str(headers.get("location", "") or "").strip()
+ if not location:
+ return ""
+ parts = urlsplit(location)
+ return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+ except Exception:
+ return ""
+
+
+def _require_model_listing(response: Any, label: str, api_base: str, url: str) -> None:
+ """Fail a 200 whose body is not a model listing, naming the likely cause."""
+ try:
+ body = response.json()
+ except (ValueError, TypeError):
+ body = None
+
+ if is_model_listing(body):
+ models = model_ids(body)
+ if models:
+ logger.debug(
+ "OpenShift AI endpoint answered with a model list",
+ endpoint=label,
+ url=url,
+ models=len(models),
+ )
+ else:
+ # Still a real endpoint — vLLM may be loading its weights — so
+ # this passes; the warning is for whoever wonders why the picker
+ # shows the configured fallback.
+ logger.warning(
+ "The OpenShift AI endpoint answered with an empty model list",
+ endpoint=label,
+ url=url,
+ )
+ return
+
+ content_type = _content_type(response)
+ if _looks_like_html(response):
+ logger.error(
+ "The OpenShift AI endpoint answered with a web page instead of a model list",
+ endpoint=label,
+ url=url,
+ content_type=content_type,
+ )
+ raise Exception(
+ f"The OpenShift AI {label} endpoint at {api_base} returned a web page "
+ f"({content_type or 'text/html'}) instead of a model list. This is usually an "
+ "OAuth proxy or SSO login page in front of the route; point the URL at the "
+ "model's inference route or predictor Service, which accepts a ServiceAccount token."
+ )
+ logger.error(
+ "The OpenShift AI endpoint answered with something other than a model list",
+ endpoint=label,
+ url=url,
+ content_type=content_type,
+ body_type=type(body).__name__,
+ )
+ raise Exception(
+ f"The OpenShift AI {label} endpoint at {api_base} answered /v1/models with something "
+ "other than an OpenAI-compatible model list. Check the URL points at the model's "
+ "inference route and ends in /v1."
+ )
diff --git a/enhancements/providers/registry.py b/enhancements/providers/registry.py
index d1a44aa05..00ae4ceee 100644
--- a/enhancements/providers/registry.py
+++ b/enhancements/providers/registry.py
@@ -11,11 +11,13 @@ import functools
import inspect
from collections.abc import Mapping
from types import ModuleType
-from typing import Any
+from typing import Any, Literal
from enhancements.providers.redhat import openshift_ai
from enhancements.providers.watsonx import onprem
+CallKind = Literal["chat", "embedding"]
+
_ENHANCEMENTS: dict[str, ModuleType] = {
onprem.PROVIDER_KEY: onprem,
openshift_ai.PROVIDER_KEY: openshift_ai,
@@ -50,6 +52,50 @@ def enhancements() -> tuple[ModuleType, ...]:
return tuple(_ENHANCEMENTS.values())
+def live_models_for(provider: str, kind: CallKind) -> tuple[str, ...] | None:
+ """What `provider` last said it serves for `kind`, if it can say at all.
+
+ None means *unknown*, never "serves nothing": the provider has no
+ enhancement, the enhancement cannot list its own models, or nothing fresh
+ is cached for that endpoint. Callers must treat None as no information —
+ telling an operator their model is missing on the strength of a listing
+ that never arrived would be worse than staying quiet.
+
+ An empty tuple is the opposite: a real answer. The provider was listed and
+ that endpoint serves nothing of this kind — watsonx on-prem caches exactly
+ that for a cluster with chat models and no embedding model — so it must
+ reach the caller as `()`, not be folded into unknown. An enhancement that
+ cannot tell "empty" from "unreadable" keeps that half None itself.
+
+ The lists come from the enhancement's own TTL cache, so this is a dict
+ lookup; refreshing them is `model_catalog.refresh_live_models()`.
+ """
+ enhancement = get(provider)
+ if enhancement is None or not hasattr(enhancement, "cached_models"):
+ return None
+ try:
+ models = enhancement.cached_models()
+ except Exception as exc: # a diagnostic must never take down its caller
+ _log_hook_failure(provider, "cached_models", exc)
+ return None
+ if models is None:
+ return None
+ listed = getattr(models, kind, None)
+ if listed is None:
+ return None
+ # A bare string would split into characters, and a scalar cannot be
+ # iterated at all; either is a malformed cache, so unknown rather than a
+ # list of nonsense model ids.
+ if isinstance(listed, str | bytes):
+ _log_unreadable_listing(provider, kind, listed)
+ return None
+ try:
+ return tuple(str(model) for model in listed)
+ except TypeError:
+ _log_unreadable_listing(provider, kind, listed)
+ return None
+
+
def credentials_for(
enhancement: ModuleType,
stored: Mapping[str, Any],
@@ -84,6 +130,54 @@ def runtime_kwargs_for(
return dict(build(stored)) if callable(build) else {}
+def embedding_concurrency_for(
+ provider: str,
+ stored: Mapping[str, Any] | None,
+) -> int | None:
+ """How many embedding calls the gateway may have in flight to `provider`.
+
+ The optional `embedding_max_concurrency(stored)` member of the enhancement
+ contract. None means unlimited: the provider has no enhancement, the
+ enhancement sets no limit, or the hook failed. A broken hook must not fail
+ the request it would only have throttled, so errors are logged and ignored.
+ """
+ enhancement = get(provider)
+ hook = getattr(enhancement, "embedding_max_concurrency", None)
+ if not callable(hook):
+ return None
+ try:
+ limit = hook(stored or {})
+ except Exception as exc:
+ _log_hook_failure(provider, "embedding_max_concurrency", exc)
+ return None
+ if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
+ return None
+ return limit
+
+
+def _log_hook_failure(provider: str, hook: str, exc: Exception) -> None:
+ from utils.logging_config import get_logger
+
+ get_logger(__name__).warning(
+ "A provider enhancement hook failed; continuing without it",
+ provider=provider,
+ hook=hook,
+ error_type=type(exc).__name__,
+ error=str(exc),
+ )
+
+
+def _log_unreadable_listing(provider: str, kind: str, listed: object) -> None:
+ from utils.logging_config import get_logger
+
+ get_logger(__name__).warning(
+ "A provider enhancement cached a model listing that is not a list; treating it as unknown",
+ provider=provider,
+ kind=kind,
+ listing_type=type(listed).__name__,
+ )
+
+
@functools.cache
def _accepts_kind(enhancement: ModuleType) -> bool:
"""Whether the module's `litellm_credentials` takes a `kind` keyword.
diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json
index 357649a41..9db862dcb 100644
--- a/flows/ingestion_flow.json
+++ b/flows/ingestion_flow.json
@@ -217,7 +217,7 @@
"last_updated": "2026-06-02T20:40:57.828Z",
"legacy": false,
"metadata": {
- "code_hash": "dcf9aeedd059",
+ "code_hash": "7b3186b7fe33",
"dependencies": {
"dependencies": [
{
@@ -363,7 +363,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path # noqa: TC003\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data, dotdict\nfrom lfx.utils.util import transform_localhost_url\nfrom pydantic import ValidationError\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = (\n \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n )\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n StrInput(\n name=\"verify_ssl\",\n display_name=\"Verify SSL\",\n info=\"Whether to verify SSL certificates for Docling Serve.\",\n value=\"DOCLING_SERVE_VERIFY_SSL\",\n load_from_db=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(\n f\"The status requests got a http response {response.status_code} too many times.\"\n )\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if result.get(\"status\") == \"failure\" or result.get(\"errors\"):\n errors = result.get(\"errors\", [])\n err_msg_list = []\n for err in errors:\n if isinstance(err, dict) and \"error_message\" in err:\n err_msg_list.append(err[\"error_message\"])\n elif isinstance(err, str):\n err_msg_list.append(err)\n\n err_details = (\n \"; \".join(err_msg_list) if err_msg_list else \"Unknown Docling processing error\"\n )\n\n msg = f\"Docling processing failed: {err_details}\"\n raise ValueError(msg)\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _get_verify_ssl(self) -> bool:\n \"\"\"Determine whether to verify SSL certificates for Docling Serve.\n\n Returns:\n bool: True if SSL verification should be enforced, False otherwise.\n \"\"\"\n verify = getattr(self, \"verify_ssl\", None)\n if hasattr(verify, \"get_secret_value\"):\n verify = verify.get_secret_value()\n if hasattr(verify, \"text\"):\n verify = verify.text\n if isinstance(verify, bool):\n return verify\n val_str = str(verify or \"\").strip().lower()\n if val_str in (\"false\", \"0\", \"no\"):\n return False\n return True\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(\n self, file_list: list[BaseFileComponent.BaseFile]\n ) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(\n client: httpx.Client, file_path: Path, options: dict[str, Any]\n ) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [\n {\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}\n ],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = [None] * len(file_list)\n with (\n httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is not None:\n futures.append(\n (i, executor.submit(_convert_document, client, file.path, docling_options))\n )\n\n for idx, future in futures:\n try:\n processed_data[idx] = future.result()\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"
+ "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path # noqa: TC003\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data, dotdict\nfrom lfx.utils.util import transform_localhost_url\nfrom pydantic import ValidationError\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = (\n \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n )\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n StrInput(\n name=\"verify_ssl\",\n display_name=\"Verify SSL\",\n info=\"Whether to verify SSL certificates for Docling Serve.\",\n value=\"DOCLING_SERVE_VERIFY_SSL\",\n load_from_db=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _raise_if_task_missing(self, response: httpx.Response, task_id: str) -> None:\n \"\"\"Raise a clear error when Docling Serve no longer knows the task.\n\n Docling Serve deletes a finished task some minutes after its result is\n first fetched, and forgets every task when it restarts. Polling such a\n task returns 404, which would otherwise surface as a bare HTTP error.\n \"\"\"\n if response.status_code != httpx.codes.NOT_FOUND:\n return\n msg = (\n f\"Docling task {task_id} was not found on the Docling Serve server. Its result \"\n \"may have expired (Docling Serve deletes finished results after a delay) or the \"\n \"server restarted. Re-ingest the file.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n self._raise_if_task_missing(response, task_id)\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(\n f\"The status requests got a http response {response.status_code} too many times.\"\n )\n return None\n continue\n\n self._raise_if_task_missing(response, task_id)\n response.raise_for_status()\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if result.get(\"status\") == \"failure\" or result.get(\"errors\"):\n errors = result.get(\"errors\", [])\n err_msg_list = []\n for err in errors:\n if isinstance(err, dict) and \"error_message\" in err:\n err_msg_list.append(err[\"error_message\"])\n elif isinstance(err, str):\n err_msg_list.append(err)\n\n err_details = (\n \"; \".join(err_msg_list) if err_msg_list else \"Unknown Docling processing error\"\n )\n\n msg = f\"Docling processing failed: {err_details}\"\n raise ValueError(msg)\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _get_verify_ssl(self) -> bool:\n \"\"\"Determine whether to verify SSL certificates for Docling Serve.\n\n Returns:\n bool: True if SSL verification should be enforced, False otherwise.\n \"\"\"\n verify = getattr(self, \"verify_ssl\", None)\n if hasattr(verify, \"get_secret_value\"):\n verify = verify.get_secret_value()\n if hasattr(verify, \"text\"):\n verify = verify.text\n if isinstance(verify, bool):\n return verify\n val_str = str(verify or \"\").strip().lower()\n if val_str in (\"false\", \"0\", \"no\"):\n return False\n return True\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(\n self, file_list: list[BaseFileComponent.BaseFile]\n ) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(\n client: httpx.Client, file_path: Path, options: dict[str, Any]\n ) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [\n {\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}\n ],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = [None] * len(file_list)\n with (\n httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is not None:\n futures.append(\n (i, executor.submit(_convert_document, client, file.path, docling_options))\n )\n\n for idx, future in futures:\n try:\n processed_data[idx] = future.result()\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"
},
"delete_server_file_after_processing": {
"_input_type": "BoolInput",
diff --git a/frontend/app/api/queries/useDoclingHealthQuery.test.ts b/frontend/app/api/queries/useDoclingHealthQuery.test.ts
new file mode 100644
index 000000000..0164d02ba
--- /dev/null
+++ b/frontend/app/api/queries/useDoclingHealthQuery.test.ts
@@ -0,0 +1,85 @@
+import { HttpResponse, http } from "msw";
+import { describe, expect, it } from "vitest";
+import { server } from "@/test-utils/msw/server";
+import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render";
+import { useDoclingHealthQuery } from "./useDoclingHealthQuery";
+
+function renderDoclingHealth() {
+ return renderHook(() => useDoclingHealthQuery({ retry: false }), {
+ wrapper: createQueryWrapper(),
+ });
+}
+
+describe("useDoclingHealthQuery", () => {
+ it("reports healthy on a 200 healthy body", async () => {
+ server.use(
+ http.get("/api/docling/health", () =>
+ HttpResponse.json({ status: "healthy", host: "localhost" }),
+ ),
+ );
+
+ const { result } = renderDoclingHealth();
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(result.current.data).toEqual({ status: "healthy" });
+ });
+
+ it("reports degraded when docling-serve is slow to respond", async () => {
+ server.use(
+ http.get("/api/docling/health", () =>
+ HttpResponse.json({
+ status: "degraded",
+ message: "Docling Serve is slow to respond",
+ host: "localhost",
+ }),
+ ),
+ );
+
+ const { result } = renderDoclingHealth();
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(result.current.data).toEqual({
+ status: "degraded",
+ message: "Docling Serve is slow to respond",
+ });
+ });
+
+ it("treats a 200 without a JSON body as healthy", async () => {
+ server.use(
+ http.get(
+ "/api/docling/health",
+ () => new HttpResponse(null, { status: 200 }),
+ ),
+ );
+
+ const { result } = renderDoclingHealth();
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(result.current.data).toEqual({ status: "healthy" });
+ });
+
+ it("reports unhealthy on a 503", async () => {
+ server.use(
+ http.get("/api/docling/health", () =>
+ HttpResponse.json(
+ { status: "unhealthy", message: "Connection timeout" },
+ { status: 503 },
+ ),
+ ),
+ );
+
+ const { result } = renderDoclingHealth();
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(result.current.data?.status).toBe("unhealthy");
+ });
+
+ it("reports backend-unavailable when the request itself fails", async () => {
+ server.use(http.get("/api/docling/health", () => HttpResponse.error()));
+
+ const { result } = renderDoclingHealth();
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(result.current.data?.status).toBe("backend-unavailable");
+ });
+});
diff --git a/frontend/app/api/queries/useDoclingHealthQuery.ts b/frontend/app/api/queries/useDoclingHealthQuery.ts
index 5e77eecc1..e9acb4ddb 100644
--- a/frontend/app/api/queries/useDoclingHealthQuery.ts
+++ b/frontend/app/api/queries/useDoclingHealthQuery.ts
@@ -5,7 +5,9 @@ import {
} from "@tanstack/react-query";
export interface DoclingHealthResponse {
- status: "healthy" | "unhealthy" | "backend-unavailable";
+ // "degraded": docling-serve is up but slow to answer (busy converting), so
+ // ingest still works and the "stopped" banner must not show.
+ status: "healthy" | "degraded" | "unhealthy" | "backend-unavailable";
message?: string;
}
@@ -19,6 +21,10 @@ async function checkDoclingHealth(): Promise<DoclingHealthResponse> {
});
if (response.ok) {
+ const body = await response.json().catch(() => ({}));
+ if (body?.status === "degraded") {
+ return { status: "degraded", message: body.message };
+ }
return { status: "healthy" };
} else if (response.status === 503) {
return {
@@ -53,8 +59,10 @@ export const useDoclingHealthQuery = (
queryFn: checkDoclingHealth,
retry: 1,
refetchInterval: (query) => {
- // If healthy, check every 30 seconds; otherwise check every 3 seconds
- return query.state.data?.status === "healthy" ? 30000 : 3000;
+ // Up (healthy or busy): check every 30 seconds; down: every 3 seconds.
+ // Polling a busy docling-serve fast would only add to its load.
+ const status = query.state.data?.status;
+ return status === "healthy" || status === "degraded" ? 30000 : 3000;
},
refetchOnWindowFocus: false, // Disabled to reduce unnecessary calls on tab switches
refetchOnMount: true,
diff --git a/frontend/app/api/queries/useProviderHealthQuery.test.tsx b/frontend/app/api/queries/useProviderHealthQuery.test.tsx
index 60e582e0f..8587502a7 100644
--- a/frontend/app/api/queries/useProviderHealthQuery.test.tsx
+++ b/frontend/app/api/queries/useProviderHealthQuery.test.tsx
@@ -112,6 +112,39 @@ describe("useProviderHealthQuery", () => {
expect(result.current.data?.llm_error).toBe("Invalid API Key");
});
+ it("keeps the warnings a 503 carries beside its errors", async () => {
+ const warning = {
+ code: "stale_embedding_space",
+ provider: "rhoai",
+ message:
+ "Documents are indexed with 'old', which this endpoint no longer serves",
+ };
+ server.use(
+ http.get("/api/settings", () =>
+ HttpResponse.json(makeSettings({ edited: true })),
+ ),
+ http.get(
+ "/api/provider/health",
+ () =>
+ new HttpResponse(
+ JSON.stringify({
+ message: "Provider validation failed",
+ embedding_error: "endpoint unreachable",
+ warnings: [warning],
+ }),
+ { status: 503 },
+ ),
+ ),
+ );
+
+ const { result } = renderHook(() => useProviderHealthQuery(), {
+ wrapper: wrapper(),
+ });
+
+ await waitFor(() => expect(result.current.data?.status).toBe("unhealthy"));
+ expect(result.current.data?.warnings).toEqual([warning]);
+ });
+
it("maps any other non-ok status to an error status", async () => {
server.use(
http.get("/api/settings", () =>
diff --git a/frontend/app/api/queries/useProviderHealthQuery.ts b/frontend/app/api/queries/useProviderHealthQuery.ts
index a35b1a74f..f62992be9 100644
--- a/frontend/app/api/queries/useProviderHealthQuery.ts
+++ b/frontend/app/api/queries/useProviderHealthQuery.ts
@@ -14,6 +14,20 @@ export interface ProviderHealthDetails {
endpoint?: string | null;
}
+/**
+ * Something degraded that is not a provider failure, reported beside the
+ * verdict rather than in it — e.g. `stale_embedding_space`: documents indexed
+ * with a model the provider no longer serves. The fix is in the corpus, not in
+ * provider settings.
+ */
+export interface ProviderHealthWarning {
+ code: string;
+ provider: string;
+ message: string;
+ models?: string[];
+ served?: string[];
+}
+
export interface ProviderHealthResponse {
status: "healthy" | "unhealthy" | "error" | "backend-unavailable";
message: string;
@@ -23,6 +37,7 @@ export interface ProviderHealthResponse {
llm_error?: string | null;
embedding_error?: string | null;
details?: ProviderHealthDetails;
+ warnings?: ProviderHealthWarning[];
}
export interface ProviderHealthParams {
@@ -97,6 +112,7 @@ export const useProviderHealthQuery = (
llm_error: errorData.llm_error,
embedding_error: errorData.embedding_error,
details: errorData.details,
+ warnings: errorData.warnings,
};
} else {
const errorData = await response.json().catch(() => ({}));
diff --git a/frontend/components/layout-wrapper.tsx b/frontend/components/layout-wrapper.tsx
index f07cf2468..6ec84ba9e 100644
--- a/frontend/components/layout-wrapper.tsx
+++ b/frontend/components/layout-wrapper.tsx
@@ -72,7 +72,8 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
});
const { isUnhealthy: isDoclingUnhealthy } = useDoclingHealth();
- const { isUnhealthy: isProviderUnhealthy } = useProviderHealth();
+ const { isUnhealthy: isProviderUnhealthy, isDegraded: isProviderDegraded } =
+ useProviderHealth();
if (isAuthPage) {
return <div className="h-full">{children}</div>;
@@ -120,7 +121,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
{settings?.edited && isOnboardingComplete && (
<AnimatedConditional
vertical
- isOpen={isProviderUnhealthy}
+ isOpen={isProviderUnhealthy || isProviderDegraded}
className="w-full"
>
<ProviderHealthBanner />
diff --git a/frontend/components/provider-health-banner.test.tsx b/frontend/components/provider-health-banner.test.tsx
index a41f5a5cf..3055423d2 100644
--- a/frontend/components/provider-health-banner.test.tsx
+++ b/frontend/components/provider-health-banner.test.tsx
@@ -67,4 +67,60 @@ describe("ProviderHealthBanner", () => {
screen.getByText(/OpenAI error - Invalid API Key/i),
).toBeInTheDocument();
});
+
+ const staleSpace = {
+ code: "stale_embedding_space",
+ provider: "rhoai",
+ message:
+ "Documents are indexed with 'old-model', which this endpoint no longer serves",
+ };
+
+ function mockHealth(data: object) {
+ vi.mocked(useProviderHealthQuery).mockReturnValue({
+ data,
+ isLoading: false,
+ isFetching: false,
+ error: null,
+ isError: false,
+ } as unknown as ReturnType<typeof useProviderHealthQuery>);
+ }
+
+ it("renders a healthy provider with warnings as a warning that points at Knowledge", async () => {
+ const user = userEvent.setup();
+ mockPush.mockClear();
+ mockHealth({ status: "healthy", message: "ok", warnings: [staleSpace] });
+
+ render(<ProviderHealthBanner />);
+
+ expect(
+ screen.getByText(/warning - Documents are indexed with 'old-model'/i),
+ ).toBeInTheDocument();
+ expect(screen.queryByRole("button", { name: "Fix Setup" })).toBeNull();
+ await user.click(screen.getByRole("button", { name: "Open Knowledge" }));
+ expect(mockPush).toHaveBeenCalledWith("/knowledge");
+ });
+
+ it("lets an unhealthy verdict outrank a warning", () => {
+ mockHealth({
+ status: "unhealthy",
+ provider: "openai",
+ llm_error: "Invalid API Key",
+ warnings: [staleSpace],
+ });
+
+ render(<ProviderHealthBanner />);
+
+ expect(
+ screen.getByText(/OpenAI error - Invalid API Key/i),
+ ).toBeInTheDocument();
+ expect(screen.queryByRole("button", { name: "Open Knowledge" })).toBeNull();
+ });
+
+ it("renders nothing for a healthy provider without warnings", () => {
+ mockHealth({ status: "healthy", message: "ok", warnings: [] });
+
+ const { container } = render(<ProviderHealthBanner />);
+
+ expect(container).toBeEmptyDOMElement();
+ });
});
diff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsx
index f026ec9b0..85e9772e4 100644
--- a/frontend/components/provider-health-banner.tsx
+++ b/frontend/components/provider-health-banner.tsx
@@ -36,6 +36,9 @@ export function useProviderHealth() {
health?.status === "unhealthy" || health?.status === "error";
const isBackendUnavailable =
health?.status === "backend-unavailable" || isError;
+ // Serving, but something outside provider setup needs attention. An
+ // unhealthy verdict outranks it, so the two banners never compete.
+ const isDegraded = isHealthy && (health?.warnings?.length ?? 0) > 0;
return {
health,
@@ -45,20 +48,61 @@ export function useProviderHealth() {
isError,
isHealthy,
isUnhealthy,
+ isDegraded,
isBackendUnavailable,
};
}
export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {
- const { isLoading, isHealthy, isUnhealthy, health } = useProviderHealth();
+ const { isLoading, isHealthy, isUnhealthy, isDegraded, health } =
+ useProviderHealth();
const router = useRouter();
const isNarrow = useNarrowLayout();
- // Only show banner when provider is unhealthy (not when backend is unavailable)
- if (isLoading || isHealthy) {
+ // Only show banner when provider is unhealthy or degraded (not when the
+ // backend is unavailable)
+ if (isLoading || (isHealthy && !isDegraded)) {
return null;
}
+ if (isDegraded) {
+ // A warning, not an error: the provider is serving and its setup is fine.
+ // The remedy (re-ingest or delete) lives on the Knowledge page, so that is
+ // where the action goes rather than to provider settings.
+ const [warning] = health?.warnings ?? [];
+ const providerTitle = getProviderChrome(warning.provider).name;
+
+ return (
+ <Banner
+ className={cn(
+ "bg-amber-50 dark:bg-amber-950 text-foreground border-accent-amber border-b w-full",
+ isNarrow && "flex-wrap gap-y-1 py-2",
+ className,
+ )}
+ >
+ <BannerIcon
+ className="text-accent-amber-foreground shrink-0"
+ icon={AlertTriangle}
+ />
+ <BannerTitle
+ className={cn(
+ "font-medium flex items-center gap-2",
+ isNarrow && "text-xs",
+ )}
+ >
+ {`${providerTitle} warning - ${warning.message}`}
+ </BannerTitle>
+ <Button
+ size="sm"
+ className="shrink-0"
+ onClick={() => router.push("/knowledge")}
+ >
+ Open Knowledge
+ </Button>
+ </Banner>
+ );
+ }
+
if (isUnhealthy) {
const llmProvider = health?.llm_provider || health?.provider;
const embeddingProvider = health?.embedding_provider;
diff --git a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md b/kubernetes/operator/internal/controller/ENV_OVERRIDE.md
index 359345b38..c9e699645 100644
--- a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md
+++ b/kubernetes/operator/internal/controller/ENV_OVERRIDE.md
@@ -40,7 +40,7 @@ Environment variables are merged using the following priority (highest to lowest
│ Priority Level 1 (Lowest): Hardcoded Defaults │
│ ┌─────────────────────────────────────────────────────────┐ │
│ │ env.go DefaultLangflowEnvVars: │ │
-│ │ "LANGFLOW_LOG_LEVEL": "DEBUG" │ │
+│ │ "LANGFLOW_LOG_LEVEL": "INFO" │ │
│ │ "LANGFLOW_WORKERS": "4" │ │
│ │ "LANGFLOW_AUTO_LOGIN": "true" │ │
│ └─────────────────────────────────────────────────────────┘ │
diff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.go
index f75cf5b8f..1f2575f32 100644
--- a/kubernetes/operator/internal/controller/env.go
+++ b/kubernetes/operator/internal/controller/env.go
@@ -45,7 +45,7 @@ func NewEnvVarManager() *EnvVarManager {
// Langflow runtime configuration
"LANGFLOW_WORKERS": "4",
"LANGFLOW_CONFIG_DIR": "/tmp",
- "LANGFLOW_LOG_LEVEL": "DEBUG",
+ "LANGFLOW_LOG_LEVEL": "INFO",
"HIDE_GETTING_STARTED_PROGRESS": "true",
"LANGFLOW_ALEMBIC_LOG_TO_STDOUT": "true",
"LANGFLOW_DEACTIVATE_TRACING": "true",
diff --git a/src/api/docling.py b/src/api/docling.py
index 40c860b6a..dd405114f 100644
--- a/src/api/docling.py
+++ b/src/api/docling.py
@@ -21,6 +21,10 @@ logger = get_logger(__name__)
DOCLING_SERVICE_URL = DOCLING_SERVE_URL
HOST_IP = DOCLING_HOST_IP
+# Per-phase httpx budget, matching the console status check. A single-worker
+# docling-serve converting documents can stall its HTTP endpoints for seconds.
+_HEALTH_CHECK_TIMEOUT_S = 5.0
+
async def health(
request: Request, user: Annotated[User | None, Depends(get_optional_user)] = None
@@ -37,7 +41,9 @@ async def health(
try:
async with httpx.AsyncClient(verify=DOCLING_SERVE_VERIFY_SSL) as client:
- response = await client.get(health_url, headers=headers, timeout=2.0)
+ response = await client.get(
+ health_url, headers=headers, timeout=_HEALTH_CHECK_TIMEOUT_S
+ )
if response.status_code == 200:
return JSONResponse({"status": "healthy", "host": HOST_IP})
@@ -54,12 +60,29 @@ async def health(
status_code=503,
)
- except httpx.TimeoutException:
- logger.warning("Docling health check timeout", url=health_url)
+ except httpx.ConnectTimeout:
+ logger.warning("Docling health check connect timeout", url=health_url)
return JSONResponse(
{"status": "unhealthy", "message": "Connection timeout", "host": HOST_IP},
status_code=503,
)
+ except httpx.TimeoutException as e:
+ # The connection opened but the reply was slow: docling-serve is busy, not
+ # stopped, and ingest still works. Answer 200 so the UI doesn't announce
+ # that the service is stopped; the body says it is degraded.
+ logger.warning(
+ "Docling health check slow to respond",
+ url=health_url,
+ error_type=type(e).__name__,
+ timeout_s=_HEALTH_CHECK_TIMEOUT_S,
+ )
+ return JSONResponse(
+ {
+ "status": "degraded",
+ "message": "Docling Serve is slow to respond",
+ "host": HOST_IP,
+ }
+ )
except Exception as e:
logger.error("Docling health check failed", url=health_url, error=str(e))
return JSONResponse(
diff --git a/src/api/provider_health.py b/src/api/provider_health.py
index 579e989bd..5497bf151 100644
--- a/src/api/provider_health.py
+++ b/src/api/provider_health.py
@@ -1,12 +1,17 @@
"""Provider health check endpoint."""
import asyncio
+from typing import Any
import httpx
from fastapi import Depends
from fastapi.responses import JSONResponse
-from api.provider_validation import sanitize_provider_error_content, validate_provider_setup
+from api.provider_validation import (
+ ProbeResult,
+ sanitize_provider_error_content,
+ validate_provider_setup,
+)
from config.settings import get_openrag_config
from dependencies import require_permission
from services import provider_error_log
@@ -17,6 +22,202 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+#: `warnings[].code` for indexed vectors whose model the provider no longer serves.
+STALE_EMBEDDING_SPACE = "stale_embedding_space"
+
+
+#: Ceiling on re-listing a provider's models from inside a health check. The
+#: listing is only an input to a diagnostic, so a slow cluster costs this at
+#: most; the probes report an unreachable one in their own words.
+_MODEL_REFRESH_TIMEOUT_SECONDS = 5.0
+
+
+async def _refresh_live_models(provider: str) -> None:
+ """Re-list what `provider` serves, if it can say, best effort.
+
+ Only the provider being checked: the listing of any other would not be
+ read, and waiting on its cluster would only slow this one's verdict. A
+ provider with no enhancement, or one that cannot list its own models, has
+ nothing to refresh, and is skipped without a call.
+
+ TTL-guarded inside each enhancement, so this is a network call once every
+ few minutes rather than once per poll — while the cluster answers. A failed
+ listing is not cached, so an unreachable cluster is asked again on every
+ poll; hence the timeout. A failure or timeout leaves the previous answer in
+ place and must never fail the health check.
+ """
+ try:
+ from enhancements.providers.registry import get as get_enhancement
+ from services.model_catalog import refresh_live_models
+
+ enhancement = get_enhancement(provider)
+ if enhancement is None or not hasattr(enhancement, "fetch_models"):
+ return
+ await asyncio.wait_for(
+ refresh_live_models(provider), timeout=_MODEL_REFRESH_TIMEOUT_SECONDS
+ )
+ except TimeoutError:
+ logger.debug(
+ "Timed out refreshing live model listing",
+ provider=provider,
+ timeout_seconds=_MODEL_REFRESH_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ logger.debug("Could not refresh live model listings", provider=provider, exc_info=True)
+
+
+#: Spaces per aggregation page — the same page size retrieval discovers them
+#: with. A corpus has a handful: one per embedding model it was indexed with.
+_EMBEDDING_SPACE_PAGE_SIZE = 100
+
+#: Pages read before giving up. Far past any real corpus; it bounds what one
+#: health poll can cost if the index is somehow full of distinct spaces.
+_EMBEDDING_SPACE_MAX_PAGES = 10
+
+
+async def _indexed_spaces(provider: str) -> tuple[str, ...] | None:
+ """Embedding models this corpus holds vectors for, under `provider`.
+
+ None means unknown — OpenSearch unreachable, nothing indexed, or an answer
+ that may be incomplete. A partial list is never returned as if it were the
+ whole: a space missing from it is a stale space that goes unreported. So
+ every page of the composite aggregation is read, shard failures raise
+ rather than quietly dropping buckets, and running out of pages before the
+ cursor does is treated as not knowing.
+
+ Read with the admin client, not a user-scoped one, because this is a
+ property of the corpus rather than of whoever is looking; DLS would
+ silently shrink it.
+ """
+ try:
+ from config.settings import clients, get_index_name
+ from utils.embedding_fields import (
+ build_embedding_space_aggregation,
+ embedding_space_after_keys,
+ embedding_spaces_from_aggregation,
+ split_embedding_space_id,
+ )
+
+ client = clients.opensearch
+ if client is None:
+ return None
+ space_ids: list[str] = []
+ after: dict[str, Any] | None = None
+ for _ in range(_EMBEDDING_SPACE_MAX_PAGES):
+ result = await client.search(
+ index=get_index_name(),
+ body={
+ "size": 0,
+ # Qualified spaces only. A legacy space records no provider,
+ # so it cannot be attributed to this one, and it is governed
+ # by OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP rather than by what
+ # the cluster serves today.
+ "aggs": build_embedding_space_aggregation(
+ size=_EMBEDDING_SPACE_PAGE_SIZE,
+ qualified_after=after,
+ include_legacy=False,
+ ),
+ },
+ # A failed shard must fail the read, not shrink its buckets.
+ params={"allow_partial_search_results": "false"},
+ )
+ if result.get("timed_out"):
+ logger.debug("Reading indexed embedding spaces timed out")
+ return None
+ space_ids.extend(space.space_id for space in embedding_spaces_from_aggregation(result))
+ # Raw buckets, not parsed spaces: the parser skips blank ids, and a
+ # page it thinned out is not a short one.
+ buckets = (
+ result.get("aggregations", {}).get("embedding_spaces", {}).get("buckets") or []
+ )
+ next_after, _ = embedding_space_after_keys(result)
+ # A short page is the last one; a composite aggregation still hands
+ # back an `after_key` for it, and following it costs an empty read.
+ if len(buckets) < _EMBEDDING_SPACE_PAGE_SIZE or not next_after or next_after == after:
+ break
+ after = next_after
+ else:
+ logger.debug(
+ "Indexed embedding spaces exceed the read limit",
+ pages=_EMBEDDING_SPACE_MAX_PAGES,
+ page_size=_EMBEDDING_SPACE_PAGE_SIZE,
+ )
+ return None
+ except Exception:
+ logger.debug("Could not read indexed embedding spaces", exc_info=True)
+ return None
+
+ key = (provider or "").strip().lower()
+ models = []
+ for space_id in space_ids:
+ space_provider, model = split_embedding_space_id(space_id)
+ if space_provider == key and model:
+ models.append(model)
+ return tuple(dict.fromkeys(models)) or None
+
+
+async def _stale_embedding_spaces(provider: str) -> dict[str, Any] | None:
+ """Indexed vector spaces whose model the provider has stopped serving.
+
+ Nothing re-checks a corpus once it is indexed. A chunk records the space it
+ was embedded in (`provider:model`), retrieval embeds the query once per
+ space it finds there, and an `InferenceService` redeployed under a new
+ `--served-model-name` leaves every earlier chunk pointing at a model that
+ is gone. No save can be rejected over it and no probe reaches it — the
+ *configured* model is fine. Retrieval skips a space it cannot embed for and
+ carries on, so nothing fails outright: those documents quietly stop being
+ found by meaning, and the only trace is a 502 quoting a model name that
+ appears nowhere in Settings.
+
+ A warning, never an error: the provider is serving and nothing in its setup
+ is wrong. The remedy is in the corpus — re-ingest or delete — so reporting
+ it as a provider failure would send the operator to the one screen that
+ cannot fix it.
+
+ Silent unless both halves are known: what the provider serves, and what the
+ corpus holds. Neither absence is evidence of the other. "Known" is
+ `served is not None`: an empty listing is a provider that answered and
+ serves no embedding model, which makes every indexed space stale.
+ """
+ try:
+ from enhancements.providers.registry import live_models_for
+
+ served = live_models_for(provider, "embedding")
+ except Exception:
+ logger.debug("Could not read live models for %s", provider, exc_info=True)
+ return None
+ if served is None:
+ return None
+
+ indexed = await _indexed_spaces(provider)
+ if not indexed:
+ return None
+ stale = [model for model in indexed if model not in served]
+ if not stale:
+ return None
+ serving = f"it serves: {', '.join(served)}" if served else "it lists no embedding models at all"
+ # Debug, not warning: the banner polls this every 30s per open tab, and the
+ # condition lasts until someone re-ingests. The response carries it to the
+ # user; the log is for tracing why.
+ logger.debug(
+ "Indexed embedding spaces are no longer served by the provider",
+ provider=provider,
+ stale=stale,
+ served=list(served),
+ )
+ return {
+ "code": STALE_EMBEDDING_SPACE,
+ "provider": provider,
+ "models": stale,
+ "served": list(served),
+ "message": (
+ f"Documents are indexed with {', '.join(repr(model) for model in stale)}, which "
+ f"this endpoint no longer serves ({serving}). Search skips "
+ "those vectors, so the documents are found by keyword only until they are "
+ "re-ingested or deleted."
+ ),
+ }
+
async def check_provider_health(
provider: str | None = None,
@@ -255,7 +456,7 @@ async def check_provider_health(
# Validate LLM provider
try:
- await validate_provider_setup(
+ llm_probe: ProbeResult = await validate_provider_setup(
provider=provider,
api_key=api_key,
llm_model=llm_model,
@@ -276,6 +477,23 @@ async def check_provider_health(
except Exception as e:
llm_error = sanitize_provider_error_content(e)
logger.error(f"LLM provider ({provider}) validation failed: {llm_error}")
+ else:
+ # Without this the banner can only be cleared by chat traffic:
+ # one failed turn latches it, the frontend then polls every 5s
+ # with `test_completion` while it stays latched, and a provider
+ # that has recovered keeps being reported broken until the entry
+ # goes stale 15 minutes later.
+ #
+ # But a probe cannot reproduce the request that failed (see
+ # `provider_error_log`), so only one that sent the same shape
+ # of request may speak for it. A passing validation can mean a
+ # deployment listing (Azure), no call at all (no model set), or
+ # a tool-less completion (the LiteLLM probe) — none of which
+ # reaches the tool-calling failures agent traffic actually hits.
+ # Clearing on those would drop the banner while chat is still
+ # broken, only for the next turn to raise it again.
+ if test_completion and llm_probe.model_probed and llm_probe.tools_exercised:
+ provider_error_log.record_success(provider, "chat")
# Validate embedding provider
# For WatsonX with test_completion=True, wait 2 seconds between completion and embedding tests
@@ -291,7 +509,7 @@ async def check_provider_health(
await asyncio.sleep(2)
try:
- await validate_provider_setup(
+ embedding_probe: ProbeResult = await validate_provider_setup(
provider=embedding_provider,
api_key=embedding_api_key,
embedding_model=embedding_model,
@@ -316,6 +534,12 @@ async def check_provider_health(
logger.error(
f"Embedding provider ({embedding_provider}) validation failed: {embedding_error}"
)
+ else:
+ # An embedding request has no shape beyond model and input, so a
+ # real call to the configured model is the same request traffic
+ # makes. Anything short of one proves nothing about it.
+ if test_completion and embedding_probe.model_probed:
+ provider_error_log.record_success(embedding_provider, "embedding")
# A real call beats a probe. The probe sends its own request, so it
# hits its own failure: OpenAI checks request shape before billing,
@@ -330,6 +554,14 @@ async def check_provider_health(
or embedding_error
)
+ # Nothing above looks at the corpus, and a vector space outlives
+ # the model that made it. Reported beside the verdict, never in it:
+ # a stale space degrades search but says nothing about whether the
+ # provider is serving, so it must not turn the status code.
+ await _refresh_live_models(embedding_provider)
+ stale_spaces = await _stale_embedding_spaces(embedding_provider)
+ warnings = [stale_spaces] if stale_spaces else []
+
# Return combined status
if llm_error or embedding_error:
errors = []
@@ -349,6 +581,7 @@ async def check_provider_health(
"embedding_provider": embedding_provider,
"llm_error": llm_error,
"embedding_error": embedding_error,
+ "warnings": warnings,
},
status_code=503,
)
@@ -362,6 +595,7 @@ async def check_provider_health(
"llm_model": llm_model,
"embedding_model": embedding_model,
},
+ "warnings": warnings,
}
provider_health_cache.set_and_release(health_cache_key, healthy_payload)
_health_leader_key = None
diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py
index c9afe1fc1..dce10af92 100644
--- a/src/api/provider_validation.py
+++ b/src/api/provider_validation.py
@@ -4,6 +4,7 @@ import asyncio
import json
import re
from collections.abc import Mapping
+from dataclasses import dataclass
from typing import Any
from urllib.parse import urlparse
@@ -155,6 +156,87 @@ def is_provider_tls_error(text: str | BaseException | None) -> bool:
return any(marker in lowered for marker in _PROVIDER_TLS_ERROR_MARKERS)
+#: Markers for a response the connection dropped before it finished: the server
+#: promised more body than arrived. Nearly always something between OpenRAG and
+#: the model server closing a long response — an authenticating proxy with a
+#: fixed request timeout (RHOAI's kube-rbac-proxy cuts at 30s), a load balancer
+#: idle timeout. aiohttp names it with a "400" of its own, which reads like the
+#: model rejecting the request.
+_PROVIDER_TRUNCATED_RESPONSE_MARKERS = (
+ "transferencodingerror",
+ "response payload is not completed",
+ "not enough data to satisfy transfer length",
+ "incomplete chunked read",
+)
+
+
+def is_provider_truncated_response_error(text: str | BaseException | None) -> bool:
+ """True when the provider's response was cut off mid-transfer."""
+ if text is None:
+ return False
+ lowered = (str(text) if not isinstance(text, str) else text).lower()
+ return any(marker in lowered for marker in _PROVIDER_TRUNCATED_RESPONSE_MARKERS)
+
+
+#: How deep to follow `__cause__` / `__context__`. LiteLLM wraps the transport
+#: error three times (its exception, the provider's, `httpx.ReadError`).
+_MAX_EXCEPTION_CHAIN_DEPTH = 8
+
+
+def is_provider_stale_connection_error(exc: BaseException | None) -> bool:
+ """True when the connection closed before any response to the request began.
+
+ Nearly always a pooled keep-alive connection the model server had already
+ closed: LiteLLM keeps idle connections for 120s, vLLM (uvicorn) closes them
+ after 5s, and a request sent while the close is still in flight finds a
+ dead socket. The server never saw the request, so sending it again on a
+ fresh connection is safe. Matched on the transport's own exception type,
+ not its text: a response cut off mid-body raises a different error
+ (`is_provider_truncated_response_error`) and must not be replayed.
+ """
+ disconnect_types: tuple[type[BaseException], ...] = ()
+ try:
+ from aiohttp import ServerDisconnectedError
+ except ImportError: # pragma: no cover - aiohttp ships with LiteLLM
+ pass
+ else:
+ disconnect_types = (ServerDisconnectedError,)
+
+ current = exc
+ for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
+ if current is None:
+ return False
+ if isinstance(current, disconnect_types):
+ return True
+ # The same failure on LiteLLM's httpx transport (a provider handed its
+ # own client). Other RemoteProtocolErrors can follow a partial response.
+ if isinstance(current, httpx.RemoteProtocolError) and (
+ "without sending a response" in str(current).lower()
+ ):
+ return True
+ current = current.__cause__ or current.__context__
+ return False
+
+
+#: Markers for a prompt that does not fit the model's context window. That is a
+#: property of one request — a long conversation, a lot of retrieved text — and
+#: says nothing about whether the provider is serving.
+_CONTEXT_WINDOW_ERROR_MARKERS = (
+ "maximum context length",
+ "context_length_exceeded",
+ "contextwindowexceeded",
+ "prompt is too long",
+)
+
+
+def is_context_window_error(text: str | BaseException | None) -> bool:
+ """True when the call failed because the prompt exceeds the model's context window."""
+ if text is None:
+ return False
+ lowered = (str(text) if not isinstance(text, str) else text).lower()
+ return any(marker in lowered for marker in _CONTEXT_WINDOW_ERROR_MARKERS)
+
+
_GENERIC_UPSTREAM_ERROR_MARKERS = (
"an unknown error occurred",
"an error occurred while generating a response",
@@ -662,6 +744,26 @@ def is_azure_ai_foundry_endpoint(api_base: str | None) -> bool:
return (urlparse(api_base).hostname or "").endswith(".services.ai.azure.com")
+@dataclass(frozen=True)
+class ProbeResult:
+ """What a successful `validate_provider_setup()` call actually exercised.
+
+ Passing validation is weaker evidence than it looks: depending on the
+ provider and arguments it may have listed deployments, checked a key, or
+ made no call at all. Callers deciding whether a provider is *serving* — as
+ the health check does before erasing a real-traffic failure — need to know
+ which.
+
+ ``model_probed``: a real call was made to the model under test — the
+ embedding model when one was given, otherwise the LLM.
+ ``tools_exercised``: that call was a completion carrying tool definitions,
+ the shape the agent's own traffic has.
+ """
+
+ model_probed: bool = False
+ tools_exercised: bool = False
+
+
async def validate_provider_setup(
provider: str,
api_key: str = None,
@@ -672,7 +774,8 @@ async def validate_provider_setup(
test_completion: bool = False,
credentials: dict[str, str] | None = None,
stored_credentials: Mapping[str, Any] | None = None,
-) -> None:
+ verify_model: bool = False,
+) -> ProbeResult:
"""
Validate provider setup by testing completion with tool calling and embedding.
@@ -690,6 +793,15 @@ async def validate_provider_setup(
Only a provider enhancement's lightweight check reads it: a provider with
separate chat and embedding endpoints has ``credentials`` narrowed to one
of them, and the check has to see both to probe both.
+ verify_model: Call the given model through LiteLLM even without ``test_completion``.
+ The settings save passes it: for a provider enhancement that real call
+ is the only check that rejects a model the cluster does not serve
+ before it is stored. Without it, and without ``test_completion``, an
+ enhancement gets its model-free lightweight check like any other
+ provider.
+
+ Returns:
+ ProbeResult: what the successful validation exercised.
Raises:
Exception: If validation fails, raises the original exception with the actual error message.
@@ -708,17 +820,21 @@ async def validate_provider_setup(
f"Starting validation for provider: {provider_lower} (test_completion={test_completion})"
)
- # watsonx.ai on-prem has no bespoke model probe of its own: a real call
- # through LiteLLM *is* its model probe, and it is what makes switching to
- # a model the cluster does not serve fail instead of saving silently. Its
- # catalogue check is only for the case there is no model to probe with,
- # which is the one that used to report "A model is required".
+ # A provider enhancement has no bespoke model probe of its own: a real
+ # call through LiteLLM *is* its model probe, and it is what makes
+ # switching to a model the cluster does not serve fail instead of saving
+ # silently. It runs only when asked for — on save (`verify_model`) or a
+ # full check (`test_completion`). The polled health check asks for
+ # neither and gets the model-free lightweight check, as every other
+ # provider does: polling inference every 30s holds a GPU slot (RHOAI) or
+ # bills (watsonx.ai on-prem), and its pooled connections race the
+ # model server's keep-alive and fail with "Server disconnected".
enhancement = get_provider_enhancement(provider_lower)
azure_ai_foundry_endpoint = provider_lower == "azure" and is_azure_ai_foundry_endpoint(
supplied.get("api_base")
)
probes_the_model = (
- bool(embedding_model or llm_model)
+ bool(embedding_model or llm_model) and (test_completion or verify_model)
if enhancement is not None
else (
provider_lower not in _NATIVELY_VALIDATED_PROVIDERS
@@ -742,6 +858,10 @@ async def validate_provider_setup(
embedding_model=embedding_model,
llm_model=llm_model,
)
+ # The LiteLLM probe is a plain completion: it proves the model
+ # answers, not that it can take the tool-calling request the agent
+ # sends.
+ result = ProbeResult(model_probed=True)
elif test_completion:
if provider_lower == "azure":
# Azure deployments are user-defined, so a model completion is
@@ -755,7 +875,7 @@ async def validate_provider_setup(
credentials=supplied,
stored_credentials=stored_credentials,
)
- return
+ return ProbeResult()
# Full validation with completion/embedding tests (consumes credits)
if embedding_model:
# Test embedding
@@ -766,6 +886,7 @@ async def validate_provider_setup(
endpoint=endpoint,
project_id=project_id,
)
+ result = ProbeResult(model_probed=True)
elif llm_model:
# Test completion with tool calling
await test_completion_with_tools(
@@ -775,6 +896,10 @@ async def validate_provider_setup(
endpoint=endpoint,
project_id=project_id,
)
+ result = ProbeResult(model_probed=True, tools_exercised=True)
+ else:
+ # No model to test with, so nothing was called at all.
+ result = ProbeResult()
else:
# Lightweight validation (no credits consumed)
await test_lightweight_health(
@@ -785,8 +910,10 @@ async def validate_provider_setup(
credentials=supplied,
stored_credentials=stored_credentials,
)
+ result = ProbeResult()
logger.info(f"Validation successful for provider: {provider_lower}")
+ return result
except Exception as e:
logger.error(f"Validation failed for provider {provider_lower}: {str(e)}")
@@ -805,6 +932,7 @@ async def _test_litellm_provider(
"""Validate arbitrary providers through the same LiteLLM adapter used at runtime."""
import litellm
+ from services.llm_gateway import with_stale_connection_retry
from services.model_catalog import litellm_provider_key
model = embedding_model or llm_model
@@ -813,26 +941,38 @@ async def _test_litellm_provider(
# Same aliasing the gateway applies, so the probe hits the route the real
# call will: `watsonx_onprem/<model>` is not a prefix LiteLLM can resolve.
litellm_model = f"{litellm_provider_key(provider)}/{model}"
+ # And the same retry: a dead pooled connection is not a provider failure,
+ # and reporting one would block a save or raise the banner for nothing.
if embedding_model:
- await litellm.aembedding(
+ await with_stale_connection_retry(
+ lambda: litellm.aembedding(
+ model=litellm_model,
+ # A list, never a bare string. OpenAI accepts either and LiteLLM
+ # forwards whatever it is given, so a string reaches watsonx.ai as
+ # `"inputs": "..."` where it requires `[]string` — the cluster then
+ # answers "Mismatch type []string with value string" and the
+ # pre-save check fails for every embedding model, which leaves a
+ # wrongly-chosen one impossible to change.
+ input=["OpenRAG provider validation"],
+ **credentials,
+ **(runtime_kwargs or {}),
+ ),
+ provider=provider,
model=litellm_model,
- # A list, never a bare string. OpenAI accepts either and LiteLLM
- # forwards whatever it is given, so a string reaches watsonx.ai as
- # `"inputs": "..."` where it requires `[]string` — the cluster then
- # answers "Mismatch type []string with value string" and the
- # pre-save check fails for every embedding model, which leaves a
- # wrongly-chosen one impossible to change.
- input=["OpenRAG provider validation"],
- **credentials,
- **(runtime_kwargs or {}),
+ kind="embedding",
)
return
- await litellm.acompletion(
+ await with_stale_connection_retry(
+ lambda: litellm.acompletion(
+ model=litellm_model,
+ messages=[{"role": "user", "content": "Reply with OK."}],
+ max_tokens=4,
+ **credentials,
+ **(runtime_kwargs or {}),
+ ),
+ provider=provider,
model=litellm_model,
- messages=[{"role": "user", "content": "Reply with OK."}],
- max_tokens=4,
- **credentials,
- **(runtime_kwargs or {}),
+ kind="chat",
)
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.py
index 2cd99dc3f..8f732d733 100644
--- a/src/api/settings/endpoints.py
+++ b/src/api/settings/endpoints.py
@@ -101,6 +101,7 @@ from dependencies import (
get_task_service,
require_permission,
)
+from services import provider_error_log
from services.docling_service import DoclingConfig, get_docling_preset_configs
from services.model_catalog import secret_field_keys
from services.rbac_service import is_rbac_enforced
@@ -574,6 +575,9 @@ async def update_settings(
submitted_credentials.get(llm_provider_key, {}),
remove=removals_by_provider.get(llm_provider_key, set()),
),
+ # A model the cluster does not serve must fail the
+ # save, not the first chat.
+ verify_model=True,
)
logger.info(f"LLM provider validation successful for {llm_provider}")
@@ -632,6 +636,7 @@ async def update_settings(
submitted_credentials.get(embedding_provider_key, {}),
remove=removals_by_provider.get(embedding_provider_key, set()),
),
+ verify_model=True,
)
logger.info(
f"Embedding provider validation successful for {embedding_provider}"
@@ -1111,6 +1116,10 @@ async def update_settings(
return JSONResponse({"error": "Failed to save configuration"}, status_code=500)
provider_health_cache.invalidate()
+ if should_validate or provider_updated:
+ # A recorded failure describes the provider setup that just changed.
+ # Real traffic re-raises it if the new setup fails the same way.
+ provider_error_log.clear()
# Refresh patched client immediately so subsequent requests pick up latest config.
await clients.refresh_patched_client()
@@ -1609,6 +1618,7 @@ async def onboarding(
if config_manager.save_config_file(current_config):
provider_health_cache.invalidate()
+ provider_error_log.clear()
set_fields = [k for k, v in body.model_dump(exclude_unset=True).items()]
logger.info(
"Onboarding configuration updated successfully",
diff --git a/src/api/settings/langflow_sync.py b/src/api/settings/langflow_sync.py
index 91dec1b29..775787381 100644
--- a/src/api/settings/langflow_sync.py
+++ b/src/api/settings/langflow_sync.py
@@ -226,7 +226,10 @@ async def ensure_required_langflow_global_variables(config=None):
if sync_value:
patch_payload["value"] = target_val
patch_resp = await clients.langflow_request(
- "PATCH", f"/api/v1/variables/{var_id}", json=patch_payload
+ "PATCH",
+ f"/api/v1/variables/{var_id}",
+ json=patch_payload,
+ idempotent=True,
)
if not (200 <= patch_resp.status_code < 300):
raise RuntimeError(
diff --git a/src/api/v1/llm.py b/src/api/v1/llm.py
index 8a1331a8c..3b1c56ed1 100644
--- a/src/api/v1/llm.py
+++ b/src/api/v1/llm.py
@@ -10,6 +10,7 @@ from fastapi import Depends, Request
from fastapi.responses import JSONResponse, StreamingResponse
from dependencies import require_llm_proxy_any_permission, require_llm_proxy_permission
+from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from services.llm_gateway import LlmGatewayError, chat_completions, embeddings
from services.model_catalog import (
CATALOG_UNAVAILABLE_MESSAGE,
@@ -138,8 +139,11 @@ async def embeddings_endpoint(
body = await _read_json_body(request)
if isinstance(body, JSONResponse):
return body
+ # A chat run's hop token marks query embeddings, which must not queue
+ # behind a bulk ingest at a concurrency-limited provider.
+ interactive = getattr(request.state, "llm_hop_purpose", None) == HOP_PURPOSE_CHAT
try:
- result = await embeddings(body)
+ result = await embeddings(body, interactive=interactive)
except LlmGatewayError as exc:
return _gateway_error(exc)
return JSONResponse(result)
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 3259a475b..61a2ab923 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -783,7 +783,10 @@ class ConfigManager:
# user-selected model preference. It must remain overridable after the
# settings file is marked edited so existing installations can resolve
# legacy vector spaces without modifying persisted application state.
- from config.settings import get_legacy_embedding_provider_map_json
+ from config.settings import (
+ get_legacy_embedding_provider_map_json,
+ get_rhoai_env_credentials,
+ )
legacy_provider_map_json = get_legacy_embedding_provider_map_json()
if legacy_provider_map_json:
@@ -890,21 +893,15 @@ class ConfigManager:
# lets a Helm (`llmProviders.rhoai.*`) or operator (`spec.rhoai`) install
# come up configured with no human clicking through Settings, which is
# the point on an air-gapped cluster.
- rhoai_endpoint = os.getenv("RHOAI_ENDPOINT")
- rhoai_embeddings_endpoint = os.getenv("RHOAI_EMBEDDINGS_ENDPOINT")
- rhoai_api_key = os.getenv("RHOAI_API_KEY")
- rhoai_tls_verify = os.getenv("RHOAI_TLS_VERIFY")
- if rhoai_endpoint or rhoai_embeddings_endpoint or rhoai_api_key or rhoai_tls_verify:
+ rhoai_env = get_rhoai_env_credentials()
+ if any(rhoai_env.values()):
+ # Field names only: `api_key` is a bearer token.
+ logger.debug(
+ "Seeding the rhoai provider from environment variables",
+ fields=sorted(name for name, value in rhoai_env.items() if value),
+ )
self._seed_custom_provider_credentials(
- config_data,
- "rhoai",
- {
- "api_base": rhoai_endpoint,
- "embedding_api_base": rhoai_embeddings_endpoint,
- "api_key": rhoai_api_key,
- "ssl_verify": rhoai_tls_verify,
- },
- required=("api_base", "api_key"),
+ config_data, "rhoai", rhoai_env, required=("api_base", "api_key")
)
# Knowledge settings
diff --git a/src/config/model_providers.yaml b/src/config/model_providers.yaml
index 51580e012..6d7e760d1 100644
--- a/src/config/model_providers.yaml
+++ b/src/config/model_providers.yaml
@@ -140,7 +140,7 @@ providers:
# These are `--served-model-name` values, not image tags: whatever the
# InferenceService was started with is the id that has to be sent.
models:
- - granite-3.3-2b-instruct
+ - gpt-oss-120b
embedding_models:
- granite-embedding-english-r2
diff --git a/src/config/settings.py b/src/config/settings.py
index 19cb8054f..e5c39494b 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -33,6 +33,23 @@ def get_legacy_embedding_provider_map_json() -> str | None:
return os.getenv("OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP")
+def get_rhoai_env_credentials() -> dict[str, str | None]:
+ """Return the raw ``RHOAI_*`` first-boot seed values, keyed by credential field.
+
+ Seeds the ``rhoai`` provider enhancement (see
+ ``enhancements/providers/redhat/openshift_ai.py``). Values are passed through
+ unvalidated: the enhancement owns parsing, and an invalid
+ ``embedding_max_concurrency`` falls back to its default with a warning.
+ """
+ return {
+ "api_base": os.getenv("RHOAI_ENDPOINT"),
+ "embedding_api_base": os.getenv("RHOAI_EMBEDDINGS_ENDPOINT"),
+ "api_key": os.getenv("RHOAI_API_KEY"),
+ "ssl_verify": os.getenv("RHOAI_TLS_VERIFY"),
+ "embedding_max_concurrency": os.getenv("RHOAI_EMBEDDING_MAX_CONCURRENCY"),
+ }
+
+
def get_opensearch_index_name_override() -> str | None:
"""Return the raw ``OPENSEARCH_INDEX_NAME`` env override.
@@ -698,8 +715,57 @@ UPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)
LANGFLOW_TIMEOUT = get_env_float("LANGFLOW_TIMEOUT", 2400.0) # 40 minutes
LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30 seconds
# Retries for transient Langflow HTTP failures (disconnects, 502/503/504).
+# Only idempotent requests are replayed after the server may have acted on
+# them; see `_retryable_langflow_statuses`.
LANGFLOW_REQUEST_RETRIES = get_env_int("LANGFLOW_REQUEST_RETRIES", 2)
+#: RFC 9110 §9.2.2 idempotent methods: repeating one leaves the server in the
+#: same state, so a failure after the request was sent can be replayed safely.
+_IDEMPOTENT_HTTP_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})
+#: Statuses retried for any method, because they arrive before the request is
+#: processed: 429 is a rate-limit rejection, and a 503 from Langflow itself is
+#: emitted before processing. RFC 9110 does not guarantee this for 503 in
+#: general: a proxy in front of Langflow (for example Envoy's "reset before
+#: headers") can send one after forwarding the request. This relies on the
+#: backend reaching Langflow directly, as it does in compose and over a
+#: Kubernetes Service with no service mesh.
+_LANGFLOW_ALWAYS_RETRYABLE_STATUSES = frozenset({429, 503})
+#: Statuses that can arrive *after* the server acted. Langflow answers 500 when
+#: a flow run's graph fails, so replaying `POST /api/v1/run` re-executes the
+#: whole flow (re-embedding, re-polling a Docling task that has since been
+#: deleted) and the replay's error masks the original one.
+_LANGFLOW_IDEMPOTENT_ONLY_STATUSES = frozenset({408, 500, 502, 504})
+#: Transport errors raised before the request reached the server.
+_LANGFLOW_PRE_SEND_ERRORS: tuple[type[httpx.RequestError], ...] = (
+ httpx.ConnectError,
+ httpx.ConnectTimeout,
+ httpx.PoolTimeout,
+)
+
+
+def _is_idempotent_request(method: str, idempotent: bool | None) -> bool:
+ """Whether a request may be replayed after the server could have acted on it."""
+ if idempotent is not None:
+ return idempotent
+ return (method or "").upper() in _IDEMPOTENT_HTTP_METHODS
+
+
+def _retryable_langflow_statuses(idempotent: bool) -> frozenset[int]:
+ """HTTP statuses worth retrying for a request of the given idempotency."""
+ if idempotent:
+ return _LANGFLOW_ALWAYS_RETRYABLE_STATUSES | _LANGFLOW_IDEMPOTENT_ONLY_STATUSES
+ return _LANGFLOW_ALWAYS_RETRYABLE_STATUSES
+
+
+def _is_retry_safe_transport_error(exc: httpx.RequestError, idempotent: bool) -> bool:
+ """Whether a transport error may be retried without risking a double execution.
+
+ A read timeout or dropped connection can happen after Langflow started a
+ flow run; only connection-phase failures prove the request was never seen.
+ """
+ return idempotent or isinstance(exc, _LANGFLOW_PRE_SEND_ERRORS)
+
+
# Per-file processing timeout for document ingestion tasks (in seconds)
# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete
# Default: 3600 seconds (60 minutes)
@@ -1479,13 +1545,32 @@ class AppClients:
"""Alias for cleanup() for convenience."""
await self.cleanup()
- async def langflow_request(self, method: str, endpoint: str, **kwargs):
+ async def langflow_request(
+ self,
+ method: str,
+ endpoint: str,
+ *,
+ idempotent: bool | None = None,
+ **kwargs,
+ ):
"""Central method for all Langflow API requests.
- Retries transient transport/server errors and once with a fresh API key on
- auth failures (401/403).
+ Retries once with a fresh API key on auth failures (401/403), and retries
+ transient failures according to whether the request is safe to repeat:
+
+ - Any request is retried when it provably was not processed: connection
+ errors before the request was sent, 429, and 503.
+ - Only idempotent requests are also retried on 408/500/502/504 and on
+ transport errors after sending (read timeouts, dropped connections).
+
+ `idempotent` defaults to the method's RFC 9110 semantics (GET, HEAD,
+ OPTIONS, PUT, DELETE). Pass `idempotent=True` for a POST/PATCH that sets
+ absolute state and can be repeated harmlessly. Replaying a non-idempotent
+ request such as `POST /api/v1/run/{flow}` re-executes the flow, and the
+ replay's error hides the original failure.
"""
- _TRANSIENT_STATUS_CODES = {408, 429, 500, 502, 503, 504}
+ is_idempotent = _is_idempotent_request(method, idempotent)
+ retryable_statuses = _retryable_langflow_statuses(is_idempotent)
max_attempts = max(1, LANGFLOW_REQUEST_RETRIES + 1)
last_error: Exception | None = None
auth_retry_attempted = False
@@ -1539,12 +1624,24 @@ class AppClients:
except httpx.RequestError as exc:
last_error = exc
if attempt + 1 < max_attempts:
+ if not _is_retry_safe_transport_error(exc, is_idempotent):
+ logger.warning(
+ "Langflow request failed; not retrying non-idempotent request",
+ method=method,
+ endpoint=endpoint,
+ attempt=attempt + 1,
+ error_type=type(exc).__name__,
+ error=str(exc),
+ )
+ raise
delay = min(2**attempt, 4)
logger.warning(
"Langflow request transport error, retrying",
+ method=method,
endpoint=endpoint,
attempt=attempt + 1,
max_attempts=max_attempts,
+ error_type=type(exc).__name__,
error=str(exc),
retry_in_seconds=delay,
)
@@ -1583,6 +1680,17 @@ class AppClients:
except httpx.RequestError as exc:
last_error = exc
if attempt + 1 < max_attempts:
+ if not _is_retry_safe_transport_error(exc, is_idempotent):
+ logger.warning(
+ "Langflow auth retry failed; not retrying "
+ "non-idempotent request",
+ method=method,
+ endpoint=endpoint,
+ attempt=attempt + 1,
+ error_type=type(exc).__name__,
+ error=str(exc),
+ )
+ raise
delay = min(2**attempt, 4)
logger.warning(
"Langflow auth retry transport error, retrying",
@@ -1609,10 +1717,26 @@ class AppClients:
await asyncio.sleep(delay)
continue
- if response.status_code in _TRANSIENT_STATUS_CODES and attempt + 1 < max_attempts:
+ if (
+ response.status_code in _LANGFLOW_IDEMPOTENT_ONLY_STATUSES
+ and response.status_code not in retryable_statuses
+ and attempt + 1 < max_attempts
+ ):
+ # Surface the first failure as-is: a replay would re-execute
+ # the request (e.g. a whole flow run) and could mask the error.
+ logger.warning(
+ "Langflow request failed; not retrying non-idempotent request",
+ method=method,
+ endpoint=endpoint,
+ status_code=response.status_code,
+ attempt=attempt + 1,
+ )
+
+ if response.status_code in retryable_statuses and attempt + 1 < max_attempts:
delay = min(2**attempt, 4)
logger.warning(
"Langflow request returned transient status, retrying",
+ method=method,
endpoint=endpoint,
status_code=response.status_code,
attempt=attempt + 1,
@@ -1787,7 +1911,10 @@ class AppClients:
}
patch_response = await self.langflow_request(
- "PATCH", f"/api/v1/variables/{variable_id}", json=update_payload
+ "PATCH",
+ f"/api/v1/variables/{variable_id}",
+ json=update_payload,
+ idempotent=True,
)
if patch_response.status_code == 200:
diff --git a/src/dependencies.py b/src/dependencies.py
index 746bd1f68..0be0ff44f 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -197,11 +197,15 @@ async def get_llm_proxy_user(
token = raw[7:].strip() if raw.lower().startswith("bearer ") else raw.strip()
if token and not token.startswith("orag_"):
try:
- user = token_service.validate_token(token)
+ claims = token_service.validate_claims(token)
except ValueError:
- user = None
+ claims = None
else:
+ user = token_service.user_from_claims(claims)
request.state.user = user
+ # Which kind of Langflow run minted it; the gateway gives chat
+ # (query) embeddings their own lane beside bulk ingestion.
+ request.state.llm_hop_purpose = claims.get("purpose")
return user
return await resolve_api_key_user(request, api_key_service, session_manager)
diff --git a/src/services/chat_service.py b/src/services/chat_service.py
index 6d9d020c8..dec8501cf 100644
--- a/src/services/chat_service.py
+++ b/src/services/chat_service.py
@@ -88,6 +88,7 @@ class ChatService:
# Pass the selected embedding model as a global variable
from config.settings import get_openrag_config
+ from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
@@ -154,6 +155,7 @@ class ChatService:
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=user_id,
+ purpose=HOP_PURPOSE_CHAT,
)
# Get context variables for filters, limit, and threshold
from auth_context import (
@@ -274,6 +276,7 @@ class ChatService:
extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_token
from config.settings import get_openrag_config
+ from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
@@ -289,6 +292,7 @@ class ChatService:
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=user_id,
+ purpose=HOP_PURPOSE_CHAT,
)
# Build the complete filter expression like the chat service does
@@ -530,6 +534,7 @@ class ChatService:
extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_token
from config.settings import get_openrag_config
+ from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
@@ -596,6 +601,7 @@ class ChatService:
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=user_id,
+ purpose=HOP_PURPOSE_CHAT,
)
# Ensure the Langflow client exists; try lazy init if needed
diff --git a/src/services/flows_service.py b/src/services/flows_service.py
index 825843643..d75976cc2 100644
--- a/src/services/flows_service.py
+++ b/src/services/flows_service.py
@@ -518,7 +518,10 @@ class FlowsService:
"""Unlock a flow before making changes."""
try:
response = await clients.langflow_request(
- "PATCH", f"/api/v1/flows/{flow_id}", json={"locked": False}
+ "PATCH",
+ f"/api/v1/flows/{flow_id}",
+ json={"locked": False},
+ idempotent=True,
)
if response.status_code not in (200, 404):
logger.warning(
@@ -531,7 +534,10 @@ class FlowsService:
"""Lock a flow after making changes."""
try:
response = await clients.langflow_request(
- "PATCH", f"/api/v1/flows/{flow_id}", json={"locked": True}
+ "PATCH",
+ f"/api/v1/flows/{flow_id}",
+ json={"locked": True},
+ idempotent=True,
)
if response.status_code not in (200, 404):
logger.warning(
@@ -566,7 +572,10 @@ class FlowsService:
# Make PATCH request to Langflow API to update the flow using shared client
try:
response = await clients.langflow_request(
- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data
+ "PATCH",
+ f"/api/v1/flows/{flow_id}",
+ json=flow_data,
+ idempotent=True,
)
if response.status_code == 200:
@@ -729,7 +738,10 @@ class FlowsService:
# Update the flow via PATCH request
patch_response = await clients.langflow_request(
- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data
+ "PATCH",
+ f"/api/v1/flows/{flow_id}",
+ json=flow_data,
+ idempotent=True,
)
if patch_response.status_code != 200:
@@ -1319,7 +1331,10 @@ class FlowsService:
# PATCH the updated flow
response = await clients.langflow_request(
- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data
+ "PATCH",
+ f"/api/v1/flows/{flow_id}",
+ json=flow_data,
+ idempotent=True,
)
if response.status_code != 200:
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 87bd1d260..a530c377c 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -490,6 +490,7 @@ class LangflowFileService:
# Get the current embedding model and provider credentials from config
from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE
+ from services.langflow_llm_token_service import HOP_PURPOSE_INGEST
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
@@ -570,6 +571,7 @@ class LangflowFileService:
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=owner,
+ purpose=HOP_PURPOSE_INGEST,
)
if self.ingest_token_service is None:
await self._ensure_langflow_ingest_index(embedding_model)
@@ -696,6 +698,7 @@ class LangflowFileService:
tweaks = {}
from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, get_openrag_config
+ from services.langflow_llm_token_service import HOP_PURPOSE_INGEST
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
@@ -758,6 +761,7 @@ class LangflowFileService:
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=owner,
+ purpose=HOP_PURPOSE_INGEST,
)
if self.ingest_token_service is None:
await self._ensure_langflow_ingest_index(embedding_model)
diff --git a/src/services/langflow_llm_token_service.py b/src/services/langflow_llm_token_service.py
index 5226eace7..6af222dc8 100644
--- a/src/services/langflow_llm_token_service.py
+++ b/src/services/langflow_llm_token_service.py
@@ -29,6 +29,13 @@ LANGFLOW_HOP_AUDIENCES = frozenset(
}
)
+#: What a Langflow run's hop token is for, carried as its `purpose` claim. The
+#: gateway gives `chat` embedding calls (query retrieval) their own lane so they
+#: are not queued behind a bulk ingest. Anything else, including no claim at
+#: all, is treated as bulk traffic.
+HOP_PURPOSE_CHAT = "chat"
+HOP_PURPOSE_INGEST = "ingest"
+
def langflow_hop_audience(
token: str,
@@ -89,6 +96,7 @@ class LangflowLlmTokenService:
user_id: str,
email: str | None = None,
name: str | None = None,
+ purpose: str | None = None,
) -> str:
now = int(time.time())
subject = (user_id or "").strip() or "anonymous"
@@ -103,9 +111,15 @@ class LangflowLlmTokenService:
"email": (email or subject),
"name": (name or subject),
}
+ if purpose:
+ payload["purpose"] = purpose
return jwt.encode(payload, self._signing_key, algorithm=self.algorithm)
def validate_token(self, token: str) -> User:
+ return self.user_from_claims(self.validate_claims(token))
+
+ def validate_claims(self, token: str) -> dict[str, Any]:
+ """Verified claims of a hop token; raises `ValueError` if it is not one."""
try:
payload = jwt.decode(
token,
@@ -127,7 +141,11 @@ class LangflowLlmTokenService:
user_id = str(payload.get("user_id") or payload.get("sub") or "").strip()
if not user_id:
raise ValueError("Langflow LLM proxy token is missing user_id")
+ return payload
+ @staticmethod
+ def user_from_claims(payload: dict[str, Any]) -> User:
+ user_id = str(payload.get("user_id") or payload.get("sub") or "").strip()
return User(
user_id=user_id,
email=str(payload.get("email") or user_id),
diff --git a/src/services/langflow_mcp_service.py b/src/services/langflow_mcp_service.py
index 826c30621..21522a560 100644
--- a/src/services/langflow_mcp_service.py
+++ b/src/services/langflow_mcp_service.py
@@ -257,6 +257,7 @@ class LangflowMCPService:
method="PATCH",
endpoint=f"/api/v2/mcp/servers/{server_name}",
json=payload,
+ idempotent=True,
)
if response.status_code in (502, 503, 504):
response.raise_for_status()
diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.py
index 3dc42ea41..f545c9de1 100644
--- a/src/services/llm_gateway.py
+++ b/src/services/llm_gateway.py
@@ -7,10 +7,15 @@ routes by model prefix / configured provider. Callers never see upstream keys.
from __future__ import annotations
+import asyncio
+import contextlib
+import functools
import json
import re
-from collections.abc import AsyncIterator, Mapping
+from collections import deque
+from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
from typing import Any, Literal
+from uuid import uuid4
from services import provider_error_log
from services.model_catalog import is_known_provider, litellm_provider_key
@@ -219,6 +224,222 @@ def _provider_runtime_kwargs(provider: str, config=None) -> dict[str, Any]:
return runtime_kwargs_for(enhancement, stored)
+#: Lanes of a provider's embedding limit. `interactive` is query embedding —
+#: search and chat retrieval, where a person is waiting; `bulk` is everything
+#: else, chiefly ingestion. One slot of the limit is reserved for queries, so a
+#: folder upload cannot queue them behind hundreds of chunks.
+_BULK_LANE = "bulk"
+_INTERACTIVE_LANE = "interactive"
+
+
+def _lane_limit(limit: int, interactive: bool) -> tuple[str, int]:
+ """The lane a call waits in and that lane's size, out of `limit` in total.
+
+ The lanes split the operator's limit rather than add to it, so the endpoint
+ never sees more than `limit` calls at once: `limit - 1` for bulk traffic
+ and one for queries. A limit of 1 cannot be split without exceeding it, so
+ both kinds of call then share the single slot.
+ """
+ if limit < 2:
+ return _BULK_LANE, limit
+ if interactive:
+ return _INTERACTIVE_LANE, 1
+ return _BULK_LANE, limit - 1
+
+
+class _EmbeddingLimiter:
+ """Bounds one provider's in-flight embedding calls under a limit that can change.
+
+ A pair of per-lane semaphores cannot be resized: replacing them when the
+ operator changes the limit forgets the calls still holding the old ones, so
+ the endpoint briefly sees more than the new limit. This keeps one count per
+ lane for the provider's lifetime and admits a call only while the total is
+ under the current limit *and* its lane is under its share (`_lane_limit`).
+
+ A freed slot is handed straight to the next waiter (as `asyncio.Semaphore`
+ does), queries first, so neither a new arrival nor a bulk chunk can take a
+ slot a waiting query is entitled to. `limit is None` means unbounded: calls
+ are still counted, so turning a limit back on measures it against them.
+
+ Not thread-safe; it lives on one event loop (see `_embedding_limiters`).
+ """
+
+ def __init__(self, provider: str, limit: int | None, loop: asyncio.AbstractEventLoop):
+ self.provider = provider
+ self.limit = limit
+ self.loop = loop
+ self._active = {_BULK_LANE: 0, _INTERACTIVE_LANE: 0}
+ self._waiters: dict[str, deque[asyncio.Future[None]]] = {
+ _BULK_LANE: deque(),
+ _INTERACTIVE_LANE: deque(),
+ }
+
+ @property
+ def in_flight(self) -> int:
+ return sum(self._active.values())
+
+ def _admits(self, lane: str) -> bool:
+ if self.limit is None:
+ return True
+ if self.in_flight >= self.limit:
+ return False
+ _, lane_limit = _lane_limit(self.limit, lane == _INTERACTIVE_LANE)
+ return self._active[lane] < lane_limit
+
+ def _wake(self) -> None:
+ """Grant free slots to waiters, queries before bulk, FIFO within a lane."""
+ for lane in (_INTERACTIVE_LANE, _BULK_LANE):
+ waiters = self._waiters[lane]
+ while waiters and self._admits(lane):
+ waiter = waiters.popleft()
+ if waiter.done(): # cancelled while queued
+ continue
+ self._active[lane] += 1
+ waiter.set_result(None)
+
+ async def acquire(self, lane: str, model: str = "") -> None:
+ if not self._waiters[lane] and self._admits(lane):
+ self._active[lane] += 1
+ return
+ waiter: asyncio.Future[None] = self.loop.create_future()
+ self._waiters[lane].append(waiter)
+ logger.debug(
+ "Embedding call waiting for a free provider slot",
+ provider=self.provider,
+ model=model,
+ lane=lane,
+ max_concurrency=self.limit,
+ in_flight=self.in_flight,
+ )
+ try:
+ await waiter
+ except asyncio.CancelledError:
+ if waiter.done() and not waiter.cancelled():
+ # Granted a slot in the same tick the caller was cancelled:
+ # hand it on, or it is lost for the provider's lifetime.
+ self.release(lane)
+ else:
+ with contextlib.suppress(ValueError):
+ self._waiters[lane].remove(waiter)
+ raise
+
+ def release(self, lane: str) -> None:
+ if self._active[lane] <= 0:
+ logger.error(
+ "Embedding slot released more often than acquired; ignoring",
+ provider=self.provider,
+ lane=lane,
+ )
+ self._active[lane] = 0
+ else:
+ self._active[lane] -= 1
+ self._wake()
+
+ def resize(self, limit: int | None) -> None:
+ """Apply a limit changed in Settings, counting every call still in flight."""
+ if limit == self.limit:
+ return
+ previous = self.limit
+ self.limit = limit
+ logger.info(
+ "Resizing embedding concurrency limit for provider",
+ provider=self.provider,
+ previous_max_concurrency=previous,
+ max_concurrency=limit,
+ bulk_in_flight=self._active[_BULK_LANE],
+ interactive_in_flight=self._active[_INTERACTIVE_LANE],
+ )
+ if limit is not None and self.in_flight > limit:
+ logger.info(
+ "More embedding calls in flight than the new limit allows; "
+ "new calls wait until they finish",
+ provider=self.provider,
+ max_concurrency=limit,
+ in_flight=self.in_flight,
+ )
+ self._wake()
+
+ @contextlib.asynccontextmanager
+ async def slot(self, lane: str, model: str = "") -> AsyncIterator[None]:
+ """Hold one of the provider's embedding slots for the duration of a call."""
+ await self.acquire(lane, model)
+ try:
+ yield
+ finally:
+ self.release(lane)
+
+
+#: Per-provider embedding bulkheads, keyed by provider key. The backend runs a
+#: single worker (enforced in `app/lifespan.py`), so a process-local limiter
+#: bounds the provider's traffic globally. Each limiter records its loop, and a
+#: different running loop gets a new one, because asyncio futures must not be
+#: shared across loops.
+_embedding_limiters: dict[str, _EmbeddingLimiter] = {}
+
+
+def _embedding_limiter(provider: str, config=None) -> _EmbeddingLimiter | None:
+ """The limiter bounding embedding calls to `provider`, if any.
+
+ The limit comes from the provider enhancement (`embedding_max_concurrency`),
+ so providers without one — and every provider whose enhancement sets no
+ limit — are unaffected. Excess callers wait here instead of queueing at the
+ upstream, where a slow model server lets a fronting proxy time them out
+ (RHOAI's kube-rbac-proxy answers 502 after 30s). Interactive calls wait in
+ their own lane (`_lane_limit`). A limit changed in Settings takes effect on
+ the next call and is measured against every call still in flight, so a
+ lowered limit admits nothing new until enough of them finish.
+ """
+ cfg = config or _get_config()
+ prov = getattr(cfg, "providers", None)
+ if prov is None or not hasattr(prov, "credential_values"):
+ return None
+
+ from enhancements.providers.registry import embedding_concurrency_for, get
+
+ key = (provider or "").strip().lower()
+ if get(key) is None:
+ return None
+ stored = (
+ prov.stored_credentials(key)
+ if hasattr(prov, "stored_credentials")
+ else prov.credential_values(key)
+ )
+ limit = embedding_concurrency_for(key, stored)
+ loop = asyncio.get_running_loop()
+ cached = _embedding_limiters.get(key)
+ if cached is not None and cached.loop is loop:
+ # Resized in place (also to "no limit"), so the calls it is already
+ # counting stay counted against whatever limit applies next.
+ cached.resize(limit)
+ return cached
+ if limit is None:
+ _embedding_limiters.pop(key, None)
+ return None
+
+ limiter = _EmbeddingLimiter(key, limit, loop)
+ _embedding_limiters[key] = limiter
+ logger.info(
+ "Bounding concurrent embedding calls for provider",
+ provider=key,
+ max_concurrency=limit,
+ bulk_max_concurrency=_lane_limit(limit, False)[1],
+ interactive_max_concurrency=_lane_limit(limit, True)[1],
+ )
+ return limiter
+
+
+async def _embedding_slot(
+ limiter: _EmbeddingLimiter | None, provider: str, model: str, lane: str = _BULK_LANE
+) -> AsyncIterator[None]:
+ """Hold one of the provider's embedding slots for the duration of a call."""
+ if limiter is None:
+ yield
+ return
+ async with limiter.slot(lane, model):
+ yield
+
+
def resolve_call(
model: str | None,
*,
@@ -276,6 +497,36 @@ _UPSTREAM_TLS_MESSAGE = (
"The provider's TLS certificate is not trusted by this deployment. An operator needs to "
"add the provider's CA certificate to OpenRAG's trust store."
)
+#: An upstream 502/504 with no body to quote — typically a proxy in front of
+#: the model server (RHOAI's kube-rbac-proxy) giving up while the server is
+#: still busy. LiteLLM then reports only its own wrapper names
+#: ("BadGatewayError: Hosted_vllmException -"), which reads like a bug.
+_UPSTREAM_GATEWAY_MESSAGE = (
+ "The model server did not respond in time or is unavailable (upstream 502/504). It may "
+ "be overloaded: check its capacity and any proxy timeout in front of it, or lower the "
+ "provider's max concurrent requests."
+)
+#: The connection closed before the response finished. A proxy or load balancer
+#: in front of the model server timing out a long response, not the model
+#: rejecting anything — see `is_provider_truncated_response_error`.
+_UPSTREAM_TRUNCATED_MESSAGE = (
+ "The connection to the model server closed before the response finished. A proxy or "
+ "load balancer in front of it likely timed out a long response: raise that timeout, "
+ "or point the provider at an endpoint that bypasses it."
+)
+_GATEWAY_FAILURE_STATUSES = frozenset({502, 504})
+#: LiteLLM exception class names that wrap an upstream error, e.g.
+#: "litellm.BadGatewayError: BadGatewayError: Hosted_vllmException -".
+_EXCEPTION_WRAPPER_RE = re.compile(r"(?:litellm\.)?[A-Za-z_]+(?:Error|Exception)\b\s*[:\-]?\s*")
+
+
+def _is_opaque_gateway_failure(upstream: str, exc: BaseException | None) -> bool:
+ """True for an upstream 502/504 whose text is nothing but exception wrappers."""
+ status = getattr(exc, "status_code", None)
+ if not isinstance(status, int) or status not in _GATEWAY_FAILURE_STATUSES:
+ return False
+ remainder = _EXCEPTION_WRAPPER_RE.sub("", upstream or "").strip(" -:.")
+ return not remainder
#: Longest upstream explanation we will echo. Provider bodies can embed a whole
@@ -449,6 +700,7 @@ def _upstream_client_message(
is_generic_upstream_error,
is_provider_credential_error,
is_provider_tls_error,
+ is_provider_truncated_response_error,
)
label = _call_label(provider, model)
@@ -457,10 +709,17 @@ def _upstream_client_message(
# someone to rotate a working API key wastes the trip.
if is_provider_tls_error(detail):
return f"{_UPSTREAM_TLS_MESSAGE} ({label})"
+ # Also ahead of the credential branch: aiohttp reports the cut-off with a
+ # "400" of its own, and LiteLLM's wrapper chain around it says nothing an
+ # operator can act on.
+ if is_provider_truncated_response_error(detail):
+ return f"{_UPSTREAM_TRUNCATED_MESSAGE} ({label})"
if is_provider_credential_error(detail):
return f"{_UPSTREAM_CREDENTIAL_MESSAGE} ({label})"
upstream = _provider_error_text(detail, exc)
upstream = _sanitise_upstream_detail(upstream) if upstream else ""
+ if _is_opaque_gateway_failure(upstream, exc):
+ return f"{_UPSTREAM_GATEWAY_MESSAGE} ({label})"
if not upstream or is_generic_upstream_error(upstream):
return f"{_UPSTREAM_FAILURE_MESSAGE} ({label})"
# lgtm[py/stack-trace-exposure] — provider error text only; traceback frames,
@@ -468,6 +727,24 @@ def _upstream_client_message(
return f"{label}: {upstream}"
+def _is_request_scoped_failure(detail: str, exc: BaseException | None) -> bool:
+ """True when the failure belongs to this request, not to the provider.
+
+ A prompt that overflows the context window fails because of what this one
+ call carried; the next, shorter one succeeds. Recording it in
+ `provider_error_log` would latch the health banner against a provider that
+ is serving fine — and on a provider whose probe cannot clear it, until the
+ entry goes stale. The caller still gets the error.
+ """
+ from api.provider_validation import is_context_window_error
+
+ if exc is not None and any(
+ cls.__name__ == "ContextWindowExceededError" for cls in type(exc).__mro__
+ ):
+ return True
+ return is_context_window_error(detail)
+
+
def _redact(message: str, credentials: Mapping[str, Any]) -> str:
redacted = message
for value in credentials.values():
@@ -565,6 +842,118 @@ def _repair_tool_calls(tool_calls: Any, provider: str, model: str) -> int:
return repaired
+def _finalise_tool_call(call: dict[str, Any]) -> bool:
+ """Make `call` something a client can execute. False when it cannot be.
+
+ A call with no name names no tool, so nothing can run it and no result can
+ ever come back for it; forwarding it only corrupts the assistant message the
+ client will replay on its next turn. A missing `id` is recoverable — the id
+ is only the handle a tool result is correlated by, so any stable string
+ serves — and minting one keeps a real call alive that clients would
+ otherwise reject for carrying `id: null`. Blank `arguments` are recoverable
+ too: a named call streamed with no arguments fragment is a zero-argument
+ call, and the contract is a JSON *object* — `""` does not parse as one.
+ """
+ function = call.get("function")
+ if not isinstance(function, dict) or not function.get("name"):
+ return False
+ if not isinstance(call.get("id"), str) or not call["id"]:
+ call["id"] = f"call_{uuid4().hex}"
+ arguments = function.get("arguments")
+ if arguments is None or (isinstance(arguments, str) and not arguments.strip()):
+ function["arguments"] = "{}"
+ logger.debug(
+ "Defaulted blank tool call arguments to an empty object", tool=function["name"]
+ )
+ return True
+
+
+def _sanitise_messages(messages: Any) -> tuple[list[Any], int]:
+ """`messages` with unusable tool calls repaired or dropped, and a repair count.
+
+ The gateway forwards a client's conversation verbatim, and one malformed
+ tool call in it is not a one-turn problem. Clients replay their whole
+ history on every turn, so the same bad message goes back up again and again,
+ and providers reject the *request* rather than the message that spoiled it —
+ vLLM answers the lot with "Please ensure `tool_calls` are iterable of tool
+ calls". The conversation is then wedged for good, with nothing in the UI
+ pointing at the turn that broke it. A proxy sitting between many clients and
+ many providers is the right place to stop that.
+
+ Three shapes are handled, all of them things real clients emit: `arguments`
+ sent as an object rather than the JSON *string* OpenAI's contract requires;
+ a tool call carrying no id or no name, which nothing can execute and no
+ result can be matched to; and the tool results left behind by one, which are
+ dropped alongside it so no provider is handed a reply to a call that is no
+ longer there.
+
+ What survives must still be a sequence providers accept: a `tool` message
+ is kept only when it answers a call kept on the assistant message just
+ before it, and an assistant message left with neither tool calls nor
+ content is dropped, since `content` is only optional beside `tool_calls`.
+ """
+ cleaned: list[Any] = []
+ repairs = 0
+ # Ids of the calls kept on the most recent assistant message: the only ones
+ # a `tool` message may answer.
+ live_ids: set[str] = set()
+ for message in messages or []:
+ if not isinstance(message, dict):
+ cleaned.append(message)
+ continue
+ if message.get("role") == "tool":
+ if message.get("tool_call_id") not in live_ids:
+ repairs += 1
+ continue
+ cleaned.append(message)
+ continue
+ live_ids = set()
+ if "tool_calls" not in message:
+ cleaned.append(message)
+ continue
+
+ raw = message.get("tool_calls")
+ kept: list[dict[str, Any]] = []
+ for call in raw if isinstance(raw, list) else []:
+ if not isinstance(call, dict):
+ repairs += 1
+ continue
+ # The client controls this value, and `dict()` raises on a string or
+ # a number. A non-mapping names no tool, so it becomes an empty one
+ # and the call is dropped below like any other nameless call.
+ function = call.get("function")
+ call = {**call, "function": dict(function) if isinstance(function, dict) else {}}
+ arguments, changed = _normalise_tool_arguments(call["function"].get("arguments"))
+ if changed:
+ call["function"]["arguments"] = arguments
+ repairs += 1
+ # Unlike the response path, a missing id is not minted here: the
+ # result for this call is already somewhere in the history under the
+ # id the client used, and inventing a different one would orphan it.
+ if isinstance(call.get("id"), str) and call["id"] and call["function"].get("name"):
+ kept.append(call)
+ else:
+ repairs += 1
+
+ message = dict(message)
+ if kept:
+ message["tool_calls"] = kept
+ live_ids = {call["id"] for call in kept}
+ else:
+ # `tool_calls: null` is itself a shape some providers iterate
+ # without a None check, so the key goes rather than emptying.
+ message.pop("tool_calls")
+ # A list's entries were each counted above; anything else was
+ # never iterable in the first place and is one repair on its own.
+ repairs += bool(raw) and not isinstance(raw, list)
+ if raw and message.get("role") == "assistant" and not message.get("content"):
+ # The dropped calls were the whole message; what is left has
+ # nothing to say, and no provider accepts it saying nothing.
+ continue
+ cleaned.append(message)
+ return cleaned, repairs
+
+
def _repair_completion_payload(payload: dict[str, Any], provider: str, model: str) -> int:
repaired = 0
for choice in payload.get("choices") or []:
@@ -655,6 +1044,36 @@ def _reasoning_off_retry(model: str, kwargs: dict[str, Any]) -> bool:
return True
+async def with_stale_connection_retry(
+ call: Callable[[], Awaitable[Any]], *, provider: str, model: str, kind: str
+) -> Any:
+ """Run one LiteLLM call, sending it once more if a dead pooled connection ate it.
+
+ LiteLLM keeps idle connections for 120s; vLLM closes them after 5s. A call
+ that picks one up just as the server closes it fails with "Server
+ disconnected" before the server reads a byte (it logs nothing), and the
+ replay goes out on a fresh connection. Only that failure is retried — see
+ `is_provider_stale_connection_error` — so a request the server may have
+ acted on is never sent twice. For a stream only the call that opens it is
+ wrapped, so nothing has reached the caller yet; a mid-stream failure is
+ not retried.
+ """
+ from api.provider_validation import is_provider_stale_connection_error
+
+ try:
+ return await call()
+ except Exception as exc:
+ if not is_provider_stale_connection_error(exc):
+ raise
+ logger.warning(
+ "Model server closed an idle connection before answering; retrying once",
+ provider=provider,
+ model=model,
+ kind=kind,
+ )
+ return await call()
+
+
async def chat_completions(
body: Mapping[str, Any], *, config=None
) -> dict[str, Any] | AsyncIterator[str]:
@@ -667,24 +1086,37 @@ async def chat_completions(
if litellm_model in _TOOLS_NEED_REASONING_OFF:
# Already learned about this model; do not spend a round-trip relearning.
_reasoning_off_retry(litellm_model, kwargs)
+ messages, repairs = _sanitise_messages(body.get("messages"))
+ if repairs:
+ logger.warning(
+ "Repaired malformed tool calls in the request's conversation",
+ provider=provider,
+ model=litellm_model,
+ repairs=repairs,
+ )
async def _call() -> Any:
import litellm
- return await litellm.acompletion(
+ return await with_stale_connection_retry(
+ lambda: litellm.acompletion(
+ model=litellm_model,
+ messages=messages,
+ stream=stream,
+ # OpenAI-compatible clients send OpenAI's full parameter set, but
+ # providers accept different subsets — watsonx rejects
+ # `parallel_tool_calls`, `max_completion_tokens` and `logit_bias`,
+ # and LiteLLM raises UnsupportedParamsError rather than ignoring
+ # them. A proxy that fans out to many providers must degrade to
+ # the provider's capabilities instead of failing the request.
+ drop_params=True,
+ **credentials,
+ **runtime_kwargs,
+ **kwargs,
+ ),
+ provider=provider,
model=litellm_model,
- messages=list(body.get("messages") or []),
- stream=stream,
- # OpenAI-compatible clients send OpenAI's full parameter set, but
- # providers accept different subsets — watsonx rejects
- # `parallel_tool_calls`, `max_completion_tokens` and `logit_bias`,
- # and LiteLLM raises UnsupportedParamsError rather than ignoring
- # them. A proxy that fans out to many providers must degrade to the
- # provider's capabilities instead of failing the request.
- drop_params=True,
- **credentials,
- **runtime_kwargs,
- **kwargs,
+ kind="chat",
)
try:
@@ -716,7 +1148,8 @@ async def chat_completions(
# The health banner otherwise reports whatever its own probe hit, which
# is a different request and so often a different error. Hand it the
# text this caller is being shown.
- provider_error_log.record_failure(provider, "chat", message)
+ if not _is_request_scoped_failure(detail, exc):
+ provider_error_log.record_failure(provider, "chat", message)
raise LlmGatewayError(
message,
_upstream_status_code(exc),
@@ -783,6 +1216,57 @@ class _ToolCallBuffer:
def __bool__(self) -> bool:
return bool(self._order)
+ def _open_call(self, choice_index: int) -> dict[str, Any] | None:
+ """The call most recently opened on this choice, if there is one."""
+ for key in reversed(self._order):
+ if key[0] == choice_index:
+ return self._calls[key]
+ return None
+
+ def _target(self, choice_index: int, index: int, raw: Mapping[str, Any]) -> dict[str, Any]:
+ """The call `raw` belongs to: one already open, or a new one.
+
+ A delta that names no call — no `id`, no `function.name` — cannot be the
+ start of one, so it continues the call already open on this choice
+ rather than opening another. vLLM's tool parsers can attribute a call's
+ last `arguments` fragment, often the closing brace by itself, to the
+ *next* index; taken at face value that does two kinds of damage at once.
+ The real call loses the fragment and its arguments no longer parse, and
+ a second call appears with no id and no name that nothing can execute.
+ Neither survives the round trip: the client folds both into the
+ assistant message it replays on the next turn, and the provider then
+ rejects every later request in that conversation rather than the one
+ message that is malformed.
+
+ With nothing open to continue, the fragment instead opens a provisional
+ call at its own index. Some providers send an `arguments` fragment ahead
+ of the delta that carries the call's `id` and `name`; dropping it would
+ truncate the arguments once that delta arrives at the same index and
+ finds the call. A provisional call that is never named is discarded by
+ `drain`, which logs it.
+ """
+ key = (choice_index, index)
+ call = self._calls.get(key)
+ if call is not None:
+ return call
+ function = raw.get("function")
+ names_a_call = bool(raw.get("id")) or bool(
+ isinstance(function, dict) and function.get("name")
+ )
+ if not names_a_call:
+ open_call = self._open_call(choice_index)
+ if open_call is not None:
+ return open_call
+ logger.debug(
+ "Holding an unnamed tool-call fragment until its call is named",
+ choice_index=choice_index,
+ index=index,
+ )
+ call = {"index": index, "type": "function", "function": {"name": "", "arguments": ""}}
+ self._calls[key] = call
+ self._order.append(key)
+ return call
+
def absorb(self, choice_index: int, tool_calls: Any) -> None:
for position, raw in enumerate(tool_calls or []):
if not isinstance(raw, dict):
@@ -790,16 +1274,7 @@ class _ToolCallBuffer:
index = raw.get("index")
if not isinstance(index, int):
index = position
- key = (choice_index, index)
- call = self._calls.get(key)
- if call is None:
- call = {
- "index": index,
- "type": "function",
- "function": {"name": "", "arguments": ""},
- }
- self._calls[key] = call
- self._order.append(key)
+ call = self._target(choice_index, index, raw)
if raw.get("id"):
call["id"] = raw["id"]
if raw.get("type"):
@@ -825,9 +1300,18 @@ class _ToolCallBuffer:
grouped.setdefault(choice_index, [])
for key in self._order:
grouped[key[0]].append(self._calls[key])
- for calls in grouped.values():
+ for choice_index, calls in grouped.items():
calls.sort(key=lambda call: call.get("index", 0))
- repaired += _repair_tool_calls(calls, provider, model)
+ usable = [call for call in calls if _finalise_tool_call(call)]
+ if len(usable) != len(calls):
+ logger.warning(
+ "Dropped tool calls that named no tool",
+ provider=provider,
+ model=model,
+ dropped=len(calls) - len(usable),
+ )
+ grouped[choice_index] = usable
+ repaired += _repair_tool_calls(usable, provider, model)
self._calls.clear()
self._order.clear()
return grouped, repaired
@@ -1007,7 +1491,8 @@ async def _stream_sse(
tally.error = detail
logger.error("LLM chat stream failed", provider=provider, model=model, error=detail)
message = _upstream_client_message(detail, provider, model, exc)
- provider_error_log.record_failure(provider, "chat", message)
+ if not _is_request_scoped_failure(detail, exc):
+ provider_error_log.record_failure(provider, "chat", message)
yield _error_frame(message, provider, model)
finally:
close = getattr(stream, "aclose", None) or getattr(stream, "close", None)
@@ -1056,13 +1541,22 @@ def _embedding_input(value: Any) -> Any:
return [value] if isinstance(value, str) else value
-async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:
- """OpenAI `POST /v1/embeddings`."""
+async def embeddings(
+ body: Mapping[str, Any], *, config=None, interactive: bool = False
+) -> dict[str, Any]:
+ """OpenAI `POST /v1/embeddings`.
+
+ `interactive` marks a query embedding someone is waiting on (search, chat
+ retrieval). At a provider with a concurrency limit it waits in its own
+ lane instead of behind bulk ingestion; see `_embedding_limiter`.
+ """
cfg = config or _get_config()
litellm_model, provider, credentials = resolve_call(
body.get("model"), kind="embedding", config=cfg
)
runtime_kwargs = _provider_runtime_kwargs(provider, cfg)
+ limiter = _embedding_limiter(provider, cfg)
+ lane = _INTERACTIVE_LANE if interactive else _BULK_LANE
embedding_input = _embedding_input(body.get("input"))
should_batch = (
provider == "watsonx_onprem"
@@ -1074,12 +1568,18 @@ async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:
import litellm
if not should_batch:
- result = await litellm.aembedding(
- model=litellm_model,
- input=embedding_input,
- **credentials,
- **runtime_kwargs,
- )
+ async with _embedding_slot(limiter, provider, litellm_model, lane):
+ result = await with_stale_connection_retry(
+ lambda: litellm.aembedding(
+ model=litellm_model,
+ input=embedding_input,
+ **credentials,
+ **runtime_kwargs,
+ ),
+ provider=provider,
+ model=litellm_model,
+ kind="embedding",
+ )
response = _to_openai_dict(result)
else:
response = {}
@@ -1091,12 +1591,19 @@ async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:
_WATSONX_ONPREM_EMBEDDING_BATCH_SIZE,
):
batch = embedding_input[offset : offset + _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE]
- result = await litellm.aembedding(
- model=litellm_model,
- input=batch,
- **credentials,
- **runtime_kwargs,
- )
+ async with _embedding_slot(limiter, provider, litellm_model, lane):
+ result = await with_stale_connection_retry(
+ functools.partial(
+ litellm.aembedding,
+ model=litellm_model,
+ input=batch,
+ **credentials,
+ **runtime_kwargs,
+ ),
+ provider=provider,
+ model=litellm_model,
+ kind="embedding",
+ )
payload = _to_openai_dict(result)
if not response:
response = {
diff --git a/src/services/model_catalog.py b/src/services/model_catalog.py
index 2fceffd9f..8dc743c35 100644
--- a/src/services/model_catalog.py
+++ b/src/services/model_catalog.py
@@ -436,17 +436,24 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:
)
-async def refresh_live_models() -> None:
+async def refresh_live_models(provider: str | None = None) -> None:
"""Re-list the models on any provider that can only answer for itself.
Called by the catalogue routes before the payload is built. The provider
caches with a TTL, so this is a network call once every few minutes rather
than once per request, and a failure leaves the previous answer — or the
configured fallback — in place.
+
+ `provider` narrows it to that one provider. The catalogue needs every
+ listing; a caller asking about one provider — the health check — should not
+ wait on a slow cluster that belongs to another.
"""
from config.settings import get_openrag_config
+ only = (provider or "").strip().lower() or None
for enhancement in provider_enhancements():
+ if only is not None and enhancement.PROVIDER_KEY != only:
+ continue
if enhancement.PROVIDER_KEY not in supported_provider_keys() or not hasattr(
enhancement, "fetch_models"
):
diff --git a/src/services/provider_error_log.py b/src/services/provider_error_log.py
index 424ffe481..2672f6654 100644
--- a/src/services/provider_error_log.py
+++ b/src/services/provider_error_log.py
@@ -69,5 +69,9 @@ def latest_failure(provider: str | None, kind: CallKind) -> str | None:
def clear() -> None:
- """Drop every recorded failure. For tests."""
+ """Drop every recorded failure.
+
+ For settings saves that change a provider, model or credentials — the
+ entries describe a setup that no longer exists — and for tests.
+ """
_failures.clear()
diff --git a/src/services/search_service.py b/src/services/search_service.py
index 9e16e01f3..762f4c671 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -430,8 +430,10 @@ class SearchService:
# Use the same credential-aware gateway as Langflow.
# Provider-qualified and legacy routes are resolved in
# one place and upstream credentials never leave OpenRAG.
+ # A query, so it must not queue behind bulk ingestion.
resp = await gateway_embeddings(
- {"model": space.route_model, "input": [query]}
+ {"model": space.route_model, "input": [query]},
+ interactive=True,
)
data = resp.get("data", [])
embedding = data[0].get("embedding") if data else None
diff --git a/src/services/status_checks.py b/src/services/status_checks.py
index f3da68da4..690929902 100644
--- a/src/services/status_checks.py
+++ b/src/services/status_checks.py
@@ -1,5 +1,7 @@
from time import perf_counter
+import httpx
+
from api.schemas.status import ComponentBuild, ComponentState, ComponentStatus
from config.settings import DOCLING_SERVE_URL, LANGFLOW_URL, clients, get_openrag_config
from services.component_logs import record_check_result
@@ -8,7 +10,36 @@ from utils.version_utils import OPENRAG_VERSION
logger = get_logger(__name__)
-_CHECK_TIMEOUT_S = 2.0
+# Per-phase httpx budget (connect, read, ... each), not a total deadline. Keep it well
+# below status_service.CHECK_TIMEOUT_S, or that outer deadline cancels the check first and
+# the specific diagnosis below is replaced by a generic UNKNOWN.
+_CHECK_TIMEOUT_S = 5.0
+
+
+def _record_http_check_failure(
+ component: str,
+ exc: Exception,
+ target_url: str,
+ *,
+ unreachable_message: str,
+ slow_message: str,
+) -> tuple[ComponentState, str, str]:
+ """Classify a failed HTTP probe and record it; returns (state, message, last_error).
+
+ A timeout after the connection opened means the service is busy, not down: a
+ single-worker Langflow or docling-serve stalls for seconds under ingest load and
+ then catches up. That reads as DEGRADED. A connect timeout or any other failure
+ means nothing answered, which is UNHEALTHY.
+ """
+ if isinstance(exc, httpx.TimeoutException) and not isinstance(exc, httpx.ConnectTimeout):
+ state, message = ComponentState.DEGRADED, slow_message
+ # str() of an httpx timeout is usually empty; say how long we waited instead.
+ detail = f"{type(exc).__name__} after {_CHECK_TIMEOUT_S}s — target: {target_url}"
+ else:
+ state, message = ComponentState.UNHEALTHY, unreachable_message
+ detail = f"{type(exc).__name__}: {exc} — target: {target_url}"
+ record_check_result(component, state, message, detail=detail)
+ return state, message, detail
async def check_openrag_backend() -> ComponentStatus:
@@ -109,16 +140,14 @@ async def check_docling() -> ComponentStatus:
)
last_error = message
except Exception as e:
- logger.warning("Docling status check failed", error=str(e))
- message = "Docling Serve unreachable"
- status = ComponentState.UNHEALTHY
- record_check_result(
+ logger.warning("Docling status check failed", error_type=type(e).__name__, error=str(e))
+ status, message, last_error = _record_http_check_failure(
"docling",
- ComponentState.UNHEALTHY,
- message,
- detail=f"{type(e).__name__}: {e} — target: {target_url}",
+ e,
+ target_url,
+ unreachable_message="Docling Serve unreachable",
+ slow_message="Docling Serve is slow to respond",
)
- last_error = f"{type(e).__name__}: {e} — target: {target_url}"
return ComponentStatus(
name="docling",
@@ -174,16 +203,14 @@ async def check_langflow() -> ComponentStatus:
)
last_error = message
except Exception as e:
- logger.warning("Langflow status check failed", error=str(e))
- message = "Langflow is unreachable"
- status = ComponentState.UNHEALTHY
- record_check_result(
+ logger.warning("Langflow status check failed", error_type=type(e).__name__, error=str(e))
+ status, message, last_error = _record_http_check_failure(
"langflow",
- ComponentState.UNHEALTHY,
- message,
- detail=f"{type(e).__name__}: {e} — target: {target_url}",
+ e,
+ target_url,
+ unreachable_message="Langflow is unreachable",
+ slow_message="Langflow is slow to respond",
)
- last_error = f"{type(e).__name__}: {e} — target: {target_url}"
return ComponentStatus(
name="langflow",
diff --git a/src/services/status_service.py b/src/services/status_service.py
index 5033a8d39..975e26c72 100644
--- a/src/services/status_service.py
+++ b/src/services/status_service.py
@@ -14,7 +14,11 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
-CHECK_TIMEOUT_S = 5.0
+# Total wall-clock deadline per check. It must exceed status_checks._CHECK_TIMEOUT_S, which
+# httpx applies per phase, so a slow service is diagnosed by its own check rather than
+# cancelled here into UNKNOWN. It is also the only bound on the OpenSearch check, and it
+# caps /status latency, since that endpoint waits for its slowest check.
+CHECK_TIMEOUT_S = 8.0
CHECK_SPECS = [check_openrag_backend, check_docling, check_langflow, check_opensearch]
diff --git a/src/utils/langflow_headers.py b/src/utils/langflow_headers.py
index b5059f25d..ca8ee071d 100644
--- a/src/utils/langflow_headers.py
+++ b/src/utils/langflow_headers.py
@@ -73,6 +73,7 @@ async def add_provider_credentials_to_headers(
flows_service=None,
jwt_token: str = None,
user_id: str | None = None,
+ purpose: str | None = None,
) -> None:
"""Add Langflow global variables for the OpenRAG LLM proxy and infra URLs.
@@ -81,7 +82,9 @@ async def add_provider_credentials_to_headers(
speak OpenAI-compatible HTTP to OpenRAG (`OPENRAG_LLM_BASE_URL`) and
authenticate with a short-lived hop token as `OPENRAG_LLM_TOKEN` — same
pattern as `OPENRAG_INGEST_TOKEN`, scoped to the LLM proxy only.
- Chat and embeddings share that base URL and hop token.
+ Chat and embeddings share that base URL and hop token. `purpose` is
+ stamped on the token (`HOP_PURPOSE_CHAT` / `HOP_PURPOSE_INGEST`) so the
+ gateway can tell a query embedding from a bulk ingest one.
NOTE: `headers` may hold a JWT after this call. Never log it directly —
use utils.logging_config.sanitize_headers() if a header dict must be logged.
@@ -92,7 +95,7 @@ async def add_provider_credentials_to_headers(
headers["X-LANGFLOW-GLOBAL-VAR-OPENRAG_LLM_BASE_URL"] = get_langflow_llm_base_url()
subject = (user_id or "").strip() or "anonymous"
- hop_token = LangflowLlmTokenService().create_token(user_id=subject)
+ hop_token = LangflowLlmTokenService().create_token(user_id=subject, purpose=purpose)
headers["X-LANGFLOW-GLOBAL-VAR-OPENRAG_LLM_TOKEN"] = hop_token
# Stock Language/Embedding Model nodes still bind api_key to OPENAI_API_KEY.
headers["X-LANGFLOW-GLOBAL-VAR-OPENAI_API_KEY"] = hop_token
diff --git a/src/utils/langflow_utils.py b/src/utils/langflow_utils.py
index 5c42ef861..492e8df48 100644
--- a/src/utils/langflow_utils.py
+++ b/src/utils/langflow_utils.py
@@ -232,6 +232,7 @@ async def enable_mcp_none_for_project(project_id: str | None, max_attempts: int
"PATCH",
f"/api/v1/mcp/project/{project_id}",
json={"settings": [], "auth_settings": {"auth_type": "none"}},
+ idempotent=True,
)
if patch_resp.status_code in (200, 201):
logger.info(
diff --git a/tests/unit/api/test_docling_health.py b/tests/unit/api/test_docling_health.py
new file mode 100644
index 000000000..0ca1b577f
--- /dev/null
+++ b/tests/unit/api/test_docling_health.py
@@ -0,0 +1,84 @@
+"""`GET /docling/health`: a busy docling-serve is degraded, not stopped."""
+
+import json
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from api import docling
+
+
+def _patch_client(monkeypatch, *, raises=None, status_code=200):
+ client = MagicMock()
+ if raises is not None:
+ client.get = AsyncMock(side_effect=raises)
+ else:
+ response = MagicMock(spec=httpx.Response)
+ response.status_code = status_code
+ client.get = AsyncMock(return_value=response)
+ client.__aenter__ = AsyncMock(return_value=client)
+ client.__aexit__ = AsyncMock(return_value=False)
+ monkeypatch.setattr(docling.httpx, "AsyncClient", MagicMock(return_value=client))
+ return client
+
+
+async def _call():
+ response = await docling.health(MagicMock(), user=None)
+ return response.status_code, json.loads(response.body)
+
+
+async def test_healthy(monkeypatch):
+ client = _patch_client(monkeypatch, status_code=200)
+
+ status_code, body = await _call()
+
+ assert status_code == 200
+ assert body["status"] == "healthy"
+ assert client.get.call_args.kwargs["timeout"] == docling._HEALTH_CHECK_TIMEOUT_S
+
+
+ "exc", [httpx.ReadTimeout(""), httpx.WriteTimeout(""), httpx.PoolTimeout("")]
+)
+async def test_slow_reply_is_degraded_not_unhealthy(monkeypatch, exc):
+ _patch_client(monkeypatch, raises=exc)
+
+ status_code, body = await _call()
+
+ assert status_code == 200
+ assert body["status"] == "degraded"
+ assert "slow to respond" in body["message"]
+
+
+async def test_connect_timeout_is_unhealthy(monkeypatch):
+ _patch_client(monkeypatch, raises=httpx.ConnectTimeout(""))
+
+ status_code, body = await _call()
+
+ assert status_code == 503
+ assert body["status"] == "unhealthy"
+ assert body["message"] == "Connection timeout"
+
+
+async def test_connection_refused_is_unhealthy(monkeypatch):
+ _patch_client(monkeypatch, raises=httpx.ConnectError("refused"))
+
+ status_code, body = await _call()
+
+ assert status_code == 503
+ assert body["status"] == "unhealthy"
+
+
+async def test_non_200_is_unhealthy(monkeypatch):
+ _patch_client(monkeypatch, status_code=500)
+
+ status_code, body = await _call()
+
+ assert status_code == 503
+ assert body["status"] == "unhealthy"
diff --git a/tests/unit/api/test_provider_health_latch.py b/tests/unit/api/test_provider_health_latch.py
new file mode 100644
index 000000000..6e47b0992
--- /dev/null
+++ b/tests/unit/api/test_provider_health_latch.py
@@ -0,0 +1,171 @@
+"""The health banner has to be able to clear itself.
+
+`provider_error_log` is erased by the next call to a provider that succeeds,
+and until this was wired up the only such call was chat traffic. One failed
+turn latched the banner, the frontend then polled every 5s with
+`test_completion` while it stayed latched, and a provider that was
+demonstrably serving kept being reported broken — offering "Fix Setup" for a
+setup that had just passed its own check — until the entry went stale a
+quarter of an hour later.
+
+It must not clear itself on weaker evidence than the failure it erases,
+though. Only a probe that sent the same shape of request — a tool-calling
+completion for chat, a real embedding call for embeddings — may do it.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from api import provider_health
+from api.provider_validation import ProbeResult
+from services import provider_error_log
+
+#: What each kind of passing validation actually exercised.
+TOOL_COMPLETION = ProbeResult(model_probed=True, tools_exercised=True)
+PLAIN_COMPLETION = ProbeResult(model_probed=True)
+NO_CALL = ProbeResult()
+
+
[email protected](autouse=True)
+def _clean():
+ provider_error_log.clear()
+ yield
+ provider_error_log.clear()
+
+
+def _config():
+ providers = SimpleNamespace(
+ credential_values=lambda *_a, **_k: {},
+ stored_credentials=lambda *_a, **_k: {},
+ get_provider_config=lambda *_a, **_k: SimpleNamespace(api_key="k"),
+ )
+ return SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="gpt-oss-120b"),
+ knowledge=SimpleNamespace(embedding_provider="rhoai", embedding_model="granite-embedding"),
+ get_llm_provider_config=lambda: SimpleNamespace(api_key="k"),
+ get_embedding_provider_config=lambda: SimpleNamespace(api_key="k"),
+ )
+
+
+def _probes(monkeypatch, *, chat: ProbeResult, embedding: ProbeResult) -> None:
+ """Validation passes for both roles, having exercised what is given."""
+
+ async def ok(**kwargs):
+ return embedding if kwargs.get("embedding_model") else chat
+
+ monkeypatch.setattr(provider_health, "validate_provider_setup", ok)
+
+
+def _healthy_probe(monkeypatch):
+ """A provider whose probes all pass, with the caches out of the way."""
+ monkeypatch.setattr(provider_health, "get_openrag_config", _config)
+ monkeypatch.setattr(provider_health, "is_known_provider", lambda _p: True)
+
+ _probes(monkeypatch, chat=TOOL_COMPLETION, embedding=PLAIN_COMPLETION)
+ monkeypatch.setattr(provider_health.provider_health_cache, "cache_key", lambda **_k: "key")
+ monkeypatch.setattr(provider_health.provider_health_cache, "get", lambda _k: None)
+ monkeypatch.setattr(
+ provider_health.provider_health_cache, "set_and_release", lambda *_a, **_k: None
+ )
+ monkeypatch.setattr(provider_health.provider_health_cache, "release_error", lambda *_a: None)
+
+ async def acquire(_key):
+ return True
+
+ monkeypatch.setattr(provider_health.provider_health_cache, "acquire", acquire)
+
+ # Keep the corpus diagnostics out of it: unstubbed, they read the real
+ # config, can list models on a real cluster, and query OpenSearch.
+ async def _no_refresh(_provider):
+ return None
+
+ async def _no_stale(_provider):
+ return None
+
+ monkeypatch.setattr(provider_health, "_refresh_live_models", _no_refresh)
+ monkeypatch.setattr(provider_health, "_stale_embedding_spaces", _no_stale)
+
+
+async def test_a_passing_tool_calling_probe_clears_a_latched_chat_failure(_healthy_probe):
+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert provider_error_log.latest_failure("rhoai", "chat") is None
+
+
+async def test_a_passing_embedding_call_clears_a_latched_embedding_failure(_healthy_probe):
+ provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert provider_error_log.latest_failure("rhoai", "embedding") is None
+
+
+async def test_a_tool_less_completion_leaves_a_latched_chat_failure(monkeypatch, _healthy_probe):
+ """The LiteLLM probe (RHOAI and friends) sends a plain completion.
+
+ Agent traffic fails on tool calls, which that probe never makes, so it can
+ pass while every chat turn still fails.
+ """
+ _probes(monkeypatch, chat=PLAIN_COMPLETION, embedding=PLAIN_COMPLETION)
+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 503
+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
+
+
+async def test_a_validation_that_called_no_model_clears_nothing(monkeypatch, _healthy_probe):
+ """Azure's deployment listing, or a role with no model set, makes no model call."""
+ _probes(monkeypatch, chat=NO_CALL, embedding=NO_CALL)
+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
+ provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 503
+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
+ assert provider_error_log.latest_failure("rhoai", "embedding") == "embeddings fell over"
+
+
+async def test_a_model_free_check_proves_too_little_to_clear_anything(_healthy_probe):
+ """Without a completion there is no evidence the provider can serve a call."""
+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")
+
+ response = await provider_health.check_provider_health(test_completion=False, user=None)
+
+ assert response.status_code == 503
+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"
+
+
+async def test_a_failing_probe_leaves_the_recorded_failure_in_place(monkeypatch, _healthy_probe):
+ """A provider that is genuinely broken keeps saying so, in its own words."""
+
+ async def boom(**kwargs):
+ if kwargs.get("llm_model"):
+ raise RuntimeError("endpoint unreachable")
+ return PLAIN_COMPLETION
+
+ monkeypatch.setattr(provider_health, "validate_provider_setup", boom)
+ provider_error_log.record_failure("rhoai", "chat", "the real traffic failure")
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 503
+ # A real call still beats a probe when both have something to say.
+ assert provider_error_log.latest_failure("rhoai", "chat") == "the real traffic failure"
diff --git a/tests/unit/api/test_provider_health_stale_spaces.py b/tests/unit/api/test_provider_health_stale_spaces.py
new file mode 100644
index 000000000..f37d5dea9
--- /dev/null
+++ b/tests/unit/api/test_provider_health_stale_spaces.py
@@ -0,0 +1,446 @@
+"""A vector space outlives the model that made it.
+
+A chunk records the space it was embedded in (`provider:model`), retrieval
+embeds the query once per space found in the corpus, and an `InferenceService`
+redeployed under a new `--served-model-name` leaves every earlier chunk
+pointing at a model that is gone. Nothing else in the health path looks at the
+corpus: the *configured* model is still fine, so every probe passes and the
+banner stays green while those documents quietly drop out of vector search.
+These pin that it gets said instead — as a warning beside the verdict, not as a
+provider failure, because the provider is serving and its setup is not what
+needs fixing.
+"""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from api import provider_health
+from api.provider_validation import ProbeResult
+from services import provider_error_log
+
+SERVED = "redhataigranite-embedding-engl"
+STALE = "granite-embedding-english-r2"
+
+
[email protected](autouse=True)
+def _clean():
+ provider_error_log.clear()
+ yield
+ provider_error_log.clear()
+
+
+def _config():
+ providers = SimpleNamespace(
+ credential_values=lambda *_a, **_k: {},
+ stored_credentials=lambda *_a, **_k: {},
+ get_provider_config=lambda *_a, **_k: SimpleNamespace(api_key="k"),
+ )
+ return SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="gpt-oss-120b"),
+ knowledge=SimpleNamespace(embedding_provider="rhoai", embedding_model=SERVED),
+ get_llm_provider_config=lambda: SimpleNamespace(api_key="k"),
+ get_embedding_provider_config=lambda: SimpleNamespace(api_key="k"),
+ )
+
+
+def _healthy_probe(monkeypatch):
+ """Everything the probes touch passes, so only the corpus can speak."""
+ monkeypatch.setattr(provider_health, "get_openrag_config", _config)
+ monkeypatch.setattr(provider_health, "is_known_provider", lambda _p: True)
+
+ async def ok(**_kwargs):
+ return ProbeResult()
+
+ monkeypatch.setattr(provider_health, "validate_provider_setup", ok)
+ monkeypatch.setattr(provider_health.provider_health_cache, "cache_key", lambda **_k: "key")
+ monkeypatch.setattr(provider_health.provider_health_cache, "get", lambda _k: None)
+ monkeypatch.setattr(
+ provider_health.provider_health_cache, "set_and_release", lambda *_a, **_k: None
+ )
+ monkeypatch.setattr(provider_health.provider_health_cache, "release_error", lambda *_a: None)
+
+ async def acquire(_key):
+ return True
+
+ monkeypatch.setattr(provider_health.provider_health_cache, "acquire", acquire)
+
+ async def no_refresh(_provider):
+ return None
+
+ monkeypatch.setattr(provider_health, "_refresh_live_models", no_refresh)
+
+
+def _served(monkeypatch, models):
+ import enhancements.providers.registry as registry
+
+ monkeypatch.setattr(registry, "live_models_for", lambda _provider, _kind: models)
+
+
+def _indexed(monkeypatch, models):
+ async def spaces(_provider):
+ return models
+
+ monkeypatch.setattr(provider_health, "_indexed_spaces", spaces)
+
+
+def _body(response):
+ return json.loads(response.body)
+
+
+async def test_a_space_the_endpoint_no_longer_serves_is_a_warning_not_a_failure(
+ monkeypatch, _healthy_probe
+):
+ _served(monkeypatch, (SERVED,))
+ _indexed(monkeypatch, (SERVED, STALE))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ # The provider is serving; only the corpus is out of date.
+ assert response.status_code == 200
+ body = _body(response)
+ assert body["status"] == "healthy"
+ [warning] = body["warnings"]
+ assert warning["code"] == provider_health.STALE_EMBEDDING_SPACE
+ assert warning["provider"] == "rhoai"
+ assert warning["models"] == [STALE]
+ assert warning["served"] == [SERVED]
+ assert "re-ingested or deleted" in warning["message"]
+
+
+async def test_the_warning_says_search_degrades_rather_than_fails(monkeypatch, _healthy_probe):
+ """Retrieval skips a space it cannot embed for; it does not fail the search."""
+ _served(monkeypatch, (SERVED,))
+ _indexed(monkeypatch, (STALE,))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ message = _body(response)["warnings"][0]["message"]
+ assert "keyword only" in message
+ assert "will fail" not in message
+
+
+async def test_a_corpus_that_matches_what_is_served_says_nothing(monkeypatch, _healthy_probe):
+ _served(monkeypatch, (SERVED,))
+ _indexed(monkeypatch, (SERVED,))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert _body(response)["warnings"] == []
+
+
+async def test_an_unknown_listing_is_not_evidence_of_a_stale_corpus(monkeypatch, _healthy_probe):
+ """An unreachable cluster must not read as every document being stale."""
+ _served(monkeypatch, None)
+ _indexed(monkeypatch, (STALE,))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert _body(response)["warnings"] == []
+
+
+async def test_a_provider_serving_no_embedding_model_makes_every_space_stale(
+ monkeypatch, _healthy_probe
+):
+ """An empty listing is an answer, not an unknown: the provider listed
+ itself and serves nothing to embed a query with, so every indexed space
+ has stopped being searchable by meaning."""
+ _served(monkeypatch, ())
+ _indexed(monkeypatch, (SERVED, STALE))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ [warning] = _body(response)["warnings"]
+ assert warning["code"] == provider_health.STALE_EMBEDDING_SPACE
+ assert warning["models"] == [SERVED, STALE]
+ assert warning["served"] == []
+ assert "lists no embedding models" in warning["message"]
+ assert "it serves: )" not in warning["message"]
+
+
+async def test_an_empty_listing_with_an_empty_corpus_says_nothing(monkeypatch, _healthy_probe):
+ _served(monkeypatch, ())
+ _indexed(monkeypatch, None)
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert _body(response)["warnings"] == []
+
+
+async def test_an_empty_corpus_is_not_evidence_of_drift(monkeypatch, _healthy_probe):
+ _served(monkeypatch, (SERVED,))
+ _indexed(monkeypatch, None)
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 200
+ assert _body(response)["warnings"] == []
+
+
+async def test_a_real_failure_keeps_its_own_words_and_the_warning_rides_beside_it(
+ monkeypatch, _healthy_probe
+):
+ async def boom(**kwargs):
+ if kwargs.get("embedding_model"):
+ raise RuntimeError("endpoint unreachable")
+ return ProbeResult()
+
+ monkeypatch.setattr(provider_health, "validate_provider_setup", boom)
+ _served(monkeypatch, (SERVED,))
+ _indexed(monkeypatch, (STALE,))
+
+ response = await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert response.status_code == 503
+ body = _body(response)
+ # The failure is reported as itself, not diluted by corpus advice...
+ assert body["embedding_error"] == "endpoint unreachable"
+ # ...and the corpus advice is still there for when the provider recovers.
+ assert body["warnings"][0]["models"] == [STALE]
+
+
+# --------------------------------------------------------------------------
+# Refreshing what the provider serves
+# --------------------------------------------------------------------------
+
+
+def _refreshes(monkeypatch):
+ """Record which providers the catalogue is asked to re-list."""
+ import services.model_catalog as model_catalog
+
+ asked: list[str | None] = []
+
+ async def refresh(provider=None):
+ asked.append(provider)
+
+ monkeypatch.setattr(model_catalog, "refresh_live_models", refresh)
+ return asked
+
+
+async def test_only_the_checked_provider_is_refreshed(monkeypatch, _refreshes):
+ import enhancements.providers.registry as registry
+
+ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace(fetch_models=lambda _c: None))
+
+ await provider_health._refresh_live_models("rhoai")
+
+ assert _refreshes == ["rhoai"]
+
+
+async def test_a_provider_that_cannot_list_its_models_is_not_refreshed(monkeypatch, _refreshes):
+ """openai and friends have no enhancement: nothing to re-list, no call made."""
+ import enhancements.providers.registry as registry
+
+ monkeypatch.setattr(registry, "get", lambda _p: None)
+ await provider_health._refresh_live_models("openai")
+
+ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace())
+ await provider_health._refresh_live_models("listless")
+
+ assert _refreshes == []
+
+
+async def test_a_slow_cluster_cannot_stall_the_health_check(monkeypatch):
+ import asyncio
+
+ import enhancements.providers.registry as registry
+ import services.model_catalog as model_catalog
+
+ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace(fetch_models=lambda _c: None))
+
+ async def hang(provider=None):
+ await asyncio.sleep(60)
+
+ monkeypatch.setattr(model_catalog, "refresh_live_models", hang)
+ monkeypatch.setattr(provider_health, "_MODEL_REFRESH_TIMEOUT_SECONDS", 0.01)
+
+ # Returns (quietly) instead of raising or waiting out the listing.
+ await asyncio.wait_for(provider_health._refresh_live_models("rhoai"), timeout=1)
+
+
+async def test_the_health_check_refreshes_the_embedding_provider(monkeypatch, _healthy_probe):
+ asked: list[str] = []
+
+ async def refresh(provider):
+ asked.append(provider)
+
+ monkeypatch.setattr(provider_health, "_refresh_live_models", refresh)
+ _served(monkeypatch, None)
+
+ await provider_health.check_provider_health(test_completion=True, user=None)
+
+ assert asked == ["rhoai"]
+
+
+# --------------------------------------------------------------------------
+# Reading the corpus
+# --------------------------------------------------------------------------
+
+
+def _aggregation(*space_ids):
+ return {
+ "aggregations": {
+ "embedding_spaces": {
+ "buckets": [{"key": {"space_id": space_id}} for space_id in space_ids]
+ }
+ }
+ }
+
+
+def _opensearch(monkeypatch):
+ """Swap the admin client for one that replays a canned aggregation."""
+
+ def install(result, *, fails=False):
+ class _Client:
+ async def search(self, **_kwargs):
+ if fails:
+ raise RuntimeError("opensearch is down")
+ return result
+
+ monkeypatch.setattr(
+ "config.settings.clients", SimpleNamespace(opensearch=_Client()), raising=False
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents", raising=False)
+
+ return install
+
+
+def _page(*space_ids, after=None, **extra):
+ """One composite-aggregation page; `after` is the cursor it hands back."""
+ result = _aggregation(*space_ids)
+ if after is not None:
+ result["aggregations"]["embedding_spaces"]["after_key"] = {"space_id": after}
+ result.update(extra)
+ return result
+
+
+def _paged_opensearch(monkeypatch):
+ """An admin client that serves `pages` in order and records each request."""
+
+ def install(*pages):
+ requests: list[dict] = []
+ remaining = list(pages)
+
+ class _Client:
+ async def search(self, **kwargs):
+ requests.append(kwargs)
+ return remaining.pop(0)
+
+ monkeypatch.setattr(
+ "config.settings.clients", SimpleNamespace(opensearch=_Client()), raising=False
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents", raising=False)
+ return requests
+
+ return install
+
+
+def _after(request):
+ return request["body"]["aggs"]["embedding_spaces"]["composite"].get("after")
+
+
+async def test_indexed_spaces_follows_the_cursor_past_the_first_page(
+ monkeypatch, _paged_opensearch
+):
+ """A stale space on page two is still a stale space."""
+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_PAGE_SIZE", 2)
+ requests = _paged_opensearch(
+ _page("openai:text-embedding-3-small", f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"),
+ _page(f"rhoai:{STALE}"),
+ )
+
+ assert await provider_health._indexed_spaces("rhoai") == (SERVED, STALE)
+ assert [_after(request) for request in requests] == [None, {"space_id": f"rhoai:{SERVED}"}]
+
+
+async def test_indexed_spaces_stops_at_a_short_page(_paged_opensearch):
+ """The last page still carries an `after_key`; following it is a wasted read."""
+ requests = _paged_opensearch(_page(f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"))
+
+ assert await provider_health._indexed_spaces("rhoai") == (SERVED,)
+ assert len(requests) == 1
+
+
+async def test_indexed_spaces_refuses_partial_results(_paged_opensearch):
+ """A failed shard must raise, not hand back fewer buckets as if complete."""
+ requests = _paged_opensearch(_page(f"rhoai:{SERVED}"))
+
+ await provider_health._indexed_spaces("rhoai")
+
+ assert requests[0]["params"] == {"allow_partial_search_results": "false"}
+
+
+async def test_indexed_spaces_is_unknown_when_the_read_timed_out(_paged_opensearch):
+ _paged_opensearch(_page(f"rhoai:{SERVED}", timed_out=True))
+
+ assert await provider_health._indexed_spaces("rhoai") is None
+
+
+async def test_indexed_spaces_is_unknown_past_the_page_limit(monkeypatch, _paged_opensearch):
+ """Running out of pages before the cursor does is not knowing, not a full list."""
+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_PAGE_SIZE", 1)
+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_MAX_PAGES", 2)
+ _paged_opensearch(
+ _page(f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"),
+ _page(f"rhoai:{STALE}", after=f"rhoai:{STALE}"),
+ )
+
+ assert await provider_health._indexed_spaces("rhoai") is None
+
+
+async def test_indexed_spaces_returns_this_providers_models(_opensearch):
+ _opensearch(_aggregation(f"rhoai:{SERVED}", f"rhoai:{STALE}"))
+
+ assert await provider_health._indexed_spaces("rhoai") == (SERVED, STALE)
+
+
+async def test_indexed_spaces_ignores_other_providers(_opensearch):
+ """A corpus can hold spaces from a provider that is not the one selected."""
+ _opensearch(_aggregation("openai:text-embedding-3-small", f"rhoai:{SERVED}"))
+
+ assert await provider_health._indexed_spaces("rhoai") == (SERVED,)
+
+
+async def test_indexed_spaces_is_unknown_when_opensearch_cannot_answer(_opensearch):
+ _opensearch(None, fails=True)
+
+ assert await provider_health._indexed_spaces("rhoai") is None
+
+
+async def test_indexed_spaces_is_unknown_when_nothing_is_indexed(_opensearch):
+ _opensearch(_aggregation())
+
+ assert await provider_health._indexed_spaces("rhoai") is None
diff --git a/tests/unit/api/test_provider_validation_probe_result.py b/tests/unit/api/test_provider_validation_probe_result.py
new file mode 100644
index 000000000..3182d26ed
--- /dev/null
+++ b/tests/unit/api/test_provider_validation_probe_result.py
@@ -0,0 +1,152 @@
+"""`validate_provider_setup` says what a passing validation actually exercised.
+
+The health check erases a real-traffic failure only on evidence as strong as
+that failure, so each branch has to report honestly: a deployment listing, a
+key check, or a missing model is not a model call, and the LiteLLM probe's
+plain completion is not a tool-calling one.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from api import provider_validation
+from api.provider_validation import ProbeResult, validate_provider_setup
+
+
+def calls(monkeypatch):
+ """Stub every probe the validator can reach, recording which one ran."""
+ ran: list[str] = []
+
+ def stub(name):
+ async def probe(*_args, **_kwargs):
+ ran.append(name)
+
+ return probe
+
+ for name in (
+ "_test_litellm_provider",
+ "test_lightweight_health",
+ "test_embedding",
+ "test_completion_with_tools",
+ ):
+ monkeypatch.setattr(provider_validation, name, stub(name))
+ monkeypatch.setattr(provider_validation, "get_provider_enhancement", lambda _p: None)
+ return ran
+
+
+async def test_a_native_tool_calling_completion_is_reported_as_one(calls):
+ result = await validate_provider_setup(
+ provider="openai", api_key="k", llm_model="gpt-4o", test_completion=True
+ )
+
+ assert calls == ["test_completion_with_tools"]
+ assert result == ProbeResult(model_probed=True, tools_exercised=True)
+
+
+async def test_a_native_embedding_call_probes_the_model(calls):
+ result = await validate_provider_setup(
+ provider="openai",
+ api_key="k",
+ embedding_model="text-embedding-3-small",
+ test_completion=True,
+ )
+
+ assert calls == ["test_embedding"]
+ assert result == ProbeResult(model_probed=True)
+
+
+async def test_the_litellm_probe_calls_the_model_without_tools(monkeypatch, calls):
+ """RHOAI and every other enhancement: a plain `Reply with OK.` completion."""
+ monkeypatch.setattr(
+ provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()
+ )
+ monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})
+
+ result = await validate_provider_setup(
+ provider="rhoai", llm_model="granite", test_completion=True, credentials={}
+ )
+
+ assert calls == ["_test_litellm_provider"]
+ assert result == ProbeResult(model_probed=True, tools_exercised=False)
+
+
+def enhancement(monkeypatch):
+ monkeypatch.setattr(
+ provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()
+ )
+ monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})
+
+
[email protected]("model", [{"llm_model": "granite"}, {"embedding_model": "granite-emb"}])
+async def test_a_polled_enhancement_check_calls_no_model(calls, enhancement, model):
+ """The 30s health poll must not run inference on RHOAI / watsonx.ai on-prem.
+
+ It held a GPU slot (or billed) on every poll, and its pooled connections
+ raced vLLM's 5s keep-alive into "Server disconnected" banners.
+ """
+ result = await validate_provider_setup(provider="rhoai", credentials={}, **model)
+
+ assert calls == ["test_lightweight_health"]
+ assert result == ProbeResult()
+
+
+async def test_a_save_still_probes_the_enhancement_model(calls, enhancement):
+ """`verify_model` keeps the pre-save check that rejects an unserved model."""
+ result = await validate_provider_setup(
+ provider="rhoai", llm_model="granite", credentials={}, verify_model=True
+ )
+
+ assert calls == ["_test_litellm_provider"]
+ assert result == ProbeResult(model_probed=True, tools_exercised=False)
+
+
+async def test_verify_model_does_not_change_a_native_provider(calls):
+ """Native providers keep their own lightweight check on save."""
+ result = await validate_provider_setup(
+ provider="openai", api_key="k", llm_model="gpt-4o", verify_model=True
+ )
+
+ assert calls == ["test_lightweight_health"]
+ assert result == ProbeResult()
+
+
+async def test_azure_lists_deployments_and_calls_no_model(calls):
+ result = await validate_provider_setup(
+ provider="azure",
+ api_key="k",
+ endpoint="https://example.openai.azure.com",
+ llm_model="my-deployment",
+ test_completion=True,
+ )
+
+ assert calls == ["test_lightweight_health"]
+ assert result == ProbeResult()
+
+
+async def test_no_model_means_no_call(calls):
+ """Azure after #2384 leaves the model empty; nothing is probed then."""
+ result = await validate_provider_setup(provider="openai", api_key="k", test_completion=True)
+
+ assert calls == []
+ assert result == ProbeResult()
+
+
+async def test_a_lightweight_check_calls_no_model(calls):
+ result = await validate_provider_setup(provider="openai", api_key="k", llm_model="gpt-4o")
+
+ assert calls == ["test_lightweight_health"]
+ assert result == ProbeResult()
diff --git a/tests/unit/api/test_settings_endpoints.py b/tests/unit/api/test_settings_endpoints.py
index 95a9f107a..8acb27f34 100644
--- a/tests/unit/api/test_settings_endpoints.py
+++ b/tests/unit/api/test_settings_endpoints.py
@@ -355,6 +355,53 @@ async def test_update_settings_validates_enhancement_credentials_before_saving()
save.assert_not_called()
+ "fields",
+ [
+ {"llm_provider": "rhoai", "llm_model": "granite-chat"},
+ {"embedding_provider": "rhoai", "embedding_model": "granite-embedding"},
+ ],
+ ids=["llm", "embedding"],
+)
+async def test_update_settings_asks_for_a_real_model_probe(fields):
+ """A model switch is verified for real, not with the polled lightweight check.
+
+ For a provider enhancement only a real call rejects a model the cluster
+ does not serve; without `verify_model` the validator would run the
+ model-free check and the switch would save, failing later in chat.
+ """
+ from api.settings.endpoints import update_settings
+ from config.config_manager import OpenRAGConfig
+
+ config = OpenRAGConfig.from_dict({})
+ config.edited = True
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+
+ with (
+ patch("api.settings.endpoints.get_openrag_config", return_value=config),
+ patch(
+ "api.settings.endpoints.validate_provider_setup",
+ new_callable=AsyncMock,
+ side_effect=Exception("stop after validation arguments are captured"),
+ ) as validate,
+ patch("api.settings.endpoints.config_manager.save_config_file") as save,
+ ):
+ response = await update_settings(
+ body=SettingsUpdateBody(**fields),
+ session_manager=AsyncMock(),
+ user=MagicMock(spec=User),
+ models_service=MagicMock(),
+ rbac=rbac,
+ )
+
+ assert response.status_code == 400
+ validate.assert_awaited_once()
+ assert validate.await_args.kwargs["verify_model"] is True
+ save.assert_not_called()
+
+
@pytest.mark.asyncio
async def test_update_settings_skips_pre_save_check_for_litellm_only_providers():
"""Providers without an enhancement have no model-free probe to run."""
@@ -742,3 +789,75 @@ async def test_onboarding_caps_chunk_size_for_watsonx_onprem_embeddings():
saved_config = save_config.call_args.args[0]
assert saved_config.knowledge.chunk_size == 500
assert response.chunk_size_adjusted_to == 500
+
+
+async def _save_settings(body: SettingsUpdateBody) -> None:
+ from api.settings.endpoints import update_settings
+ from config.config_manager import OpenRAGConfig
+
+ config = OpenRAGConfig.from_dict({})
+ config.edited = True
+ rbac = MagicMock()
+ rbac.has_permission = AsyncMock(return_value=True)
+
+ with (
+ patch("api.settings.endpoints.get_openrag_config", return_value=config),
+ patch("api.settings.endpoints.validate_provider_setup", new_callable=AsyncMock),
+ patch("api.settings.endpoints.config_manager.save_config_file", return_value=True),
+ patch("api.settings.endpoints.clients.refresh_patched_client", new_callable=AsyncMock),
+ patch(
+ "api.settings.endpoints._run_async_post_save_langflow_updates",
+ new_callable=AsyncMock,
+ ),
+ ):
+ response = await update_settings(
+ body=body,
+ session_manager=AsyncMock(),
+ user=MagicMock(spec=User),
+ models_service=MagicMock(),
+ rbac=rbac,
+ )
+ assert getattr(response, "status_code", 200) == 200
+
+
+ "body",
+ [
+ SettingsUpdateBody(llm_model="qwen2.5-1.5b-instruct"),
+ SettingsUpdateBody(
+ provider_credentials={
+ "openai_like": {"api_base": "https://llm.example", "api_key": "k"}
+ }
+ ),
+ ],
+ ids=["model", "credentials"],
+)
+async def test_provider_settings_save_clears_recorded_failures(body):
+ """A recorded failure describes the setup the save just replaced.
+
+ Without this, a provider whose probe cannot clear the banner kept showing
+ the old failure after the operator fixed the model, until it went stale.
+ """
+ from services import provider_error_log
+
+ provider_error_log.clear()
+ provider_error_log.record_failure("rhoai", "chat", "old failure")
+ try:
+ await _save_settings(body)
+ assert provider_error_log.latest_failure("rhoai", "chat") is None
+ finally:
+ provider_error_log.clear()
+
+
+async def test_non_provider_settings_save_keeps_recorded_failures():
+ from services import provider_error_log
+
+ provider_error_log.clear()
+ provider_error_log.record_failure("rhoai", "chat", "still failing")
+ try:
+ await _save_settings(SettingsUpdateBody(system_prompt="Be brief."))
+ assert provider_error_log.latest_failure("rhoai", "chat") == "still failing"
+ finally:
+ provider_error_log.clear()
diff --git a/tests/unit/api/test_v1_llm.py b/tests/unit/api/test_v1_llm.py
index 19576494d..278a39c37 100644
--- a/tests/unit/api/test_v1_llm.py
+++ b/tests/unit/api/test_v1_llm.py
@@ -2,6 +2,7 @@
import inspect
import json
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -124,6 +125,36 @@ async def test_embeddings_delegates_to_gateway(monkeypatch):
assert data["data"][0]["embedding"] == [0.2]
+ ("purpose", "interactive"),
+ [("chat", True), ("ingest", False), (None, False)],
+)
+async def test_embeddings_lane_follows_the_hop_token_purpose(monkeypatch, purpose, interactive):
+ gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})
+ monkeypatch.setattr(v1_llm, "embeddings", gateway)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})
+ request.state = SimpleNamespace(llm_hop_purpose=purpose)
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert gateway.await_args.kwargs == {"interactive": interactive}
+
+
+async def test_embeddings_without_a_hop_token_take_the_bulk_lane(monkeypatch):
+ gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})
+ monkeypatch.setattr(v1_llm, "embeddings", gateway)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})
+ request.state = SimpleNamespace() # API key / JWT: no hop purpose recorded
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert gateway.await_args.kwargs == {"interactive": False}
+
+
@pytest.mark.asyncio
async def test_list_openai_models_hides_catalog_exception_text(monkeypatch):
"""CodeQL py/stack-trace-exposure: the exception text must not reach the caller."""
diff --git a/tests/unit/config/test_cors_allowed_origins.py b/tests/unit/config/test_cors_allowed_origins.py
index 2496ca2ce..949d5b3f7 100644
--- a/tests/unit/config/test_cors_allowed_origins.py
+++ b/tests/unit/config/test_cors_allowed_origins.py
@@ -64,8 +64,10 @@ def test_empty_string_yields_empty_list():
def test_unset_defaults_to_localhost():
env = _python_env({"PYTHONPATH": str(SRC)})
env.pop("CORS_ALLOWED_ORIGINS", None)
+ # bootstrap.load_env() would otherwise re-read a developer's local .env.
+ no_dotenv = "import dotenv; dotenv.load_dotenv = lambda *a, **k: False; "
result = subprocess.run(
- [sys.executable, "-c", PRINT_ORIGINS],
+ [sys.executable, "-c", no_dotenv + PRINT_ORIGINS],
capture_output=True,
text=True,
check=True,
diff --git a/tests/unit/config/test_generic_provider_config.py b/tests/unit/config/test_generic_provider_config.py
index 50743217d..981bac317 100644
--- a/tests/unit/config/test_generic_provider_config.py
+++ b/tests/unit/config/test_generic_provider_config.py
@@ -145,6 +145,7 @@ def _clear_rhoai_env(monkeypatch):
"RHOAI_EMBEDDINGS_ENDPOINT",
"RHOAI_API_KEY",
"RHOAI_TLS_VERIFY",
+ "RHOAI_EMBEDDING_MAX_CONCURRENCY",
"OPENAI_API_KEY",
"LLM_PROVIDER",
"EMBEDDING_PROVIDER",
@@ -230,6 +231,78 @@ def test_rhoai_tls_env_alone_reaches_a_stored_config(monkeypatch, tmp_path):
}
+def test_rhoai_embedding_concurrency_env_is_seeded(monkeypatch, tmp_path):
+ """The bulkhead limit ships with the rest of the provider's first-boot seed."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")
+ monkeypatch.setenv("RHOAI_EMBEDDING_MAX_CONCURRENCY", "8")
+
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ assert config.providers.custom["rhoai"].configured is True
+ assert config.providers.stored_credentials("rhoai")["embedding_max_concurrency"] == "8"
+
+
+def test_rhoai_env_credentials_accessor_maps_env_names_to_fields(monkeypatch):
+ """Each RHOAI_* variable lands under the credential field the enhancement reads."""
+ from config.settings import get_rhoai_env_credentials
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")
+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")
+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")
+ monkeypatch.setenv("RHOAI_EMBEDDING_MAX_CONCURRENCY", "6")
+
+ assert get_rhoai_env_credentials() == {
+ "api_base": "https://chat.svc:8443/v1",
+ "embedding_api_base": "https://embed.svc:8443/v1",
+ "api_key": "sha256~token",
+ "ssl_verify": "/var/run/secrets/service-ca.crt",
+ "embedding_max_concurrency": "6",
+ }
+
+
+def test_rhoai_seed_is_read_through_settings_boundary(monkeypatch, tmp_path):
+ """Seeding goes through `config.settings`, and stores values unvalidated: an
+ invalid concurrency limit is the enhancement's to reject (default + warning)."""
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+ monkeypatch.setattr(
+ "config.settings.get_rhoai_env_credentials",
+ lambda: {
+ "api_base": "https://chat.svc:8443/v1",
+ "embedding_api_base": None,
+ "api_key": "sha256~token",
+ "ssl_verify": None,
+ "embedding_max_concurrency": "not-a-number",
+ },
+ )
+
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ assert config.providers.custom["rhoai"].configured is True
+ assert config.providers.stored_credentials("rhoai") == {
+ "api_base": "https://chat.svc:8443/v1",
+ "api_key": "sha256~token",
+ "embedding_max_concurrency": "not-a-number",
+ }
+
+
+def test_rhoai_is_not_seeded_when_no_env_is_set(monkeypatch, tmp_path):
+ from config.config_manager import ConfigManager
+
+ _clear_rhoai_env(monkeypatch)
+
+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()
+
+ assert "rhoai" not in config.providers.custom
+
+
def _clear_watsonx_onprem_env(monkeypatch):
for name in (
"WATSONX_ENDPOINT_ONPREM",
diff --git a/tests/unit/config/test_langflow_request_retry.py b/tests/unit/config/test_langflow_request_retry.py
new file mode 100644
index 000000000..517eaad32
--- /dev/null
+++ b/tests/unit/config/test_langflow_request_retry.py
@@ -0,0 +1,191 @@
+"""`AppClients.langflow_request` retries only what is safe to repeat.
+
+Regression: a folder ingest failed on a real 502 from the embedding endpoint,
+Langflow answered the flow run with 500, and the backend replayed
+`POST /api/v1/run/{flow}`. The replay re-ran the Docling component against a
+task docling-serve had already deleted, and its 404 replaced the real error.
+"""
+
+import httpx
+import pytest
+
+import config.settings as settings
+from config.settings import AppClients
+
+RUN_ENDPOINT = "/api/v1/run/ingest-flow"
+
+
+class _FakeClient:
+ """Replays a scripted sequence of responses or transport errors."""
+
+ def __init__(self, outcomes):
+ self.outcomes = list(outcomes)
+ self.calls: list[tuple[str, str]] = []
+
+ async def request(self, *, method, url, headers, **kwargs):
+ self.calls.append((method, url))
+ outcome = self.outcomes.pop(0)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return httpx.Response(outcome, request=httpx.Request(method, url))
+
+
+def clients(monkeypatch):
+ async def api_key(force_regenerate: bool = False):
+ return "lf-key"
+
+ async def no_sleep(_delay):
+ return None
+
+ monkeypatch.setattr(settings, "get_langflow_api_key", api_key)
+ monkeypatch.setattr(settings, "LANGFLOW_REQUEST_RETRIES", 2)
+ monkeypatch.setattr(settings.asyncio, "sleep", no_sleep)
+ return AppClients()
+
+
+def _install(clients, monkeypatch, outcomes) -> _FakeClient:
+ fake = _FakeClient(outcomes)
+ monkeypatch.setattr(clients, "_ensure_langflow_http_client", lambda: fake)
+ return fake
+
+
+def _request_error(cls):
+ return cls("boom", request=httpx.Request("POST", "http://langflow"))
+
+
+async def test_a_failed_flow_run_is_not_replayed(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [500, 200])
+
+ response = await clients.langflow_request("POST", RUN_ENDPOINT, json={})
+
+ assert response.status_code == 500
+ assert len(fake.calls) == 1
+
+
[email protected]("status", [408, 500, 502, 504])
+async def test_post_is_not_retried_on_statuses_after_processing(clients, monkeypatch, status):
+ fake = _install(clients, monkeypatch, [status, 200])
+
+ response = await clients.langflow_request("POST", RUN_ENDPOINT)
+
+ assert response.status_code == status
+ assert len(fake.calls) == 1
+
+
[email protected]("status", [429, 503])
+async def test_post_is_retried_when_the_server_did_not_process_it(clients, monkeypatch, status):
+ fake = _install(clients, monkeypatch, [status, 200])
+
+ response = await clients.langflow_request("POST", RUN_ENDPOINT)
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 2
+
+
[email protected]("status", [408, 500, 502, 503, 504])
+async def test_get_is_retried_on_transient_statuses(clients, monkeypatch, status):
+ fake = _install(clients, monkeypatch, [status, status, 200])
+
+ response = await clients.langflow_request("GET", "/api/v1/flows/x")
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 3
+
+
+async def test_retries_stop_at_the_configured_budget(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [502, 502, 502, 200])
+
+ response = await clients.langflow_request("GET", "/api/v1/flows/x")
+
+ assert response.status_code == 502
+ assert len(fake.calls) == 3
+
+
+async def test_patch_is_not_retried_by_default(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [500, 200])
+
+ response = await clients.langflow_request("PATCH", "/api/v1/mcp/project/p")
+
+ assert response.status_code == 500
+ assert len(fake.calls) == 1
+
+
[email protected]("method", ["POST", "PATCH"])
+async def test_an_explicitly_idempotent_request_is_retried(clients, monkeypatch, method):
+ fake = _install(clients, monkeypatch, [500, 200])
+
+ response = await clients.langflow_request(method, "/api/v1/flows/x", idempotent=True)
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 2
+
+
+async def test_an_explicitly_non_idempotent_get_is_not_retried(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [500, 200])
+
+ response = await clients.langflow_request("GET", "/api/v1/flows/x", idempotent=False)
+
+ assert response.status_code == 500
+ assert len(fake.calls) == 1
+
+
[email protected]("error_cls", [httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout])
+async def test_post_is_retried_when_it_never_reached_langflow(clients, monkeypatch, error_cls):
+ fake = _install(clients, monkeypatch, [_request_error(error_cls), 200])
+
+ response = await clients.langflow_request("POST", RUN_ENDPOINT)
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 2
+
+
+ "error_cls", [httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.ReadError]
+)
+async def test_post_is_not_retried_after_it_may_have_run(clients, monkeypatch, error_cls):
+ fake = _install(clients, monkeypatch, [_request_error(error_cls), 200])
+
+ with pytest.raises(error_cls):
+ await clients.langflow_request("POST", RUN_ENDPOINT)
+
+ assert len(fake.calls) == 1
+
+
+async def test_get_is_retried_after_a_read_timeout(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [_request_error(httpx.ReadTimeout), 200])
+
+ response = await clients.langflow_request("GET", "/api/v1/flows/x")
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 2
+
+
+async def test_auth_refresh_still_applies_to_post(clients, monkeypatch):
+ fake = _install(clients, monkeypatch, [401, 200])
+
+ response = await clients.langflow_request("POST", RUN_ENDPOINT)
+
+ assert response.status_code == 200
+ assert len(fake.calls) == 2
+
+
+def test_idempotency_follows_rfc_9110_by_default():
+ for method in ("GET", "head", "OPTIONS", "PUT", "DELETE"):
+ assert settings._is_idempotent_request(method, None) is True
+ for method in ("POST", "PATCH", "patch"):
+ assert settings._is_idempotent_request(method, None) is False
+ assert settings._is_idempotent_request("POST", True) is True
+ assert settings._is_idempotent_request("GET", False) is False
diff --git a/tests/unit/dependencies/test_llm_proxy_user_purpose.py b/tests/unit/dependencies/test_llm_proxy_user_purpose.py
new file mode 100644
index 000000000..dda507a04
--- /dev/null
+++ b/tests/unit/dependencies/test_llm_proxy_user_purpose.py
@@ -0,0 +1,51 @@
+"""`get_llm_proxy_user` records which kind of Langflow run minted a hop token.
+
+The embeddings route reads it to give chat (query) embeddings their own lane
+beside bulk ingestion at a concurrency-limited provider.
+"""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+import dependencies
+from services.langflow_llm_token_service import HOP_PURPOSE_CHAT, LangflowLlmTokenService
+
+SECRET = "llm-hop-test-secret-with-32-bytes!!"
+
+
+def _request(token: str) -> SimpleNamespace:
+ return SimpleNamespace(headers={"authorization": f"Bearer {token}"}, state=SimpleNamespace())
+
+
[email protected]("purpose", [HOP_PURPOSE_CHAT, None])
+async def test_a_hop_token_records_its_purpose(purpose):
+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)
+ request = _request(service.create_token(user_id="alice", purpose=purpose))
+
+ user = await dependencies.get_llm_proxy_user(
+ request, token_service=service, api_key_service=None, session_manager=None
+ )
+
+ assert user.user_id == "alice"
+ assert user.provider == "langflow_llm"
+ assert request.state.user is user
+ assert request.state.llm_hop_purpose == purpose
+
+
+async def test_a_non_hop_token_records_no_purpose(monkeypatch):
+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)
+ api_user = SimpleNamespace(user_id="bob")
+ fallback = AsyncMock(return_value=api_user)
+ monkeypatch.setattr(dependencies, "resolve_api_key_user", fallback)
+ request = _request("not-a-hop-token")
+
+ user = await dependencies.get_llm_proxy_user(
+ request, token_service=service, api_key_service=None, session_manager=None
+ )
+
+ assert user is api_user
+ assert not hasattr(request.state, "llm_hop_purpose")
diff --git a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py
index 0fb891931..ac084eedb 100644
--- a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py
+++ b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py
@@ -98,11 +98,19 @@ def test_the_form_asks_for_two_endpoints_a_token_and_a_ca() -> None:
"""vLLM serves one model per InferenceService, so one endpoint is not enough."""
fields = {field["key"]: field for field in model_catalog.credential_fields(PROVIDER)}
- assert set(fields) == {"api_base", "embedding_api_base", "api_key", "ssl_verify"}
+ assert set(fields) == {
+ "api_base",
+ "embedding_api_base",
+ "api_key",
+ "ssl_verify",
+ "embedding_max_concurrency",
+ }
assert fields["api_base"]["field_type"] == "text"
assert fields["embedding_api_base"]["field_type"] == "text"
assert fields["api_key"]["field_type"] == "password"
assert fields["ssl_verify"]["field_type"] == "text"
+ assert fields["embedding_max_concurrency"]["field_type"] == "text"
+ assert fields["embedding_max_concurrency"]["required"] is False
def test_only_the_token_is_encrypted_at_rest() -> None:
@@ -147,6 +155,67 @@ def test_the_embedding_url_never_reaches_litellm(kind) -> None:
assert "embedding_api_base" not in rhoai.litellm_credentials(_stored(), kind=kind)
[email protected]("kind", ["chat", "embedding"])
+def test_the_concurrency_limit_never_reaches_litellm(kind) -> None:
+ """The gateway enforces the limit; forwarded, it would land in the request body."""
+ stored = _stored(embedding_max_concurrency="2")
+ assert "embedding_max_concurrency" not in rhoai.litellm_credentials(stored, kind=kind)
+
+
+# ---------------------------------------------------------------------------
+# Embedding concurrency limit
+# ---------------------------------------------------------------------------
+
+
+ ("value", "expected"),
+ [
+ (None, rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),
+ ("", rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),
+ (" ", rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),
+ ("1", 1),
+ (" 16 ", 16),
+ (8, 8),
+ ("0", None),
+ ],
+)
+def test_the_embedding_concurrency_limit_is_parsed(value, expected) -> None:
+ stored = _stored(embedding_max_concurrency=value)
+ assert rhoai.embedding_max_concurrency(stored) == expected
+
+
[email protected]("value", ["four", "-1", "2.5"])
+def test_an_invalid_limit_falls_back_to_the_default_not_to_unlimited(value, monkeypatch) -> None:
+ """Failing open would reintroduce the overload the limit exists to prevent."""
+ warnings: list[str] = []
+ monkeypatch.setattr(rhoai.logger, "warning", lambda message, **_: warnings.append(message))
+
+ stored = _stored(embedding_max_concurrency=value)
+ assert rhoai.embedding_max_concurrency(stored) == rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY
+ assert rhoai.embedding_max_concurrency(stored) == rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY
+ assert len(warnings) == 1
+
+
+def test_the_registry_exposes_the_limit() -> None:
+ assert registry.embedding_concurrency_for(PROVIDER, _stored(embedding_max_concurrency="3")) == 3
+ assert (
+ registry.embedding_concurrency_for(PROVIDER, _stored(embedding_max_concurrency="0")) is None
+ )
+
+
+def test_the_registry_reports_no_limit_for_providers_without_the_hook() -> None:
+ assert registry.embedding_concurrency_for("openai", {}) is None
+ assert registry.embedding_concurrency_for("watsonx_onprem", {}) is None
+
+
+def test_a_failing_hook_never_breaks_the_request(monkeypatch) -> None:
+ def broken(stored):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(rhoai, "embedding_max_concurrency", broken)
+ assert registry.embedding_concurrency_for(PROVIDER, _stored()) is None
+
+
def test_no_endpoint_means_no_credentials_at_all() -> None:
"""Not tidiness: LiteLLM falls back to HOSTED_VLLM_API_BASE from the process
environment when `api_base` is absent, so returning a bare token could send a
@@ -524,10 +593,17 @@ def test_a_kindless_enhancement_is_still_callable() -> None:
class _Response:
- def __init__(self, status_code: int, payload: Any = None, text: str = "") -> None:
+ def __init__(
+ self,
+ status_code: int,
+ payload: Any = None,
+ text: str = "",
+ headers: dict[str, str] | None = None,
+ ) -> None:
self.status_code = status_code
self._payload = payload
self.text = text
+ self.headers = headers or {}
def json(self) -> Any:
if self._payload is None:
@@ -987,6 +1063,181 @@ async def test_a_rejected_token_names_the_rolebinding(monkeypatch, status) -> No
await rhoai.lightweight_health_check(_stored())
+LOGIN_PAGE = (
+ "<!DOCTYPE html>\n<html><head><title>Log in</title></head>"
+ "<body><form action='/oauth/start'>Log in with OpenShift</form></body></html>"
+)
+LOGIN_REDIRECT = (
+ "https://oauth-openshift.apps.cluster.example/oauth/authorize"
+ "?client_id=system%3Aserviceaccount&state=abc123&redirect_uri=https%3A%2F%2Fx"
+)
+
+
+def _serve(monkeypatch, responses: dict[str, Any]) -> None:
+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses))
+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)
+
+
+async def test_a_login_page_answered_with_200_is_not_healthy(monkeypatch) -> None:
+ """An OAuth proxy or SSO gateway in front of the route answers 200 with its
+ login page; reading that as healthy defers the failure to the first chat."""
+ _serve(
+ monkeypatch,
+ {
+ f"{CHAT_BASE}/models": _Response(
+ 200, text=LOGIN_PAGE, headers={"content-type": "text/html; charset=utf-8"}
+ )
+ },
+ )
+
+ with pytest.raises(Exception, match="chat endpoint .* returned a web page") as error:
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+ assert "text/html" in str(error.value)
+ assert "<html" not in str(error.value).lower()
+
+
+async def test_a_login_page_without_a_content_type_is_still_recognised(monkeypatch) -> None:
+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(200, text=f" \n{LOGIN_PAGE}")})
+
+ with pytest.raises(Exception, match="returned a web page"):
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+
+ "body",
+ [{"status": "ok"}, [], {"data": "not-a-list"}, "ok"],
+ ids=["status-object", "bare-list", "data-not-list", "bare-string"],
+)
+async def test_a_200_that_is_not_a_model_listing_is_not_healthy(monkeypatch, body) -> None:
+ _serve(
+ monkeypatch,
+ {f"{CHAT_BASE}/models": _Response(200, body, headers={"content-type": "application/json"})},
+ )
+
+ with pytest.raises(Exception, match="OpenAI-compatible model list"):
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+
+async def test_an_empty_model_listing_still_passes(monkeypatch) -> None:
+ """Only the shape is checked: a vLLM pod still loading its weights is a
+ real endpoint, and the check does not depend on a model being chosen."""
+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(200, _models_body())})
+
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+
+async def test_a_login_page_on_the_embedding_endpoint_names_that_endpoint(monkeypatch) -> None:
+ _serve(
+ monkeypatch,
+ {
+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),
+ f"{EMBED_BASE}/models": _Response(
+ 200, text=LOGIN_PAGE, headers={"content-type": "text/html"}
+ ),
+ },
+ )
+
+ with pytest.raises(Exception, match="embeddings endpoint .* returned a web page"):
+ await rhoai.lightweight_health_check(_stored())
+
+
[email protected]("status", [301, 302, 303, 307, 308])
+async def test_a_redirect_to_a_login_page_says_so(monkeypatch, status) -> None:
+ """httpx does not follow redirects, so an OAuth proxy's 302 arrives as-is;
+ the message names where it pointed, without the OAuth query noise."""
+ _serve(
+ monkeypatch,
+ {
+ f"{CHAT_BASE}/models": _Response(
+ status,
+ text=LOGIN_PAGE,
+ headers={"content-type": "text/html", "location": LOGIN_REDIRECT},
+ )
+ },
+ )
+
+ with pytest.raises(Exception, match="redirected to") as error:
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+ message = str(error.value)
+ assert "https://oauth-openshift.apps.cluster.example/oauth/authorize" in message
+ assert "state=" not in message
+ assert "<html" not in message.lower()
+
+
+async def test_a_redirect_without_a_location_is_still_reported(monkeypatch) -> None:
+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(302)})
+
+ with pytest.raises(Exception, match="redirected to another page"):
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+
+async def test_an_html_error_page_is_described_not_quoted(monkeypatch) -> None:
+ """The router's own 403 page is HTML; quoting it would put markup in the
+ Settings error. The status-specific advice still applies."""
+ _serve(
+ monkeypatch,
+ {
+ f"{CHAT_BASE}/models": _Response(
+ 403, text=LOGIN_PAGE, headers={"content-type": "text/html"}
+ )
+ },
+ )
+
+ with pytest.raises(Exception, match="inferenceservices") as error:
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+ assert "HTML page" in str(error.value)
+ assert "<html" not in str(error.value).lower()
+
+
+ "response",
+ [
+ _Response(200, text=LOGIN_PAGE, headers={"content-type": "text/html"}),
+ _Response(200, {"status": "ok"}),
+ _Response(302, headers={"location": LOGIN_REDIRECT}),
+ ],
+ ids=["login-page", "wrong-shape", "redirect"],
+)
+async def test_a_proxy_answer_is_not_mistaken_for_a_bad_token(monkeypatch, response) -> None:
+ """The fix is the URL, not the token; a credential classification would
+ tell the operator to rotate a token that works."""
+ from api.provider_validation import is_provider_credential_error
+
+ _serve(monkeypatch, {f"{CHAT_BASE}/models": response})
+
+ with pytest.raises(Exception) as error:
+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))
+
+ assert not is_provider_credential_error(error.value)
+
+
+ ("body", "expected"),
+ [
+ ({"object": "list", "data": []}, True),
+ (_models_body(CHAT_MODEL), True),
+ ({"data": None}, False),
+ ({"object": "list"}, False),
+ ([], False),
+ (None, False),
+ ],
+)
+def test_a_model_listing_is_recognised_by_its_shape(body, expected) -> None:
+ assert rhoai.is_model_listing(body) is expected
+
+
@pytest.mark.asyncio
async def test_an_unreachable_endpoint_says_which_one(monkeypatch) -> None:
import httpx
diff --git a/tests/unit/enhancements/providers/test_registry_live_models.py b/tests/unit/enhancements/providers/test_registry_live_models.py
new file mode 100644
index 000000000..c55a9a312
--- /dev/null
+++ b/tests/unit/enhancements/providers/test_registry_live_models.py
@@ -0,0 +1,142 @@
+"""What a provider says it serves, read back from its own cache.
+
+`live_models_for` is the half of model-drift detection that must never guess:
+a provider that cannot list itself, or has not listed itself yet, is *unknown*,
+and a caller that treats unknown as "serves nothing" would tell an operator to
+change a setting that was right all along.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from enhancements.providers import registry
+
+
+class _Listing(SimpleNamespace):
+ """Stands in for `ClusterModels`: one attribute per endpoint."""
+
+
+def _enhancement(monkeypatch, module):
+ monkeypatch.setitem(registry._ENHANCEMENTS, "probe", module)
+
+
+def test_a_provider_with_no_enhancement_is_unknown():
+ assert registry.live_models_for("openai", "chat") is None
+ assert registry.live_models_for("", "chat") is None
+
+
+def test_an_enhancement_that_cannot_list_its_models_is_unknown(monkeypatch):
+ _enhancement(monkeypatch, SimpleNamespace(PROVIDER_KEY="probe"))
+
+ assert registry.live_models_for("probe", "chat") is None
+
+
+def test_nothing_cached_yet_is_unknown(monkeypatch):
+ _enhancement(monkeypatch, SimpleNamespace(cached_models=lambda: None))
+
+ assert registry.live_models_for("probe", "embedding") is None
+
+
+def test_a_cached_listing_is_split_by_endpoint(monkeypatch):
+ _enhancement(
+ monkeypatch,
+ SimpleNamespace(
+ cached_models=lambda: _Listing(
+ chat=("gpt-oss-120b",), embedding=("redhataigranite-embedding-engl",)
+ )
+ ),
+ )
+
+ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)
+ assert registry.live_models_for("probe", "embedding") == ("redhataigranite-embedding-engl",)
+
+
+def test_one_endpoint_can_be_known_while_the_other_is_not(monkeypatch):
+ """A rolling embedding deployment must not make the chat list unknown too."""
+ _enhancement(
+ monkeypatch,
+ SimpleNamespace(cached_models=lambda: _Listing(chat=("gpt-oss-120b",), embedding=None)),
+ )
+
+ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)
+ assert registry.live_models_for("probe", "embedding") is None
+
+
+def test_the_provider_key_is_matched_the_way_it_is_stored(monkeypatch):
+ _enhancement(
+ monkeypatch,
+ SimpleNamespace(cached_models=lambda: _Listing(chat=("m",), embedding=("m",))),
+ )
+
+ assert registry.live_models_for(" PROBE ", "chat") == ("m",)
+
+
+def test_a_listing_that_raises_is_unknown_rather_than_fatal(monkeypatch):
+ def boom():
+ raise RuntimeError("cache exploded")
+
+ _enhancement(monkeypatch, SimpleNamespace(cached_models=boom))
+
+ assert registry.live_models_for("probe", "chat") is None
+
+
+def test_a_known_empty_half_is_an_answer_not_unknown(monkeypatch):
+ """A provider that listed itself and serves no embedding model has said so;
+ folding that into unknown would hide every indexed space going stale."""
+ _enhancement(
+ monkeypatch,
+ SimpleNamespace(cached_models=lambda: _Listing(chat=("gpt-oss-120b",), embedding=())),
+ )
+
+ assert registry.live_models_for("probe", "embedding") == ()
+ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)
+
+
+def test_a_cached_list_is_handed_back_as_a_tuple(monkeypatch):
+ _enhancement(
+ monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=["a", "b"], embedding=[]))
+ )
+
+ assert registry.live_models_for("probe", "chat") == ("a", "b")
+ assert registry.live_models_for("probe", "embedding") == ()
+
+
[email protected]("listed", ["gpt-oss-120b", b"gpt-oss-120b", 42])
+def test_a_malformed_half_is_unknown_not_split_into_nonsense(monkeypatch, listed):
+ """A bare string would iterate into one-letter model ids."""
+ _enhancement(
+ monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=listed, embedding=()))
+ )
+
+ assert registry.live_models_for("probe", "chat") is None
+
+
+def test_a_half_the_listing_does_not_have_is_unknown(monkeypatch):
+ _enhancement(monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=("m",))))
+
+ assert registry.live_models_for("probe", "embedding") is None
+
+
+def test_watsonx_onprem_with_no_embedding_model_reports_an_empty_half(monkeypatch):
+ """The reviewer's case end to end: the on-prem enhancement caches a
+ successful listing whose embedding half is empty, and that must survive
+ the registry as `()`."""
+ import time
+
+ from enhancements.providers.watsonx import onprem
+
+ monkeypatch.setattr(
+ onprem,
+ "_models_cache",
+ {
+ "key": "k",
+ "at": time.monotonic(),
+ "value": onprem.ClusterModels(chat=("ibm/granite-3-8b-instruct",), embedding=()),
+ },
+ )
+
+ assert registry.live_models_for(onprem.PROVIDER_KEY, "embedding") == ()
+ assert registry.live_models_for(onprem.PROVIDER_KEY, "chat") == ("ibm/granite-3-8b-instruct",)
diff --git a/tests/unit/enhancements/providers/watsonx/test_onprem.py b/tests/unit/enhancements/providers/watsonx/test_onprem.py
index 1861c806d..930397531 100644
--- a/tests/unit/enhancements/providers/watsonx/test_onprem.py
+++ b/tests/unit/enhancements/providers/watsonx/test_onprem.py
@@ -719,7 +719,11 @@ async def test_switching_to_a_model_the_cluster_lacks_is_still_blocked(monkeypat
)
with pytest.raises(Exception, match="not supported"):
await provider_validation.validate_provider_setup(
- provider=PROVIDER, credentials=credentials, llm_model="ibm/nope", embedding_model=None
+ provider=PROVIDER,
+ credentials=credentials,
+ llm_model="ibm/nope",
+ embedding_model=None,
+ verify_model=True,
)
assert probes == ["real-call"], "a model in hand must be probed for real"
@@ -747,6 +751,7 @@ async def test_switching_an_embedding_model_is_probed_for_real(monkeypatch) -> N
credentials=credentials,
llm_model=None,
embedding_model="ibm/slate-125m-english-rtrvr-v2",
+ verify_model=True,
)
assert probes == ["watsonx/ibm/slate-125m-english-rtrvr-v2"]
@@ -1005,6 +1010,7 @@ async def test_validating_an_embedding_model_sends_a_list_not_a_string(monkeypat
),
llm_model=None,
embedding_model="ibm/slate-30m-english-rtrvr",
+ verify_model=True,
)
assert isinstance(sent["input"], list), sent["input"]
diff --git a/tests/unit/services/test_langflow_llm_token_service.py b/tests/unit/services/test_langflow_llm_token_service.py
index 25338ffd8..2c54ea089 100644
--- a/tests/unit/services/test_langflow_llm_token_service.py
+++ b/tests/unit/services/test_langflow_llm_token_service.py
@@ -10,6 +10,8 @@ import pytest
from services.document_index_writer import DocumentIndexContext
from services.langflow_ingest_token_service import LangflowIngestTokenService
from services.langflow_llm_token_service import (
+ HOP_PURPOSE_CHAT,
+ HOP_PURPOSE_INGEST,
LANGFLOW_LLM_AUDIENCE,
LangflowLlmTokenService,
langflow_hop_audience,
@@ -84,3 +86,20 @@ def test_wrong_scope_is_rejected():
def test_orag_key_is_not_a_hop_token():
assert langflow_hop_audience("orag_abc") is None
assert langflow_hop_audience("") is None
+
+
[email protected]("purpose", [HOP_PURPOSE_CHAT, HOP_PURPOSE_INGEST])
+def test_the_purpose_claim_round_trips(purpose):
+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)
+ token = service.create_token(user_id="alice", purpose=purpose)
+ claims = service.validate_claims(token)
+ assert claims["purpose"] == purpose
+ assert service.user_from_claims(claims).user_id == "alice"
+ # The user it identifies is the same either way.
+ assert service.validate_token(token).provider == "langflow_llm"
+
+
+def test_a_token_minted_without_a_purpose_has_no_claim():
+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)
+ claims = service.validate_claims(service.create_token(user_id="alice"))
+ assert "purpose" not in claims
diff --git a/tests/unit/services/test_langflow_mcp_service.py b/tests/unit/services/test_langflow_mcp_service.py
index 76ef6c7a7..b10640b49 100644
--- a/tests/unit/services/test_langflow_mcp_service.py
+++ b/tests/unit/services/test_langflow_mcp_service.py
@@ -124,5 +124,6 @@ async def test_patch_mcp_server_url_preserves_auth_settings(monkeypatch):
"url": "http://langflow:7860/api/v1/mcp/project/project-id/streamable",
"headers": expected_headers,
},
+ "idempotent": True,
}
]
diff --git a/tests/unit/services/test_llm_gateway.py b/tests/unit/services/test_llm_gateway.py
index 67b4d166c..cefbaab08 100644
--- a/tests/unit/services/test_llm_gateway.py
+++ b/tests/unit/services/test_llm_gateway.py
@@ -576,7 +576,21 @@ async def test_stream_sse_counts_tool_calls_as_usable_output(monkeypatch):
async def gen():
yield {
- "choices": [{"delta": {"tool_calls": [{"index": 0}]}, "finish_reason": "tool_calls"}]
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "search_documents", "arguments": "{}"},
+ }
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
}
[line async for line in llm_gateway._stream_sse(gen(), "openai", "gpt-4o-mini")]
@@ -1020,6 +1034,542 @@ async def test_stream_sse_logs_the_repair(monkeypatch):
assert fields["tool_calls"] == 1
+# --------------------------------------------------------------------------
+# Tool calls a provider splits across the wrong index
+# --------------------------------------------------------------------------
+
+
+def _orphaned_fragment_deltas():
+ """The gpt-oss-on-vLLM symptom: the closing brace lands under the next index.
+
+ Taken literally this truncates the real call's arguments *and* invents a
+ second call with no id and no name — the exact pair that makes the assistant
+ message the agent replays unacceptable to vLLM on every later turn.
+ """
+ opening = {
+ "id": "chatcmpl-tool-1",
+ "type": "function",
+ "index": 0,
+ "function": {"name": "search_documents", "arguments": ""},
+ }
+ fragments = ['{\n "query": "AMCOR', ' signatories"\n']
+ chunks = [{"choices": [{"index": 0, "delta": {"tool_calls": [opening]}}]}]
+ chunks += [
+ {
+ "choices": [
+ {"index": 0, "delta": {"tool_calls": [{"index": 0, "function": {"arguments": f}}]}}
+ ]
+ }
+ for f in fragments
+ ]
+ # The stray one: same content, wrong index, and nothing naming a call.
+ chunks.append(
+ {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {"tool_calls": [{"index": 1, "function": {"arguments": "}"}}]},
+ }
+ ]
+ }
+ )
+ chunks.append({"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]})
+ return chunks
+
+
+async def test_stream_sse_rejoins_a_fragment_misfiled_under_the_next_index():
+ from services import llm_gateway
+
+ async def gen():
+ for chunk in _orphaned_fragment_deltas():
+ yield chunk
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ calls = _streamed_tool_calls(lines)
+ assert len(calls) == 1, "the stray fragment must not become a second tool call"
+ assert calls[0]["id"] == "chatcmpl-tool-1"
+ assert calls[0]["function"]["name"] == "search_documents"
+ # The closing brace is back where it belongs, so the arguments parse.
+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "AMCOR signatories"}
+
+
+async def test_stream_sse_drops_a_buffered_call_that_names_no_tool(monkeypatch):
+ """Backstop: a call with no name can never be executed, so it must not ship."""
+ from services import llm_gateway
+
+ recorder = _RecordingLogger()
+ monkeypatch.setattr(llm_gateway, "logger", recorder)
+
+ async def gen():
+ yield {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {
+ "tool_calls": [
+ {"id": "call_1", "index": 0, "function": {"arguments": "{}"}}
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ }
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ assert _streamed_tool_calls(lines) == []
+ assert any(call[0] == "warning" and "named no tool" in call[1] for call in recorder.calls)
+
+
+def _tool_call_chunk(*tool_calls, finish_reason=None):
+ choice = {"index": 0, "delta": {"tool_calls": list(tool_calls)}}
+ if finish_reason:
+ choice["finish_reason"] = finish_reason
+ return {"choices": [choice]}
+
+
+async def test_stream_sse_keeps_an_arguments_fragment_that_arrives_before_the_call_is_named():
+ """Nothing is open yet, so the fragment must wait for its call, not vanish."""
+ from services import llm_gateway
+
+ async def gen():
+ yield _tool_call_chunk({"index": 0, "function": {"arguments": '{"query": '}})
+ yield _tool_call_chunk(
+ {
+ "id": "call_1",
+ "type": "function",
+ "index": 0,
+ "function": {"name": "search_documents", "arguments": '"x"}'},
+ }
+ )
+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ calls = _streamed_tool_calls(lines)
+ assert len(calls) == 1
+ assert calls[0]["id"] == "call_1"
+ assert calls[0]["function"]["name"] == "search_documents"
+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "x"}
+
+
+async def test_stream_sse_rejoins_both_a_leading_and_a_misfiled_trailing_fragment():
+ """The provisional call must not undo the next-index rejoin, and vice versa."""
+ from services import llm_gateway
+
+ async def gen():
+ yield _tool_call_chunk({"index": 0, "function": {"arguments": "{"}})
+ yield _tool_call_chunk(
+ {
+ "id": "call_1",
+ "type": "function",
+ "index": 0,
+ "function": {"name": "search_documents", "arguments": '"query": "x"'},
+ }
+ )
+ yield _tool_call_chunk({"index": 1, "function": {"arguments": "}"}})
+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ calls = _streamed_tool_calls(lines)
+ assert len(calls) == 1, "neither stray fragment may become a call of its own"
+ assert calls[0]["id"] == "call_1"
+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "x"}
+
+
+async def test_stream_sse_drops_an_unnamed_fragment_no_call_ever_claims(monkeypatch):
+ """A provisional call that never gets a name is still discarded at drain."""
+ from services import llm_gateway
+
+ recorder = _RecordingLogger()
+ monkeypatch.setattr(llm_gateway, "logger", recorder)
+
+ async def gen():
+ yield _tool_call_chunk({"index": 0, "function": {"arguments": "{}"}})
+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ assert _streamed_tool_calls(lines) == []
+ assert any(call[0] == "warning" and "named no tool" in call[1] for call in recorder.calls)
+ # Nothing reached the client, so it is told why rather than left hanging.
+ assert any('"code": "upstream_error"' in line for line in lines)
+ assert lines[-1] == "data: [DONE]\n\n"
+
+
+async def test_stream_sse_mints_an_id_for_a_named_call_that_has_none():
+ """A named call is real work; only its correlation handle is missing."""
+ from services import llm_gateway
+
+ async def gen():
+ yield {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {"name": "search_documents", "arguments": "{}"},
+ }
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ }
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ calls = _streamed_tool_calls(lines)
+ assert len(calls) == 1
+ assert isinstance(calls[0]["id"], str) and calls[0]["id"]
+
+
[email protected]("fragments", [[], [" "]], ids=["no-fragment", "whitespace-fragment"])
+async def test_stream_sse_defaults_a_zero_argument_call_to_an_empty_object(fragments):
+ """`""` is not a JSON object; clients that parse arguments must get `{}`."""
+ from services import llm_gateway
+
+ async def gen():
+ for chunk in _tool_call_deltas(fragments, name="list_sources"):
+ yield chunk
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]
+
+ calls = _streamed_tool_calls(lines)
+ assert len(calls) == 1
+ assert calls[0]["id"] == "call_1"
+ assert calls[0]["function"]["name"] == "list_sources"
+ assert json.loads(calls[0]["function"]["arguments"]) == {}
+
+
+def test_finalise_tool_call_leaves_real_arguments_alone():
+ from services.llm_gateway import _finalise_tool_call
+
+ call = {"id": "call_1", "function": {"name": "search", "arguments": _WELL_FORMED_ARGUMENTS}}
+ assert _finalise_tool_call(call) is True
+ assert call["function"]["arguments"] == _WELL_FORMED_ARGUMENTS
+
+
+def test_finalise_tool_call_does_not_default_arguments_on_a_nameless_call():
+ from services.llm_gateway import _finalise_tool_call
+
+ call = {"id": "call_1", "function": {"name": "", "arguments": ""}}
+ assert _finalise_tool_call(call) is False
+ assert call["function"]["arguments"] == ""
+
+
+# --------------------------------------------------------------------------
+# A poisoned conversation must not wedge every later turn
+# --------------------------------------------------------------------------
+
+
+def test_sanitise_messages_drops_a_tool_call_naming_no_tool_and_its_result():
+ """vLLM rejects the whole request over this one message, so it cannot go up."""
+ from services import llm_gateway
+
+ messages = [
+ {"role": "user", "content": "For the AMCOR form, who are the signatories?"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {"name": "search_documents", "arguments": '{"query": "AMCOR"}'},
+ },
+ # What langchain-openai echoes back for an unparseable call.
+ {"type": "function", "id": None, "function": {"name": "", "arguments": "}"}},
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "…"},
+ {"role": "tool", "tool_call_id": None, "content": "…"},
+ ]
+
+ cleaned, repairs = llm_gateway._sanitise_messages(messages)
+
+ assert repairs == 2
+ assert [m["role"] for m in cleaned] == ["user", "assistant", "tool"]
+ assert [c["id"] for c in cleaned[1]["tool_calls"]] == ["call_1"]
+
+
+def test_sanitise_messages_encodes_object_arguments_as_the_json_string_openai_requires():
+ from services import llm_gateway
+
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {"name": "search", "arguments": {"query": "AMCOR"}},
+ }
+ ],
+ }
+ ]
+ )
+
+ assert repairs == 1
+ assert cleaned[0]["tool_calls"][0]["function"]["arguments"] == '{"query": "AMCOR"}'
+
+
+def test_sanitise_messages_removes_a_tool_calls_key_left_empty():
+ """`tool_calls: null` is itself a shape providers iterate without a None check."""
+ from services import llm_gateway
+
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [{"role": "assistant", "content": "hello", "tool_calls": None}]
+ )
+
+ assert "tool_calls" not in cleaned[0]
+ assert cleaned[0]["content"] == "hello"
+ assert repairs == 0
+
+
[email protected]("function", ["search", 42, ["name", "search"]])
+def test_sanitise_messages_drops_a_call_whose_function_is_not_an_object(function):
+ """Malformed history is the sanitiser's job to absorb, not to crash on."""
+ from services import llm_gateway
+
+ good = {"type": "function", "id": "call_1", "function": {"name": "search", "arguments": "{}"}}
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [good, {"type": "function", "id": "call_2", "function": function}],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "kept"},
+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},
+ ]
+ )
+
+ assert repairs == 2
+ assert [call["id"] for call in cleaned[0]["tool_calls"]] == ["call_1"]
+ assert [m["content"] for m in cleaned[1:]] == ["kept"]
+
+
+async def test_chat_completions_absorbs_a_non_object_function_instead_of_raising(monkeypatch):
+ """It runs before the call's error handling, so a raise here was a bare 500."""
+ captured = {}
+
+ async def fake_acompletion(**kwargs):
+ captured.update(kwargs)
+ return {"id": "1", "choices": [{"message": {"role": "assistant", "content": "hi"}}]}
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ await chat_completions(
+ {
+ "model": "gpt-4o-mini",
+ "messages": [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [{"type": "function", "id": "call_1", "function": "oops"}],
+ },
+ ],
+ },
+ config=_config(),
+ )
+
+ assert captured["messages"] == [{"role": "user", "content": "hi"}]
+
+
+def test_sanitise_messages_drops_the_result_of_a_dropped_call_that_had_an_id():
+ """The result carries a real id, but the call it answers is gone."""
+ from services import llm_gateway
+
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [
+ {"role": "user", "content": "q"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {"name": "search", "arguments": "{}"},
+ },
+ {
+ "type": "function",
+ "id": "call_2",
+ "function": {"name": "", "arguments": "{}"},
+ },
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "kept"},
+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},
+ ]
+ )
+
+ assert repairs == 2
+ assert [m.get("content") for m in cleaned if m["role"] == "tool"] == ["kept"]
+
+
+def test_sanitise_messages_drops_an_assistant_message_that_was_only_bad_calls():
+ """`content` is optional only beside `tool_calls`; with neither, nothing is left."""
+ from services import llm_gateway
+
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [
+ {"role": "user", "content": "q"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_2",
+ "function": {"name": "", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},
+ {"role": "user", "content": "again"},
+ ]
+ )
+
+ assert repairs == 2
+ assert cleaned == [{"role": "user", "content": "q"}, {"role": "user", "content": "again"}]
+
+
+def test_sanitise_messages_keeps_an_assistant_message_that_still_has_content():
+ from services import llm_gateway
+
+ cleaned, _ = llm_gateway._sanitise_messages(
+ [
+ {
+ "role": "assistant",
+ "content": "Let me look.",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_2",
+ "function": {"name": "", "arguments": "{}"},
+ }
+ ],
+ }
+ ]
+ )
+
+ assert cleaned == [{"role": "assistant", "content": "Let me look."}]
+
+
+def test_sanitise_messages_drops_a_result_not_answering_the_preceding_assistant_message():
+ """A result only answers the calls of the assistant message just before it."""
+ from services import llm_gateway
+
+ call = {"type": "function", "id": "call_1", "function": {"name": "search", "arguments": "{}"}}
+ cleaned, repairs = llm_gateway._sanitise_messages(
+ [
+ {"role": "assistant", "content": "", "tool_calls": [call]},
+ {"role": "tool", "tool_call_id": "call_1", "content": "answer"},
+ {"role": "user", "content": "next"},
+ {"role": "tool", "tool_call_id": "call_1", "content": "replayed out of place"},
+ ]
+ )
+
+ assert repairs == 1
+ assert [m.get("content") for m in cleaned] == ["", "answer", "next"]
+
+
+def test_sanitise_messages_keeps_every_result_of_parallel_calls():
+ from services import llm_gateway
+
+ messages = [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"type": "function", "id": "a", "function": {"name": "search", "arguments": "{}"}},
+ {"type": "function", "id": "b", "function": {"name": "search", "arguments": "{}"}},
+ ],
+ },
+ {"role": "tool", "tool_call_id": "b", "content": "second"},
+ {"role": "tool", "tool_call_id": "a", "content": "first"},
+ ]
+
+ cleaned, repairs = llm_gateway._sanitise_messages(messages)
+
+ assert repairs == 0
+ assert cleaned == messages
+
+
+def test_sanitise_messages_leaves_a_healthy_conversation_untouched():
+ from services import llm_gateway
+
+ messages = [
+ {"role": "system", "content": "be helpful"},
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {"name": "search", "arguments": '{"query": "x"}'},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "result"},
+ ]
+
+ cleaned, repairs = llm_gateway._sanitise_messages(messages)
+
+ assert repairs == 0
+ assert cleaned == messages
+
+
+async def test_chat_completions_sends_the_sanitised_conversation_upstream(monkeypatch):
+ """The repair has to reach the wire, not just exist as a helper."""
+ captured = {}
+
+ async def fake_acompletion(**kwargs):
+ captured.update(kwargs)
+ return {"id": "1", "choices": [{"message": {"role": "assistant", "content": "hi"}}]}
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ await chat_completions(
+ {
+ "model": "gpt-4o-mini",
+ "messages": [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"type": "function", "id": None, "function": {"name": "", "arguments": "}"}}
+ ],
+ },
+ ],
+ },
+ config=_config(),
+ )
+
+ # The assistant message held nothing but the unusable call, so it goes too.
+ assert captured["messages"] == [{"role": "user", "content": "hi"}]
+
+
# --------------------------------------------------------------------------
# Surfacing the real upstream failure
# --------------------------------------------------------------------------
@@ -1095,6 +1645,60 @@ async def test_chat_completions_masks_an_upstream_4xx_as_a_gateway_failure(monke
assert exc.value.status_code == 502
+class _FakeMidStreamFallbackError(Exception):
+ llm_provider = "hosted_vllm"
+
+
+_PROXY_CUTOFF = (
+ "litellm.MidStreamFallbackError: litellm.APIConnectionError: APIConnectionError: "
+ "Hosted_vllmException - Response payload is not completed: <TransferEncodingError: 400, "
+ "message='Not enough data to satisfy transfer length header.'>"
+)
+
+
+def test_a_truncated_response_names_the_proxy_not_the_wrapper_chain():
+ """aiohttp's own "400" must not read as the model rejecting the request."""
+ from services.llm_gateway import _upstream_client_message
+
+ message = _upstream_client_message(
+ f"MidStreamFallbackError: {_PROXY_CUTOFF}",
+ "rhoai",
+ "hosted_vllm/qwen2.5-0.5b-instruct",
+ _FakeMidStreamFallbackError(_PROXY_CUTOFF),
+ )
+
+ assert "closed before the response finished" in message
+ assert "timed out" in message
+ assert "rhoai/qwen2.5-0.5b-instruct" in message
+ for noise in ("MidStreamFallbackError", "TransferEncodingError", "400"):
+ assert noise not in message
+
+
+async def test_a_mid_stream_cutoff_is_reported_and_still_latches_the_banner():
+ """Unlike a context overflow, this fails every long request on that endpoint."""
+ from services import llm_gateway, provider_error_log
+
+ provider_error_log.clear()
+
+ async def gen():
+ raise _FakeMidStreamFallbackError(_PROXY_CUTOFF)
+ yield # pragma: no cover - makes this an async generator
+
+ try:
+ lines = [
+ line
+ async for line in llm_gateway._stream_sse(
+ gen(), "rhoai", "hosted_vllm/qwen2.5-0.5b-instruct"
+ )
+ ]
+ error = json.loads(lines[-2][len("data: ") :])["error"]
+ assert "closed before the response finished" in error["message"]
+ assert provider_error_log.latest_failure("rhoai", "chat") == error["message"]
+ finally:
+ provider_error_log.clear()
+
+
def test_sanitise_upstream_detail_drops_interpreter_state():
from services.llm_gateway import _sanitise_upstream_detail
@@ -1273,6 +1877,99 @@ class TestRealFailuresReachTheBanner:
assert provider_error_log.latest_failure("openai", "chat") is None
+class TestRequestScopedFailuresStayOutOfTheBanner:
+ """A prompt that overflows the context window is this request's problem.
+
+ The provider is serving; the next, shorter prompt succeeds. Recording it
+ latched the health banner against a healthy provider, and on one whose
+ probe cannot clear it (every provider enhancement), it stayed latched.
+ """
+
+ _OVERFLOW = (
+ "This model's maximum context length is 32768 tokens. However, you requested 0 "
+ "output tokens and your prompt contains at least 32769 input tokens"
+ )
+
+ @pytest.fixture(autouse=True)
+ def _clean(self, monkeypatch):
+ from services import llm_gateway, provider_error_log
+
+ provider_error_log.clear()
+ monkeypatch.setattr(
+ llm_gateway,
+ "resolve_call",
+ lambda *a, **k: ("hosted_vllm/qwen2.5-0.5b-instruct", "rhoai", {}),
+ )
+ yield
+ provider_error_log.clear()
+
+ def _raise(self, monkeypatch, exc: BaseException):
+ async def _boom(**_kwargs):
+ raise exc
+
+ monkeypatch.setattr("litellm.acompletion", _boom)
+
+ @pytest.mark.asyncio
+ async def test_a_context_window_overflow_is_not_recorded(self, monkeypatch):
+ from services import llm_gateway, provider_error_log
+
+ class ContextWindowExceededError(Exception):
+ llm_provider = "hosted_vllm"
+
+ self._raise(monkeypatch, ContextWindowExceededError(self._OVERFLOW))
+
+ with pytest.raises(llm_gateway.LlmGatewayError) as raised:
+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})
+
+ assert "maximum context length" in str(raised.value)
+ assert provider_error_log.latest_failure("rhoai", "chat") is None
+
+ @pytest.mark.asyncio
+ async def test_an_overflow_does_not_erase_a_real_failure(self, monkeypatch):
+ from services import llm_gateway, provider_error_log
+
+ provider_error_log.record_failure("rhoai", "chat", "earlier real failure")
+ self._raise(monkeypatch, RuntimeError(self._OVERFLOW))
+
+ with pytest.raises(llm_gateway.LlmGatewayError):
+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})
+
+ assert provider_error_log.latest_failure("rhoai", "chat") == "earlier real failure"
+
+ @pytest.mark.asyncio
+ async def test_other_upstream_failures_are_still_recorded(self, monkeypatch):
+ from services import llm_gateway, provider_error_log
+
+ self._raise(
+ monkeypatch,
+ RuntimeError('{"error": {"message": "The model `qwen` does not exist."}}'),
+ )
+
+ with pytest.raises(llm_gateway.LlmGatewayError):
+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})
+
+ assert "does not exist" in provider_error_log.latest_failure("rhoai", "chat")
+
+ @pytest.mark.asyncio
+ async def test_a_mid_stream_overflow_is_reported_but_not_recorded(self):
+ from services import llm_gateway, provider_error_log
+
+ overflow = self._OVERFLOW
+
+ class BadRequestError(Exception):
+ llm_provider = "hosted_vllm"
+
+ async def gen():
+ raise BadRequestError(overflow)
+ yield # pragma: no cover - makes this an async generator
+
+ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "qwen")]
+
+ error = json.loads(lines[-2][len("data: ") :])["error"]
+ assert "maximum context length" in error["message"]
+ assert provider_error_log.latest_failure("rhoai", "chat") is None
+
+
class TestToolsBesideReasoningEffort:
"""gpt-5.x refuses function tools next to its own default reasoning effort.
diff --git a/tests/unit/services/test_llm_gateway_embedding_concurrency.py b/tests/unit/services/test_llm_gateway_embedding_concurrency.py
new file mode 100644
index 000000000..375b43d3e
--- /dev/null
+++ b/tests/unit/services/test_llm_gateway_embedding_concurrency.py
@@ -0,0 +1,539 @@
+"""Per-provider embedding bulkhead and the opaque upstream 502/504 message.
+
+A folder upload fans out to several Langflow ingest runs, each embedding one
+chunk per request on eight threads. Against a slow (CPU-served) RHOAI model
+the calls queued at the endpoint until its kube-rbac-proxy timed them out with
+a body-less 502. The gateway now bounds concurrent embedding calls per
+provider (limit supplied by the provider enhancement) and explains such a 502.
+
+One slot of that limit is reserved for query embeddings (`interactive=True`:
+search and chat retrieval) so a bulk ingest cannot queue them; the two lanes
+split the limit rather than add to it.
+"""
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from config.config_manager import (
+ AnthropicConfig,
+ GenericProviderConfig,
+ OllamaConfig,
+ OpenAIConfig,
+ ProvidersConfig,
+ WatsonXConfig,
+)
+from services import llm_gateway
+from services.llm_gateway import LlmGatewayError, embeddings
+
+RHOAI_MODEL = "rhoai:granite-embedding"
+
+
+def _rhoai_config(limit: str | None = None) -> SimpleNamespace:
+ credentials = {
+ "api_base": "https://chat.example.com/v1",
+ "embedding_api_base": "https://embed.example.com/v1",
+ "api_key": "token",
+ }
+ if limit is not None:
+ credentials["embedding_max_concurrency"] = limit
+ providers = ProvidersConfig(
+ openai=OpenAIConfig(api_key="sk-openai", configured=True),
+ anthropic=AnthropicConfig(),
+ watsonx=WatsonXConfig(),
+ ollama=OllamaConfig(),
+ custom={"rhoai": GenericProviderConfig(credentials=credentials, configured=True)},
+ )
+ return SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_model="gpt-4o-mini", llm_provider="openai"),
+ knowledge=SimpleNamespace(embedding_model="granite-embedding", embedding_provider="rhoai"),
+ )
+
+
+class _InFlightTracker:
+ """A fake `litellm.aembedding` that records peak concurrency."""
+
+ def __init__(self, delay: float = 0.01):
+ self.delay = delay
+ self.in_flight = 0
+ self.peak = 0
+ self.calls: list[dict] = []
+
+ async def __call__(self, **kwargs):
+ self.calls.append(kwargs)
+ self.in_flight += 1
+ self.peak = max(self.peak, self.in_flight)
+ try:
+ await asyncio.sleep(self.delay)
+ finally:
+ self.in_flight -= 1
+ return {
+ "object": "list",
+ "data": [{"object": "embedding", "embedding": [0.1], "index": 0}],
+ "usage": {"prompt_tokens": 1, "total_tokens": 1},
+ }
+
+
[email protected](autouse=True)
+def _reset_limiters():
+ llm_gateway._embedding_limiters.clear()
+ yield
+ llm_gateway._embedding_limiters.clear()
+
+
+async def _embed_concurrently(cfg, model: str, count: int, *, interactive=False) -> None:
+ await asyncio.gather(
+ *(
+ embeddings({"model": model, "input": f"chunk {i}"}, config=cfg, interactive=interactive)
+ for i in range(count)
+ )
+ )
+
+
+RHOAI = "rhoai"
+
+
+async def test_rhoai_embeddings_are_bounded_by_the_configured_limit(monkeypatch):
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await _embed_concurrently(_rhoai_config("3"), RHOAI_MODEL, 10)
+
+ assert len(tracker.calls) == 10
+ # One of the three slots is held back for queries.
+ assert tracker.peak == 2
+
+
+async def test_rhoai_embeddings_use_the_default_limit_when_unset(monkeypatch):
+ from enhancements.providers.redhat.openshift_ai import DEFAULT_EMBEDDING_MAX_CONCURRENCY
+
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await _embed_concurrently(_rhoai_config(), RHOAI_MODEL, 2 * DEFAULT_EMBEDDING_MAX_CONCURRENCY)
+
+ assert tracker.peak == DEFAULT_EMBEDDING_MAX_CONCURRENCY - 1
+
+
+async def test_limit_zero_disables_the_bulkhead(monkeypatch):
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 10)
+
+ assert tracker.peak == 10
+ assert RHOAI not in llm_gateway._embedding_limiters
+
+
+async def test_the_limit_never_reaches_litellm(monkeypatch):
+ tracker = _InFlightTracker(delay=0)
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config("3"))
+
+ assert "embedding_max_concurrency" not in tracker.calls[0]
+ assert tracker.calls[0]["api_base"] == "https://embed.example.com/v1"
+
+
+async def test_providers_without_an_enhancement_are_unbounded(monkeypatch):
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await _embed_concurrently(_rhoai_config("1"), "openai:text-embedding-3-small", 6)
+
+ assert tracker.peak == 6
+ assert "openai" not in llm_gateway._embedding_limiters
+
+
+async def test_a_changed_limit_resizes_the_limiter_in_place():
+ first = llm_gateway._embedding_limiter("rhoai", _rhoai_config("2"))
+ same = llm_gateway._embedding_limiter("rhoai", _rhoai_config("2"))
+ changed = llm_gateway._embedding_limiter("rhoai", _rhoai_config("5"))
+
+ # One limiter for the provider's lifetime, so its in-flight count survives.
+ assert first is same is changed
+ assert llm_gateway._embedding_limiters[RHOAI].limit == 5
+
+
+async def test_a_cancelled_waiter_releases_its_slot(monkeypatch):
+ gate = asyncio.Event()
+
+ async def blocking_aembedding(**kwargs):
+ await gate.wait()
+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}
+
+ monkeypatch.setattr("litellm.aembedding", blocking_aembedding)
+ cfg = _rhoai_config("1")
+
+ holder = asyncio.create_task(embeddings({"model": RHOAI_MODEL, "input": "a"}, config=cfg))
+ waiter = asyncio.create_task(embeddings({"model": RHOAI_MODEL, "input": "b"}, config=cfg))
+ await asyncio.sleep(0.01)
+ waiter.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+ gate.set()
+ await holder
+
+ # Both slots are free again: a new call proceeds immediately.
+ result = await asyncio.wait_for(
+ embeddings({"model": RHOAI_MODEL, "input": "c"}, config=cfg), timeout=1
+ )
+ assert result["data"]
+
+
+async def test_a_failing_call_releases_its_slot(monkeypatch):
+ async def boom(**kwargs):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr("litellm.aembedding", boom)
+ cfg = _rhoai_config("1")
+ for _ in range(3):
+ with pytest.raises(LlmGatewayError):
+ await asyncio.wait_for(
+ embeddings({"model": RHOAI_MODEL, "input": "x"}, config=cfg), timeout=1
+ )
+
+
+# --------------------------------------------------------------------------
+# Interactive lane
+# --------------------------------------------------------------------------
+
+
+class _Gate:
+ """A fake `litellm.aembedding` whose calls block until released."""
+
+ def __init__(self):
+ self.release = asyncio.Event()
+ self.in_flight = 0
+ self.peak = 0
+ self.started: list[str] = []
+
+ async def __call__(self, **kwargs):
+ self.started.append(kwargs["input"][0] if isinstance(kwargs["input"], list) else "")
+ self.in_flight += 1
+ self.peak = max(self.peak, self.in_flight)
+ try:
+ await self.release.wait()
+ finally:
+ self.in_flight -= 1
+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}
+
+
+def _embed(cfg, text: str, *, interactive: bool = False) -> asyncio.Task:
+ return asyncio.create_task(
+ embeddings({"model": RHOAI_MODEL, "input": text}, config=cfg, interactive=interactive)
+ )
+
+
+async def test_a_query_is_not_queued_behind_bulk_ingestion(monkeypatch):
+ gate = _Gate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+ cfg = _rhoai_config("4")
+
+ bulk = [_embed(cfg, f"chunk {i}") for i in range(20)]
+ await asyncio.sleep(0.01)
+ assert gate.in_flight == 3 # the bulk lane is full, 17 chunks waiting
+
+ query = _embed(cfg, "query", interactive=True)
+ await asyncio.sleep(0.01)
+ assert "query" in gate.started # started ahead of every waiting chunk
+ assert gate.in_flight == 4
+
+ gate.release.set()
+ await asyncio.gather(query, *bulk)
+ assert gate.peak == 4 # never more than the operator's limit in total
+
+
+async def test_queries_are_bounded_by_their_own_lane(monkeypatch):
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await _embed_concurrently(_rhoai_config("4"), RHOAI_MODEL, 6, interactive=True)
+
+ assert len(tracker.calls) == 6
+ assert tracker.peak == 1
+ assert llm_gateway._embedding_limiters[RHOAI].limit == 4
+
+
+async def test_a_limit_of_one_is_shared_by_queries_and_ingestion(monkeypatch):
+ gate = _Gate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+ cfg = _rhoai_config("1")
+
+ chunk = _embed(cfg, "chunk")
+ await asyncio.sleep(0.01)
+ query = _embed(cfg, "query", interactive=True)
+ await asyncio.sleep(0.01)
+
+ # Splitting a single slot would exceed the limit, so the query waits.
+ assert gate.started == ["chunk"]
+ gate.release.set()
+ await asyncio.gather(chunk, query)
+ assert gate.peak == 1
+
+
+async def test_no_limit_throttles_neither_lane(monkeypatch):
+ tracker = _InFlightTracker()
+ monkeypatch.setattr("litellm.aembedding", tracker)
+
+ await asyncio.gather(
+ _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 5),
+ _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 5, interactive=True),
+ )
+
+ assert tracker.peak == 10
+ assert not llm_gateway._embedding_limiters
+
+
+ ("limit", "interactive", "expected"),
+ [
+ (4, False, ("bulk", 3)),
+ (4, True, ("interactive", 1)),
+ (2, False, ("bulk", 1)),
+ (2, True, ("interactive", 1)),
+ (1, False, ("bulk", 1)),
+ (1, True, ("bulk", 1)),
+ ],
+)
+def test_lane_limits_split_the_configured_total(limit, interactive, expected):
+ assert llm_gateway._lane_limit(limit, interactive) == expected
+
+
+# --------------------------------------------------------------------------
+# Limit changes while calls are in flight
+# --------------------------------------------------------------------------
+
+
+class _PerCallGate:
+ """A fake `litellm.aembedding` whose calls are released one at a time."""
+
+ def __init__(self):
+ self.events: dict[str, asyncio.Event] = {}
+ self.started: list[str] = []
+ self.in_flight = 0
+
+ def release(self, text: str) -> None:
+ self.events.setdefault(text, asyncio.Event()).set()
+
+ async def __call__(self, **kwargs):
+ text = kwargs["input"][0]
+ self.started.append(text)
+ self.in_flight += 1
+ try:
+ await self.events.setdefault(text, asyncio.Event()).wait()
+ finally:
+ self.in_flight -= 1
+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}
+
+
+async def _settle() -> None:
+ await asyncio.sleep(0.01)
+
+
+async def test_a_lowered_limit_counts_calls_still_in_flight(monkeypatch):
+ gate = _PerCallGate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+
+ old = [_embed(_rhoai_config("3"), "old 1"), _embed(_rhoai_config("3"), "old 2")]
+ await _settle()
+ assert gate.in_flight == 2
+
+ # Lowered to 2 (one bulk slot): the two old calls already exceed it.
+ new = _embed(_rhoai_config("2"), "new")
+ await _settle()
+ assert "new" not in gate.started
+
+ gate.release("old 1")
+ await _settle()
+ assert "new" not in gate.started # one old call still holds the bulk share
+
+ gate.release("old 2")
+ await _settle()
+ assert "new" in gate.started
+ gate.release("new")
+ await asyncio.gather(*old, new)
+
+
+async def test_a_raised_limit_admits_queued_calls_at_once(monkeypatch):
+ gate = _PerCallGate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+
+ calls = [_embed(_rhoai_config("2"), f"chunk {i}") for i in range(3)]
+ await _settle()
+ assert gate.in_flight == 1 # bulk share of a limit of 2
+
+ llm_gateway._embedding_limiter("rhoai", _rhoai_config("4"))
+ await _settle()
+ assert gate.in_flight == 3 # no call had to finish first
+
+ for i in range(3):
+ gate.release(f"chunk {i}")
+ await asyncio.gather(*calls)
+
+
+async def test_lowering_to_one_counts_a_query_from_the_interactive_lane(monkeypatch):
+ gate = _PerCallGate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+
+ query = _embed(_rhoai_config("3"), "query", interactive=True)
+ await _settle()
+
+ chunk = _embed(_rhoai_config("1"), "chunk")
+ await _settle()
+ assert gate.started == ["query"]
+
+ gate.release("query")
+ await _settle()
+ assert gate.started == ["query", "chunk"]
+ gate.release("chunk")
+ await asyncio.gather(query, chunk)
+
+
+async def test_turning_the_limit_off_and_on_keeps_counting(monkeypatch):
+ gate = _PerCallGate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+
+ first = _embed(_rhoai_config("2"), "first")
+ await _settle()
+ unbounded = [_embed(_rhoai_config("0"), f"free {i}") for i in range(2)]
+ await _settle()
+ assert gate.in_flight == 3 # "0" switches the limit off
+
+ late = _embed(_rhoai_config("2"), "late")
+ await _settle()
+ assert "late" not in gate.started # the unbounded calls still count
+
+ gate.release("first")
+ gate.release("free 0")
+ await _settle()
+ assert "late" not in gate.started # total is 1 but the bulk share (1) is used
+ gate.release("free 1")
+ await _settle()
+ assert "late" in gate.started
+ gate.release("late")
+ await asyncio.gather(first, *unbounded, late)
+
+
+async def test_a_limit_of_one_hands_the_slot_to_a_waiting_query_first(monkeypatch):
+ gate = _PerCallGate()
+ monkeypatch.setattr("litellm.aembedding", gate)
+ cfg = _rhoai_config("1")
+
+ calls = [_embed(cfg, "chunk 0")]
+ await _settle()
+ calls += [_embed(cfg, "chunk 1"), _embed(cfg, "chunk 2")]
+ await _settle()
+ calls.append(_embed(cfg, "query", interactive=True))
+ await _settle()
+
+ gate.release("chunk 0")
+ await _settle()
+ assert gate.started == ["chunk 0", "query"]
+
+ for text in ("query", "chunk 1", "chunk 2"):
+ gate.release(text)
+ await asyncio.gather(*calls)
+ assert gate.started == ["chunk 0", "query", "chunk 1", "chunk 2"]
+
+
+async def test_a_slot_granted_to_a_cancelled_waiter_is_handed_on():
+ limiter = llm_gateway._EmbeddingLimiter("rhoai", 1, asyncio.get_running_loop())
+ await limiter.acquire(llm_gateway._BULK_LANE)
+
+ waiter = asyncio.create_task(limiter.acquire(llm_gateway._BULK_LANE))
+ await _settle()
+ limiter.release(llm_gateway._BULK_LANE) # grants the waiter's future...
+ waiter.cancel() # ...but it is cancelled before it resumes
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+
+ assert limiter.in_flight == 0
+ await asyncio.wait_for(limiter.acquire(llm_gateway._BULK_LANE), timeout=1)
+
+
+async def test_an_extra_release_does_not_free_a_phantom_slot():
+ limiter = llm_gateway._EmbeddingLimiter("rhoai", 1, asyncio.get_running_loop())
+ limiter.release(llm_gateway._BULK_LANE)
+
+ await limiter.acquire(llm_gateway._BULK_LANE)
+ waiter = asyncio.create_task(limiter.acquire(llm_gateway._BULK_LANE))
+ await _settle()
+ assert not waiter.done() # the stray release did not leave a spare slot
+ waiter.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+
+
+# --------------------------------------------------------------------------
+# Opaque upstream 502/504
+# --------------------------------------------------------------------------
+
+
+class _FakeLiteLLMError(Exception):
+ """Stands in for a LiteLLM exception, which is always provider-attributable."""
+
+ def __init__(self, message: str, status_code: int):
+ super().__init__(message)
+ self.llm_provider = "hosted_vllm"
+ self.status_code = status_code
+
+
[email protected]("status", [502, 504])
+async def test_an_empty_upstream_gateway_failure_gets_an_actionable_message(monkeypatch, status):
+ async def boom(**kwargs):
+ raise _FakeLiteLLMError(
+ "litellm.BadGatewayError: BadGatewayError: Hosted_vllmException - ", status
+ )
+
+ monkeypatch.setattr("litellm.aembedding", boom)
+ with pytest.raises(LlmGatewayError) as exc:
+ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config())
+
+ assert exc.value.message.startswith(llm_gateway._UPSTREAM_GATEWAY_MESSAGE)
+ assert "(rhoai/granite-embedding)" in exc.value.message
+ assert "Hosted_vllmException" not in exc.value.message
+ assert exc.value.status_code == status
+
+
+async def test_a_gateway_failure_with_provider_text_keeps_that_text(monkeypatch):
+ async def boom(**kwargs):
+ raise _FakeLiteLLMError(
+ "litellm.BadGatewayError: BadGatewayError: upstream connect error", 502
+ )
+
+ monkeypatch.setattr("litellm.aembedding", boom)
+ with pytest.raises(LlmGatewayError) as exc:
+ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config())
+
+ assert "upstream connect error" in exc.value.message
+ assert llm_gateway._UPSTREAM_GATEWAY_MESSAGE not in exc.value.message
+
+
+def test_an_empty_non_gateway_failure_is_not_relabelled():
+ exc = _FakeLiteLLMError("litellm.InternalServerError: Hosted_vllmException - ", 500)
+ message = llm_gateway._upstream_client_message(str(exc), "rhoai", "m", exc)
+ assert llm_gateway._UPSTREAM_GATEWAY_MESSAGE not in message
diff --git a/tests/unit/services/test_llm_gateway_stale_connection_retry.py b/tests/unit/services/test_llm_gateway_stale_connection_retry.py
new file mode 100644
index 000000000..7aca2b643
--- /dev/null
+++ b/tests/unit/services/test_llm_gateway_stale_connection_retry.py
@@ -0,0 +1,216 @@
+"""One retry when a dead pooled connection eats a LiteLLM call.
+
+LiteLLM keeps idle connections for 120s; vLLM (uvicorn) closes them after 5s.
+A call that reuses one while the server's close is still in flight fails with
+"Server disconnected" before the server reads it, and surfaced as a provider
+failure in chat, search and the health banner. Reproduced against RHOAI through
+`oc port-forward`, where every reuse 4.95-6.0s after the last call failed.
+"""
+
+from types import SimpleNamespace
+
+import aiohttp
+import httpx
+import pytest
+
+from api.provider_validation import _test_litellm_provider, is_provider_stale_connection_error
+from config.config_manager import (
+ AnthropicConfig,
+ GenericProviderConfig,
+ OllamaConfig,
+ OpenAIConfig,
+ ProvidersConfig,
+ WatsonXConfig,
+)
+from services import llm_gateway, provider_error_log
+from services.llm_gateway import LlmGatewayError, chat_completions, embeddings
+
+
+def _stale_connection_error() -> Exception:
+ """The chain LiteLLM 1.102 raises, as captured from a live RHOAI endpoint."""
+ try:
+ try:
+ raise aiohttp.ServerDisconnectedError()
+ except aiohttp.ServerDisconnectedError as disconnected:
+ raise httpx.ReadError("Server disconnected") from disconnected
+ except httpx.ReadError as read_error:
+ error = Exception(
+ "litellm.InternalServerError: InternalServerError: "
+ "Hosted_vllmException - Server disconnected"
+ )
+ error.__cause__ = read_error
+ return error
+
+
+def _rhoai_config() -> SimpleNamespace:
+ providers = ProvidersConfig(
+ openai=OpenAIConfig(api_key="sk-openai", configured=True),
+ anthropic=AnthropicConfig(),
+ watsonx=WatsonXConfig(),
+ ollama=OllamaConfig(),
+ custom={
+ "rhoai": GenericProviderConfig(
+ credentials={
+ "api_base": "https://chat.example.com/v1",
+ "embedding_api_base": "https://embed.example.com/v1",
+ "api_key": "token",
+ },
+ configured=True,
+ )
+ },
+ )
+ return SimpleNamespace(
+ providers=providers,
+ agent=SimpleNamespace(llm_model="granite-chat", llm_provider="rhoai"),
+ knowledge=SimpleNamespace(embedding_model="granite-embedding", embedding_provider="rhoai"),
+ )
+
+
+class _FailsThenAnswers:
+ """A fake LiteLLM call that raises the given errors in turn, then answers."""
+
+ def __init__(self, answer, *errors: Exception):
+ self.answer = answer
+ self.errors = list(errors)
+ self.calls = 0
+
+ async def __call__(self, **_kwargs):
+ self.calls += 1
+ if self.errors:
+ raise self.errors.pop(0)
+ return self.answer
+
+
+EMBEDDING = {
+ "object": "list",
+ "data": [{"object": "embedding", "embedding": [0.1], "index": 0}],
+ "usage": {"prompt_tokens": 1, "total_tokens": 1},
+}
+COMPLETION = {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "choices": [{"message": {"role": "assistant", "content": "OK"}}],
+}
+
+
[email protected](autouse=True)
+def _clean_state():
+ llm_gateway._embedding_limiters.clear()
+ provider_error_log.clear()
+ yield
+ llm_gateway._embedding_limiters.clear()
+ provider_error_log.clear()
+
+
+# --- classification ---------------------------------------------------------
+
+
+def test_litellm_wrapped_server_disconnect_is_a_stale_connection():
+ assert is_provider_stale_connection_error(_stale_connection_error())
+
+
+def test_httpx_disconnect_before_any_response_is_a_stale_connection():
+ """LiteLLM's httpx transport, used when a provider hands it its own client."""
+ error = httpx.RemoteProtocolError("Server disconnected without sending a response.")
+
+ assert is_provider_stale_connection_error(error)
+
+
+ "error",
+ [
+ # Sent after the response began: the server may have done the work.
+ aiohttp.ClientPayloadError("Response payload is not completed"),
+ httpx.RemoteProtocolError("peer closed connection without sending complete message body"),
+ httpx.ReadError("Server disconnected"),
+ # Text alone is not evidence; only the transport's own type is.
+ Exception("Server disconnected"),
+ Exception("401 Unauthorized"),
+ None,
+ ],
+ ids=["payload-cut", "body-cut", "bare-read-error", "text-only", "auth", "none"],
+)
+def test_other_failures_are_not_stale_connections(error):
+ assert not is_provider_stale_connection_error(error)
+
+
+# --- gateway ----------------------------------------------------------------
+
+
+async def test_an_embedding_on_a_dead_connection_is_sent_again(monkeypatch):
+ fake = _FailsThenAnswers(EMBEDDING, _stale_connection_error())
+ monkeypatch.setattr("litellm.aembedding", fake)
+
+ result = await embeddings(
+ {"model": "rhoai:granite-embedding", "input": "q"}, config=_rhoai_config()
+ )
+
+ assert fake.calls == 2
+ assert result["data"][0]["embedding"] == [0.1]
+ assert provider_error_log.latest_failure("rhoai", "embedding") is None
+
+
+async def test_a_completion_on_a_dead_connection_is_sent_again(monkeypatch):
+ fake = _FailsThenAnswers(COMPLETION, _stale_connection_error())
+ monkeypatch.setattr("litellm.acompletion", fake)
+
+ result = await chat_completions(
+ {"model": "rhoai:granite-chat", "messages": [{"role": "user", "content": "hi"}]},
+ config=_rhoai_config(),
+ )
+
+ assert fake.calls == 2
+ assert result["choices"][0]["message"]["content"] == "OK"
+ assert provider_error_log.latest_failure("rhoai", "chat") is None
+
+
+async def test_only_one_retry_is_made(monkeypatch):
+ """A server that keeps dropping connections is a real failure, reported as one."""
+ fake = _FailsThenAnswers(EMBEDDING, _stale_connection_error(), _stale_connection_error())
+ monkeypatch.setattr("litellm.aembedding", fake)
+
+ with pytest.raises(LlmGatewayError):
+ await embeddings({"model": "rhoai:granite-embedding", "input": "q"}, config=_rhoai_config())
+
+ assert fake.calls == 2
+ assert provider_error_log.latest_failure("rhoai", "embedding") is not None
+
+
+async def test_other_failures_are_not_retried(monkeypatch):
+ """The server may have acted on these; sending them again could double the work."""
+ fake = _FailsThenAnswers(COMPLETION, httpx.ReadTimeout("timed out"))
+ monkeypatch.setattr("litellm.acompletion", fake)
+
+ with pytest.raises(LlmGatewayError):
+ await chat_completions(
+ {"model": "rhoai:granite-chat", "messages": [{"role": "user", "content": "hi"}]},
+ config=_rhoai_config(),
+ )
+
+ assert fake.calls == 1
+
+
+# --- health / pre-save probe -------------------------------------------------
+
+
[email protected]("kind", ["embedding", "chat"])
+async def test_the_provider_probe_retries_a_dead_connection(monkeypatch, kind):
+ """Otherwise a dead socket raises the health banner or blocks a settings save."""
+ answer = EMBEDDING if kind == "embedding" else COMPLETION
+ fake = _FailsThenAnswers(answer, _stale_connection_error())
+ monkeypatch.setattr(f"litellm.a{'embedding' if kind == 'embedding' else 'completion'}", fake)
+
+ await _test_litellm_provider(
+ provider="rhoai",
+ credentials={"api_base": "https://example.com/v1", "api_key": "token"},
+ runtime_kwargs={},
+ embedding_model="granite-embedding" if kind == "embedding" else None,
+ llm_model="granite-chat" if kind == "chat" else None,
+ )
+
+ assert fake.calls == 2
diff --git a/tests/unit/services/test_model_catalog.py b/tests/unit/services/test_model_catalog.py
index 55ee9fb97..67f8ee302 100644
--- a/tests/unit/services/test_model_catalog.py
+++ b/tests/unit/services/test_model_catalog.py
@@ -524,3 +524,32 @@ async def test_live_model_failure_does_not_block_other_provider_refreshes(monkey
await model_catalog.refresh_live_models()
assert calls == ["broken", "healthy"]
+
+
+async def test_refresh_live_models_can_be_narrowed_to_one_provider(monkeypatch):
+ """A caller asking about one provider must not wait on another's cluster."""
+ calls: list[str] = []
+
+ def lister(name):
+ async def fetch(_credentials):
+ calls.append(name)
+
+ return fetch
+
+ first = SimpleNamespace(PROVIDER_KEY="first", fetch_models=lister("first"))
+ second = SimpleNamespace(PROVIDER_KEY="second", fetch_models=lister("second"))
+ providers = SimpleNamespace(stored_credentials=lambda _provider: {"api_key": "secret"})
+
+ monkeypatch.setattr(model_catalog, "provider_enhancements", lambda: (first, second))
+ monkeypatch.setattr(
+ model_catalog, "supported_provider_keys", lambda: frozenset({"first", "second"})
+ )
+ monkeypatch.setattr(
+ "config.settings.get_openrag_config",
+ lambda: SimpleNamespace(providers=providers),
+ )
+
+ await model_catalog.refresh_live_models(" Second ")
+
+ assert calls == ["second"]
diff --git a/tests/unit/services/test_status_checks.py b/tests/unit/services/test_status_checks.py
index fcb89400e..240b4e422 100644
--- a/tests/unit/services/test_status_checks.py
+++ b/tests/unit/services/test_status_checks.py
@@ -403,3 +403,80 @@ async def test_last_error_populated_on_failure(monkeypatch):
r = await check_langflow()
assert r.last_error is not None
assert len(r.last_error) > 0
+
+
+# Busy-vs-down classification: a timeout after the connection opened is DEGRADED
+
+
+ argnames="client_attr,check,target",
+ argvalues=[
+ ("langflow_http_client", check_langflow, "/api/v1/version"),
+ ("docling_http_client", check_docling, "/version"),
+ ],
+)
+async def test_read_timeout_is_degraded(monkeypatch, client_attr, check, target):
+ monkeypatch.setattr(
+ clients, client_attr, _mock_http(raises=httpx.ReadTimeout("")), raising=False
+ )
+
+ r = await check()
+
+ assert r.status == ComponentState.DEGRADED
+ assert "slow to respond" in (r.message or "")
+ assert r.last_error is not None
+ assert "ReadTimeout after" in r.last_error
+ assert target in r.last_error
+ assert r.version is None
+
+
+ argnames="client_attr,check",
+ argvalues=[
+ ("langflow_http_client", check_langflow),
+ ("docling_http_client", check_docling),
+ ],
+)
+async def test_connect_timeout_is_unhealthy(monkeypatch, client_attr, check):
+ monkeypatch.setattr(
+ clients, client_attr, _mock_http(raises=httpx.ConnectTimeout("")), raising=False
+ )
+
+ r = await check()
+
+ assert r.status == ComponentState.UNHEALTHY
+ assert "unreachable" in (r.message or "").lower()
+ assert "ConnectTimeout" in (r.last_error or "")
+
+
+async def test_langflow_read_timeout_records_one_warning_then_recovers(monkeypatch):
+ monkeypatch.setattr(
+ clients,
+ "langflow_http_client",
+ _mock_http(raises=httpx.ReadTimeout("")),
+ raising=False,
+ )
+ await check_langflow()
+ await check_langflow()
+
+ entries = _cl.get_entries("langflow", 10)
+ assert [e["level"] for e in entries] == ["warning"]
+ assert "ReadTimeout" in (entries[0]["detail"] or "")
+ assert "target:" in (entries[0]["detail"] or "")
+
+ monkeypatch.setattr(clients, "langflow_http_client", _mock_http(200), raising=False)
+ r = await check_langflow()
+
+ assert r.status == ComponentState.HEALTHY
+ entries = _cl.get_entries("langflow", 10)
+ assert entries[-1]["level"] == "info"
+ assert entries[-1]["message"].startswith("recovered:")
+
+
+def test_check_timeout_stays_below_outer_deadline():
+ from services import status_service
+
+ assert status_checks._CHECK_TIMEOUT_S < status_service.CHECK_TIMEOUT_S
diff --git a/tests/unit/test_backend_router.py b/tests/unit/test_backend_router.py
index dc5845502..a4301ad20 100644
--- a/tests/unit/test_backend_router.py
+++ b/tests/unit/test_backend_router.py
@@ -44,6 +44,7 @@ def test_callback_url_uses_backend_when_disabled(monkeypatch):
def test_llm_base_url_uses_router_when_enabled(monkeypatch):
+ monkeypatch.delenv("OPENRAG_LLM_PROXY_URL", raising=False)
monkeypatch.setattr(settings, "OPENRAG_BACKEND_ROUTER_ENABLE", True)
monkeypatch.setattr(settings, "OPENRAG_BACKEND_ROUTER_URL", "http://router:8100")
assert settings.get_langflow_llm_base_url() == "http://router:8100"
diff --git a/tests/unit/test_docling_remote_task_not_found.py b/tests/unit/test_docling_remote_task_not_found.py
new file mode 100644
index 000000000..8905a6032
--- /dev/null
+++ b/tests/unit/test_docling_remote_task_not_found.py
@@ -0,0 +1,131 @@
+"""Docling Serve component: a 404 on a task poll is reported as an expired task.
+
+docling-serve deletes a finished task some minutes after its result is first
+fetched. When a flow run was replayed after that, the component surfaced a bare
+"Client error '404 Not Found' for url .../status/poll/<id>", which hid the real
+failure. It now raises an explicit "task not found / expired" error, both on
+the first poll and on a poll inside the wait loop (which previously fell
+through to a `KeyError` on the 404 body).
+
+`lfx` and docling-core are not in the backend venv, so they are stubbed for the
+duration of each test only.
+"""
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import httpx
+import pytest
+
+COMPONENT_PATH = (
+ Path(__file__).resolve().parents[2] / "custom_components" / "openrag" / "docling_remote.py"
+)
+BASE_URL = "http://docling:5001/v1"
+TASK_ID = "df980024-2298-48f9-84e6-eadc046221cc"
+
+
+class _BaseFileComponent:
+ class BaseFile: # referenced in annotations only
+ pass
+
+ @staticmethod
+ def get_base_inputs():
+ return []
+
+ @staticmethod
+ def get_base_outputs():
+ return []
+
+
+def component(monkeypatch):
+ def module(name, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ monkeypatch.setitem(sys.modules, name, mod)
+ return mod
+
+ def factory(*args, **kwargs):
+ return None
+
+ module("lfx")
+ module("lfx.base")
+ module("lfx.base.data", BaseFileComponent=_BaseFileComponent)
+ module(
+ "lfx.inputs",
+ IntInput=factory,
+ NestedDictInput=factory,
+ StrInput=factory,
+ TableInput=factory,
+ )
+ module("lfx.inputs.inputs", FloatInput=factory)
+ module("lfx.schema", Data=dict, dotdict=dict)
+ module("lfx.utils")
+ module("lfx.utils.util", transform_localhost_url=lambda url: url)
+ module("docling_core")
+ module("docling_core.types")
+ module("docling_core.types.doc", DoclingDocument=object)
+
+ spec = importlib.util.spec_from_file_location("_docling_remote_under_test", COMPONENT_PATH)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ monkeypatch.setattr(mod.time, "sleep", lambda _seconds: None)
+
+ instance = object.__new__(mod.DoclingRemoteComponent)
+ instance.logs = []
+ instance.log = instance.logs.append
+ instance.max_poll_timeout = 60
+ return instance
+
+
+def _client(statuses: list[tuple[int, dict]]) -> tuple[httpx.Client, list[str]]:
+ seen: list[str] = []
+ queue = list(statuses)
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request.url.path)
+ status, body = queue.pop(0)
+ return httpx.Response(status, json=body)
+
+ return httpx.Client(transport=httpx.MockTransport(handler)), seen
+
+
+def test_a_missing_task_on_the_first_poll_is_reported_as_expired(component):
+ client, seen = _client([(404, {"detail": "Task not found."})])
+
+ with pytest.raises(RuntimeError, match="not found on the Docling Serve server") as exc:
+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)
+
+ assert TASK_ID in str(exc.value)
+ assert "expired" in str(exc.value)
+ assert seen == [f"/v1/status/poll/{TASK_ID}"]
+ assert component.logs and TASK_ID in component.logs[0]
+
+
+def test_a_task_that_disappears_while_polling_is_reported_as_expired(component):
+ client, seen = _client(
+ [
+ (200, {"task_status": "started"}),
+ (404, {"detail": "Task not found."}),
+ ]
+ )
+
+ with pytest.raises(RuntimeError, match="not found on the Docling Serve server"):
+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)
+
+ assert len(seen) == 2
+
+
+def test_other_client_errors_while_polling_still_raise_http_errors(component):
+ client, _ = _client(
+ [
+ (200, {"task_status": "started"}),
+ (403, {"detail": "Forbidden"}),
+ ]
+ )
+
+ with pytest.raises(httpx.HTTPStatusError):
+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
index 33219183d..3ba1cea8c 100644
--- a/tests/unit/test_encryption.py
+++ b/tests/unit/test_encryption.py
@@ -6,11 +6,17 @@ from config.config_manager import ConfigManager
from connectors.connection_manager import ConnectionManager
from utils.encryption import decrypt_secret, encrypt_secret
+
@pytest.fixture(autouse=True)
def setup_encryption_env(monkeypatch):
import utils.encryption
+
utils.encryption._cached_master_secret = None
- monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"0123456789abcdef0123456789abcdef").decode("ascii"))
+ monkeypatch.setenv(
+ "OPENRAG_ENCRYPTION_KEY",
+ base64.b64encode(b"0123456789abcdef0123456789abcdef").decode("ascii"),
+ )
+
def test_encryption_utility():
print("Testing encryption utility...")
@@ -19,82 +25,87 @@ def test_encryption_utility():
assert isinstance(payload, dict)
assert payload["algorithm"] == "AES-256-GCM"
assert payload["tenant_id"] == "tenant-1"
-
+
# Passing explicitly matches original AAD
decrypted = decrypt_secret(payload, expected_tenant_id="tenant-1")
assert decrypted == plaintext
-
+
# Missing explicit bound works generically using payload lookup
decrypted_fallback = decrypt_secret(payload)
assert decrypted_fallback == plaintext
-
+
# Spoofing the identity dynamically rejects the AES-GCM tags!
- try:
+ with pytest.raises(ValueError):
decrypt_secret(payload, expected_tenant_id="wrong-tenant-id")
- assert False, "Should have thrown ValueError from AESGCM AAD mismatch"
- except ValueError:
- pass
print("OK")
+
def test_config_manager(tmp_path):
print("Testing config manager encryption...")
# Create an initial config manager with a temporary file
test_yaml = tmp_path / "test_openrag_config.yaml"
if test_yaml.exists():
test_yaml.unlink()
-
+
cm = ConfigManager(str(test_yaml))
config = cm.get_config()
config.providers.openai.api_key = "openai-api-key-plaintext"
-
+
# Save the config
cm.save_config_file(config)
-
+
import yaml
- with open(test_yaml, "r") as f:
+
+ with open(test_yaml) as f:
saved_data = yaml.safe_load(f)
-
+
# Verify it was encrypted on disk
assert isinstance(saved_data["providers"]["openai"]["api_key"], dict)
assert saved_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"
-
+
# Verify it can be loaded correctly
cm_new = ConfigManager(str(test_yaml))
config_new = cm_new.get_config()
assert config_new.providers.openai.api_key == "openai-api-key-plaintext"
print("OK")
+
@pytest.mark.asyncio
async def test_connection_manager(tmp_path):
print("Testing connection manager encryption...")
test_json = tmp_path / "test_openrag_connections.json"
if test_json.exists():
test_json.unlink()
-
+
cm = ConnectionManager(str(test_json))
await cm.create_connection(
connector_type="google_drive",
name="Test Drive",
config={"client_secret": "my-client-secret-plaintext", "other_setting": "not-secret"},
- user_id="user-1"
+ user_id="user-1",
)
await cm.create_connection(
connector_type="aws_s3",
name="Test S3",
config={"secret_key": "aws-secret", "access_key": "aws-access", "bucket_names": ["foo"]},
- user_id="user-1"
+ user_id="user-1",
)
await cm.create_connection(
connector_type="ibm_cos",
name="Test IBM",
- config={"api_key": "ibm-api-key", "service_instance_id": "ibm-service", "bucket_names": ["bar"]},
- user_id="user-1"
+ config={
+ "api_key": "ibm-api-key",
+ "service_instance_id": "ibm-service",
+ "bucket_names": ["bar"],
+ },
+ user_id="user-1",
)
# Should be saved encrypted
import json
- with open(test_json, "r") as f:
+
+ with open(test_json) as f:
data = json.load(f)
-
+
found_gd = False
found_s3 = False
found_ibm = False
@@ -116,11 +127,11 @@ async def test_connection_manager(tmp_path):
assert c["config"]["api_key"]["algorithm"] == "AES-256-GCM"
assert isinstance(c["config"]["service_instance_id"], dict)
assert found_gd and found_s3 and found_ibm
-
+
# Reloading should decrypt
cm2 = ConnectionManager(str(test_json))
await cm2.load_connections()
-
+
found_gd = False
found_s3 = False
found_ibm = False
@@ -140,23 +151,25 @@ async def test_connection_manager(tmp_path):
assert found_gd and found_s3 and found_ibm
print("OK")
-def test_auto_upgrade_features(tmp_path):
+
+def test_auto_upgrade_features(tmp_path, monkeypatch):
+ # A developer's OPENAI_API_KEY would seed the config and mask the file value.
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
test_yaml = tmp_path / "test_openrag_config_upgrade.yaml"
import yaml
+
# Write purely plaintext config
with open(test_yaml, "w") as f:
- yaml.dump({
- "providers": {
- "openai": {"api_key": "raw-unencrypted-openai-key-from-past"}
- }
- }, f)
-
+ yaml.dump({"providers": {"openai": {"api_key": "raw-unencrypted-openai-key-from-past"}}}, f)
+
cm = ConfigManager(str(test_yaml))
cm.get_config()
# Upon loading, the auto-upgrade should save the file over itself with the encrypted key.
- with open(test_yaml, "r") as f:
+ with open(test_yaml) as f:
upgraded_data = yaml.safe_load(f)
- assert isinstance(upgraded_data["providers"]["openai"]["api_key"], dict), "Failed to auto-upgrade config"
+ assert isinstance(upgraded_data["providers"]["openai"]["api_key"], dict), (
+ "Failed to auto-upgrade config"
+ )
assert upgraded_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"
assert cm.get_config().providers.openai.api_key == "raw-unencrypted-openai-key-from-past"
print("Auto-upgrade OK")
diff --git a/tests/unit/test_langflow_global_variables.py b/tests/unit/test_langflow_global_variables.py
index 23d8cd215..b82858f55 100644
--- a/tests/unit/test_langflow_global_variables.py
+++ b/tests/unit/test_langflow_global_variables.py
@@ -135,7 +135,8 @@ async def test_update_langflow_global_variable_patches_when_type_matches():
"value": "new-secret",
"default_fields": [],
"type": "Credential",
- }
+ },
+ "idempotent": True,
},
),
]
@@ -206,7 +207,8 @@ async def test_ensure_required_langflow_global_variables_removes_apply_to_fields
"name": "OPENAI_API_KEY",
"default_fields": [],
"type": "Credential",
- }
+ },
+ "idempotent": True,
},
)
@@ -359,7 +361,8 @@ async def test_update_langflow_global_variable_overwrites_redacted_credential():
"value": "None",
"default_fields": [],
"type": "Credential",
- }
+ },
+ "idempotent": True,
},
),
]
diff --git a/tests/unit/test_langflow_utils.py b/tests/unit/test_langflow_utils.py
index d0d818c46..6de5c06e2 100644
--- a/tests/unit/test_langflow_utils.py
+++ b/tests/unit/test_langflow_utils.py
@@ -326,6 +326,7 @@ async def test_enable_mcp_none_for_project():
"PATCH",
"/api/v1/mcp/project/proj-456",
json={"settings": [], "auth_settings": {"auth_type": "none"}},
+ idempotent=True,
)
@@ -346,6 +347,7 @@ async def test_enable_mcp_none_for_project_failure():
"PATCH",
"/api/v1/mcp/project/proj-456",
json={"settings": [], "auth_settings": {"auth_type": "none"}},
+ idempotent=True,
)
diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.py
index 2f872ee4d..dbd63ff2e 100644
--- a/tests/unit/test_provider_error_formatting.py
+++ b/tests/unit/test_provider_error_formatting.py
@@ -8,8 +8,10 @@ import pytest
from api.provider_validation import (
format_provider_error_message,
+ is_context_window_error,
is_generic_upstream_error,
is_provider_credential_error,
+ is_provider_truncated_response_error,
looks_like_provider_error_content,
resolve_chat_stream_error_message,
sanitize_provider_error_content,
@@ -514,3 +516,52 @@ async def test_probe_checks_non_selected_providers_with_keys(monkeypatch):
from api.provider_validation import probe_provider_credential_error
assert await probe_provider_credential_error() == "Provided API key is Invalid."
+
+
+ "text",
+ [
+ "This model's maximum context length is 32768 tokens. However, you requested 0 "
+ "output tokens and your prompt contains at least 32769 input tokens",
+ '{"error": {"code": "context_length_exceeded"}}',
+ "ContextWindowExceededError: litellm.ContextWindowExceededError: ...",
+ "prompt is too long: 210000 tokens > 200000 maximum",
+ ],
+)
+def test_is_context_window_error(text):
+ assert is_context_window_error(text)
+
+
+ "text",
+ [None, "", "Incorrect API key provided", "The model `x` does not exist", "rate limit"],
+)
+def test_is_context_window_error_rejects_other_failures(text):
+ assert not is_context_window_error(text)
+
+
+_KUBE_RBAC_PROXY_CUTOFF = (
+ "MidStreamFallbackError: litellm.MidStreamFallbackError: litellm.APIConnectionError: "
+ "APIConnectionError: Hosted_vllmException - Response payload is not completed: "
+ "<TransferEncodingError: 400, message='Not enough data to satisfy transfer length header.'>"
+)
+
+
+ "text",
+ [
+ _KUBE_RBAC_PROXY_CUTOFF,
+ "ClientPayloadError: Response payload is not completed",
+ "httpx.RemoteProtocolError: peer closed connection (incomplete chunked read)",
+ ],
+)
+def test_is_provider_truncated_response_error(text):
+ assert is_provider_truncated_response_error(text)
+
+
+ "text",
+ [None, "", "Incorrect API key provided", "Connection refused", "maximum context length"],
+)
+def test_is_provider_truncated_response_error_rejects_other_failures(text):
+ assert not is_provider_truncated_response_error(text)
diff --git a/tests/unit/test_search_service_embedding_spaces.py b/tests/unit/test_search_service_embedding_spaces.py
index 616972b66..76b02cbcd 100644
--- a/tests/unit/test_search_service_embedding_spaces.py
+++ b/tests/unit/test_search_service_embedding_spaces.py
@@ -64,7 +64,9 @@ async def test_backend_search_queries_exact_and_legacy_embedding_providers(monke
service.models_service = None
routes = []
- async def create_embedding(body):
+ async def create_embedding(body, *, interactive=False):
+ # Query embeddings take the interactive lane, never the bulk one.
+ assert interactive is True
routes.append(body["model"])
return {"data": [{"embedding": [0.1, 0.2]}]}
diff --git a/tests/unit/test_search_service_highlights.py b/tests/unit/test_search_service_highlights.py
index e3b23981a..387268c8b 100644
--- a/tests/unit/test_search_service_highlights.py
+++ b/tests/unit/test_search_service_highlights.py
@@ -62,7 +62,7 @@ def _make_service(opensearch: _OpenSearch, monkeypatch) -> SearchService:
)
service.models_service = None
- async def _no_embed(body):
+ async def _no_embed(body, **_kwargs):
return {"data": [{"embedding": [0.1, 0.2]}]}
monkeypatch.setattr("services.search_service.gateway_embeddings", _no_embed)