OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Improve RHOAI model provider error handling and resiliency (#2419)
Mike Pawlowskifa2619db1273
diff --git a/.env.example b/.env.exampleindex cdbc80318..d02a193a3 100644--- a/.env.example+++ b/.env.example@@ -323,6 +323,16 @@ WATSONX_PROJECT_ID=# deployed environment.# RHOAI_TLS_VERIFY=#+# Maximum embedding requests OpenRAG sends to the embeddings endpoint at once+# (default 16, sized for a GPU-served model; 0 = no limit). Extra requests wait+# inside OpenRAG. Set 4 for a CPU-served model: it embeds well under one chunk+# per second, and the endpoint's kube-rbac-proxy answers 502 once a request+# waits upstream past its 30s timeout -- which fails folder uploads (several+# files x 8 embedding threads). Also editable in Settings. One of these+# slots is reserved for search and chat queries, so they are not queued behind+# ingestion (with a limit of 1, both share the single slot).+# RHOAI_EMBEDDING_MAX_CONCURRENCY=+## Selecting the models. The ids are each InferenceService's --served-model-name,# not an image tag. OpenRAG lists the real ones from each endpoint's /v1/models.# LLM_PROVIDER=rhoai@@ -431,6 +441,14 @@ LOG_FORMAT=# - Default: INFO# LOG_LEVEL=DEBUG+# OPTIONAL: Set the logging level for the Langflow container+# - Supported values: DEBUG, INFO, WARNING, ERROR, CRITICAL+# - Default: INFO+# - DEBUG logs every component's full result, including whole parsed Docling+# documents (multi-MB lines). Writing them blocks Langflow's single worker+# during ingestion, so health checks time out. Enable only when debugging a flow.+# LANGFLOW_LOG_LEVEL=DEBUG+# OPTIONAL: Disable colored log output (https://no-color.org)# - When set (to any value), ANSI color codes are omitted from log output# NO_COLOR=1diff --git a/.github/workflows/autofix.ci.yml b/.github/workflows/autofix.ci.ymlindex 30fa2056e..4a0960b42 100644--- a/.github/workflows/autofix.ci.yml+++ b/.github/workflows/autofix.ci.yml@@ -78,6 +78,8 @@ jobs:# the App's commit triggers a fresh CI cycle that passes.- name: Apply ruff autofix via autofix.ciif: steps.changed.outputs.files != ''- uses: autofix-ci/[email protected]+ # v1.3.3+ calls autofix-api.maximilianhils.com; older pins call+ # api.autofix.ci, which no longer resolves (ENOTFOUND).+ uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4with:commit-message: "style: ruff autofix (auto)"diff --git a/Makefile b/Makefileindex ab60a079a..37337061c 100644--- a/Makefile+++ b/Makefile@@ -371,6 +371,7 @@ help_local: ## Show local development commands@echo " $(CYAN)RHOAI_EMBEDDINGS_ENDPOINT$(NC) - embedding InferenceService URL (vLLM serves one model each)"@echo " $(CYAN)RHOAI_API_KEY$(NC) - ServiceAccount bearer token for both endpoints"@echo " $(CYAN)RHOAI_TLS_VERIFY$(NC) - CA bundle path, or 'false' for a port-forward (dev only)"+ @echo " $(CYAN)RHOAI_EMBEDDING_MAX_CONCURRENCY$(NC) - max in-flight embedding calls (default 16 for GPU; use 4 for CPU; 0 = no limit)"@echo " In-cluster, use the predictor Service DNS and the projected service CA:"@echo " $(CYAN)/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt$(NC)"@echo " From a laptop, port-forward each InferenceService and set RHOAI_TLS_VERIFY=false:"diff --git a/custom_components/openrag/docling_remote.py b/custom_components/openrag/docling_remote.pyindex 522fb10ae..7842c64f3 100644--- a/custom_components/openrag/docling_remote.py+++ b/custom_components/openrag/docling_remote.py@@ -209,6 +209,23 @@ class DoclingRemoteComponent(BaseFileComponent):# Default behaviorreturn super().update_build_config(build_config, field_value, field_name)+ def _raise_if_task_missing(self, response: httpx.Response, task_id: str) -> None:+ """Raise a clear error when Docling Serve no longer knows the task.++ Docling Serve deletes a finished task some minutes after its result is+ first fetched, and forgets every task when it restarts. Polling such a+ task returns 404, which would otherwise surface as a bare HTTP error.+ """+ if response.status_code != httpx.codes.NOT_FOUND:+ return+ msg = (+ f"Docling task {task_id} was not found on the Docling Serve server. Its result "+ "may have expired (Docling Serve deletes finished results after a delay) or the "+ "server restarted. Re-ingest the file."+ )+ self.log(msg)+ raise RuntimeError(msg)+def _poll_and_fetch_result(self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None) -> Data | None:@@ -229,6 +246,7 @@ class DoclingRemoteComponent(BaseFileComponent):start_wait_time = time.monotonic()response = client.get(f"{base_url}/status/poll/{task_id}")+ self._raise_if_task_missing(response, task_id)response.raise_for_status()task = response.json()@@ -255,6 +273,8 @@ class DoclingRemoteComponent(BaseFileComponent):return Nonecontinue+ self._raise_if_task_missing(response, task_id)+ response.raise_for_status()task = response.json()result_resp = client.get(f"{base_url}/result/{task_id}")diff --git a/docker-compose.yml b/docker-compose.ymlindex 9a9cd015a..bbf2320a5 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -94,6 +94,7 @@ services:- RHOAI_EMBEDDINGS_ENDPOINT=${RHOAI_EMBEDDINGS_ENDPOINT:-}- RHOAI_API_KEY=${RHOAI_API_KEY:-}- RHOAI_TLS_VERIFY=${RHOAI_TLS_VERIFY:-}+ - RHOAI_EMBEDDING_MAX_CONCURRENCY=${RHOAI_EMBEDDING_MAX_CONCURRENCY:-}- OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT}- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}@@ -273,7 +274,7 @@ services:- FILESIZE=0- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-text-embedding-3-small}- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_SERVE_VERIFY_SSL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENRAG_LLM_TOKEN,OPENRAG_LLM_BASE_URL,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE- - LANGFLOW_LOG_LEVEL=DEBUG+ - LANGFLOW_LOG_LEVEL=${LANGFLOW_LOG_LEVEL:-INFO}- LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}- LANGFLOW_RATE_LIMIT_PER_MINUTE=${LANGFLOW_RATE_LIMIT_PER_MINUTE:-100}- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex 36ed10847..88049184e 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -74,6 +74,7 @@ These variables are initially set during the [application onboarding process](/i| `RHOAI_EMBEDDINGS_ENDPOINT` | Not set | Embeddings endpoint for the Red Hat OpenShift AI model provider, a second `InferenceService`. Leave unset only if one endpoint serves both chat and embeddings. || `RHOAI_API_KEY` | Not set | Kubernetes bearer token for the Red Hat OpenShift AI model provider, sent to both endpoints. Its subject needs `get` on `inferenceservices` in the model namespace. || `RHOAI_TLS_VERIFY` | Not set | TLS trust for the Red Hat OpenShift AI endpoints. Set to a CA bundle path, such as the service-serving CA OpenShift projects into every pod (`/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`), or `false` to skip verification for a local `oc port-forward` only. Scoped to this provider, so it does not affect other providers. |+| `RHOAI_EMBEDDING_MAX_CONCURRENCY` | `16` | Maximum number of embedding requests OpenRAG sends to the Red Hat OpenShift AI embeddings endpoint at once; further requests wait inside OpenRAG. The default suits a GPU-served model. For a CPU-served model set `4`: it answers slowly enough that more queued requests outlast the endpoint proxy's upstream timeout, which returns `502` when several files are ingested together. One slot is reserved for search and chat queries so they are not queued behind ingestion; with a limit of `1` both share it. Set `0` for no limit. Also editable in **Settings**. |Newly indexed chunks keep the model name in `embedding_model` for compatibility and also store`embedding_provider` plus the unambiguous `embedding_space_id` (`provider:model`). Retrieval queriesdiff --git a/enhancements/providers/redhat/openshift_ai.py b/enhancements/providers/redhat/openshift_ai.pyindex d153d34d8..e08889c2e 100644--- a/enhancements/providers/redhat/openshift_ai.py+++ b/enhancements/providers/redhat/openshift_ai.py@@ -145,13 +145,41 @@ CREDENTIAL_FIELDS: list[dict[str, Any]] = ["options": None,"default_value": None,},+ {+ "key": "embedding_max_concurrency",+ "label": "Max concurrent embedding requests",+ "placeholder": "16",+ "tooltip": "How many embedding requests OpenRAG sends to the embedding endpoint at "+ "once; the rest wait in OpenRAG. One slot is kept for search and chat queries so "+ "they never wait behind ingestion. Leave blank for the default of 16, sized for a "+ "GPU-served model. Lower it to 4 for a CPU-served model: it answers slowly, and "+ "the endpoint's kube-rbac-proxy returns 502 once a queued request waits past its "+ "upstream timeout (30s by default), which fails ingestion. 0 means no limit.",+ "required": False,+ "field_type": "text",+ "options": None,+ "default_value": None,+ },]#: Fields an operator fills in that are not LiteLLM kwargs. The embedding#: endpoint is selected *into* `api_base` per call; forwarding it as well would#: land it in the request body, since LiteLLM passes kwargs it does not-#: recognise straight through.-_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base"})+#: recognise straight through. The concurrency limit is enforced by the+#: gateway (`embedding_max_concurrency`), never sent to the endpoint.+_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base", "embedding_max_concurrency"})++#: Concurrent embedding calls allowed when the operator sets no limit, sized for+#: a GPU-served model, the production norm. Ingestion sends one chunk per call,+#: so vLLM's single API-server process (HTTP, tokenization, JSON) saturates+#: long before the GPU does: granite-embedding-english-r2 on an H100 served+#: about 200 calls/s at 16 in flight (p95 under 170ms) and topped out near 430+#: calls/s at 64-128, with the GPU about 20% busy. 16 leaves that headroom to+#: other OpenRAG instances sharing the endpoint. A CPU-served model is ~100x+#: slower (about 2.7s per ~1000-character chunk on 5 cores), so 16 in flight+#: queues calls past the 30s `kube-rbac-proxy` upstream timeout; CPU+#: deployments set 4.+DEFAULT_EMBEDDING_MAX_CONCURRENCY = 16#: LiteLLM kwargs that must not become request-body fields. `ssl_verify` is#: read for TLS *and* — on the `hosted_vllm` chat path, verified against litellm@@ -361,6 +389,35 @@ def litellm_credentials(stored: Mapping[str, Any], *, kind: CallKind = "chat") -return credentials+def embedding_max_concurrency(stored: Mapping[str, Any] | None) -> int | None:+ """How many embedding calls the gateway may have in flight to this endpoint.++ Optional member of the enhancement contract, read through+ `registry.embedding_concurrency_for`. Returns `None` for "no limit".++ Blank means the default. `0` switches the limit off. A value that is not a+ non-negative integer falls back to the default with a one-time warning+ rather than disabling the limit: failing open here is exactly the overload+ this setting exists to prevent.+ """+ raw = _values(stored).get("embedding_max_concurrency", "")+ if not raw:+ return DEFAULT_EMBEDDING_MAX_CONCURRENCY+ try:+ limit = int(raw)+ except ValueError:+ limit = -1+ if limit < 0:+ _warn_once(+ f"concurrency:{raw}",+ "Ignoring an invalid OpenShift AI embedding concurrency limit; using the default.",+ value=raw,+ default=DEFAULT_EMBEDDING_MAX_CONCURRENCY,+ )+ return DEFAULT_EMBEDDING_MAX_CONCURRENCY+ return limit or None++# --------------------------------------------------------------------------# Model discovery# --------------------------------------------------------------------------@@ -456,8 +513,40 @@ def _auth_headers(api_key: str) -> dict[str, str]:return {"Authorization": f"Bearer {api_key}", "Accept": "application/json"}+def _content_type(response: Any) -> str:+ """The response's media type, lower-cased and without parameters, or ''."""+ try:+ headers = getattr(response, "headers", None) or {}+ value = headers.get("content-type", "") or ""+ return str(value).split(";", 1)[0].strip().lower()+ except Exception:+ return ""+++def _looks_like_html(response: Any) -> bool:+ """Whether a response is a web page rather than an API answer.++ An OAuth proxy, SSO gateway or captive portal in front of the route answers+ with its login page. The content type usually says so; when it is missing,+ the start of the body does.+ """+ if "html" in _content_type(response):+ return True+ try:+ head = str(getattr(response, "text", "") or "").lstrip()[:64].lower()+ except Exception:+ return False+ return head.startswith(("<!doctype html", "<html"))++def _error_details(response: Any) -> str:- """The concise error an OpenAI-compatible endpoint returned."""+ """The concise error an OpenAI-compatible endpoint returned.++ A web page is described rather than quoted: half a kilobyte of login-page+ markup helps nobody in a Settings error or a log line.+ """+ if _looks_like_html(response):+ return f"The response was an HTML page ({_content_type(response) or 'text/html'})."try:body = response.json()except (ValueError, TypeError):@@ -525,6 +614,18 @@ def model_ids(body: Any) -> tuple[str, ...]:return tuple(ids)+def is_model_listing(body: Any) -> bool:+ """Whether `body` has the shape of an OpenAI `GET /v1/models` answer.++ Only the shape is checked — an object whose `data` is a list. An empty list+ still counts: a `vLLM` pod that is still loading its weights is a real+ endpoint, and the health check deliberately does not depend on a model+ having been chosen. `model_ids` cannot answer this, because it returns+ nothing both for a wrong shape and for an empty listing.+ """+ return isinstance(body, Mapping) and isinstance(body.get("data"), list)++async def _list_models(client: Any, api_base: str, api_key: str) -> tuple[str, ...] | None:"""The models one endpoint serves, or None if it could not be asked."""url = models_url(api_base)@@ -682,6 +783,10 @@ async def lightweight_health_check(credentials: Mapping[str, Any]) -> None:carries `embedding_api_base`; `validate_provider_setup(stored_credentials=...)`is how it gets here. Given only the LiteLLM form, the one endpoint it wasnarrowed to is checked.++ A 200 only passes when its body is an OpenAI-shaped model listing: an+ OAuth proxy or SSO gateway in front of the route can answer 200 with its+ login page, which would otherwise read as healthy and fail on first use."""import httpx@@ -721,6 +826,7 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)import httpxurl = models_url(api_base)+ logger.debug("Checking the OpenShift AI endpoint", endpoint=label, url=url)try:response = await _http_request_with_retry("GET",@@ -743,8 +849,26 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)f"Could not reach the OpenShift AI {label} endpoint at {api_base}: {error}") from error+ if 300 <= response.status_code < 400:+ # httpx does not follow redirects, and should not here: the target is+ # a login page, and the token must not be carried to it.+ location = _redirect_location(response)+ logger.error(+ "The OpenShift AI endpoint redirected instead of answering",+ endpoint=label,+ url=url,+ status_code=response.status_code,+ location=location,+ )+ raise Exception(+ f"The OpenShift AI {label} endpoint at {api_base} redirected to "+ f"{location or 'another page'} instead of answering /v1/models. This is usually "+ "an OAuth proxy or SSO login page in front of the route; point the URL at the "+ "model's inference route or predictor Service, which accepts a ServiceAccount token."+ )+if response.status_code == 200:- logger.debug("OpenShift AI endpoint answered", endpoint=label, url=url)+ _require_model_listing(response, label, api_base, url)returndetails = _error_details(response)@@ -769,3 +893,76 @@ async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str)raise Exception(f"The OpenShift AI {label} endpoint returned {response.status_code}: {details}".strip())+++def _redirect_location(response: Any) -> str:+ """Where a redirect points, without its query string.++ An OAuth redirect's query carries `state`, `client_id` and a nested+ `redirect_uri`, which make the message unreadable and say nothing the+ host and path do not.+ """+ try:+ headers = getattr(response, "headers", None) or {}+ location = str(headers.get("location", "") or "").strip()+ if not location:+ return ""+ parts = urlsplit(location)+ return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))+ except Exception:+ return ""+++def _require_model_listing(response: Any, label: str, api_base: str, url: str) -> None:+ """Fail a 200 whose body is not a model listing, naming the likely cause."""+ try:+ body = response.json()+ except (ValueError, TypeError):+ body = None++ if is_model_listing(body):+ models = model_ids(body)+ if models:+ logger.debug(+ "OpenShift AI endpoint answered with a model list",+ endpoint=label,+ url=url,+ models=len(models),+ )+ else:+ # Still a real endpoint — vLLM may be loading its weights — so+ # this passes; the warning is for whoever wonders why the picker+ # shows the configured fallback.+ logger.warning(+ "The OpenShift AI endpoint answered with an empty model list",+ endpoint=label,+ url=url,+ )+ return++ content_type = _content_type(response)+ if _looks_like_html(response):+ logger.error(+ "The OpenShift AI endpoint answered with a web page instead of a model list",+ endpoint=label,+ url=url,+ content_type=content_type,+ )+ raise Exception(+ f"The OpenShift AI {label} endpoint at {api_base} returned a web page "+ f"({content_type or 'text/html'}) instead of a model list. This is usually an "+ "OAuth proxy or SSO login page in front of the route; point the URL at the "+ "model's inference route or predictor Service, which accepts a ServiceAccount token."+ )+ logger.error(+ "The OpenShift AI endpoint answered with something other than a model list",+ endpoint=label,+ url=url,+ content_type=content_type,+ body_type=type(body).__name__,+ )+ raise Exception(+ f"The OpenShift AI {label} endpoint at {api_base} answered /v1/models with something "+ "other than an OpenAI-compatible model list. Check the URL points at the model's "+ "inference route and ends in /v1."+ )diff --git a/enhancements/providers/registry.py b/enhancements/providers/registry.pyindex d1a44aa05..00ae4ceee 100644--- a/enhancements/providers/registry.py+++ b/enhancements/providers/registry.py@@ -11,11 +11,13 @@ import functoolsimport inspectfrom collections.abc import Mappingfrom types import ModuleType-from typing import Any+from typing import Any, Literalfrom enhancements.providers.redhat import openshift_aifrom enhancements.providers.watsonx import onprem+CallKind = Literal["chat", "embedding"]+_ENHANCEMENTS: dict[str, ModuleType] = {onprem.PROVIDER_KEY: onprem,openshift_ai.PROVIDER_KEY: openshift_ai,@@ -50,6 +52,50 @@ def enhancements() -> tuple[ModuleType, ...]:return tuple(_ENHANCEMENTS.values())+def live_models_for(provider: str, kind: CallKind) -> tuple[str, ...] | None:+ """What `provider` last said it serves for `kind`, if it can say at all.++ None means *unknown*, never "serves nothing": the provider has no+ enhancement, the enhancement cannot list its own models, or nothing fresh+ is cached for that endpoint. Callers must treat None as no information —+ telling an operator their model is missing on the strength of a listing+ that never arrived would be worse than staying quiet.++ An empty tuple is the opposite: a real answer. The provider was listed and+ that endpoint serves nothing of this kind — watsonx on-prem caches exactly+ that for a cluster with chat models and no embedding model — so it must+ reach the caller as `()`, not be folded into unknown. An enhancement that+ cannot tell "empty" from "unreadable" keeps that half None itself.++ The lists come from the enhancement's own TTL cache, so this is a dict+ lookup; refreshing them is `model_catalog.refresh_live_models()`.+ """+ enhancement = get(provider)+ if enhancement is None or not hasattr(enhancement, "cached_models"):+ return None+ try:+ models = enhancement.cached_models()+ except Exception as exc: # a diagnostic must never take down its caller+ _log_hook_failure(provider, "cached_models", exc)+ return None+ if models is None:+ return None+ listed = getattr(models, kind, None)+ if listed is None:+ return None+ # A bare string would split into characters, and a scalar cannot be+ # iterated at all; either is a malformed cache, so unknown rather than a+ # list of nonsense model ids.+ if isinstance(listed, str | bytes):+ _log_unreadable_listing(provider, kind, listed)+ return None+ try:+ return tuple(str(model) for model in listed)+ except TypeError:+ _log_unreadable_listing(provider, kind, listed)+ return None++def credentials_for(enhancement: ModuleType,stored: Mapping[str, Any],@@ -84,6 +130,54 @@ def runtime_kwargs_for(return dict(build(stored)) if callable(build) else {}+def embedding_concurrency_for(+ provider: str,+ stored: Mapping[str, Any] | None,+) -> int | None:+ """How many embedding calls the gateway may have in flight to `provider`.++ The optional `embedding_max_concurrency(stored)` member of the enhancement+ contract. None means unlimited: the provider has no enhancement, the+ enhancement sets no limit, or the hook failed. A broken hook must not fail+ the request it would only have throttled, so errors are logged and ignored.+ """+ enhancement = get(provider)+ hook = getattr(enhancement, "embedding_max_concurrency", None)+ if not callable(hook):+ return None+ try:+ limit = hook(stored or {})+ except Exception as exc:+ _log_hook_failure(provider, "embedding_max_concurrency", exc)+ return None+ if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:+ return None+ return limit+++def _log_hook_failure(provider: str, hook: str, exc: Exception) -> None:+ from utils.logging_config import get_logger++ get_logger(__name__).warning(+ "A provider enhancement hook failed; continuing without it",+ provider=provider,+ hook=hook,+ error_type=type(exc).__name__,+ error=str(exc),+ )+++def _log_unreadable_listing(provider: str, kind: str, listed: object) -> None:+ from utils.logging_config import get_logger++ get_logger(__name__).warning(+ "A provider enhancement cached a model listing that is not a list; treating it as unknown",+ provider=provider,+ kind=kind,+ listing_type=type(listed).__name__,+ )++@functools.cachedef _accepts_kind(enhancement: ModuleType) -> bool:"""Whether the module's `litellm_credentials` takes a `kind` keyword.diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex 357649a41..9db862dcb 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -217,7 +217,7 @@"last_updated": "2026-06-02T20:40:57.828Z","legacy": false,"metadata": {- "code_hash": "dcf9aeedd059",+ "code_hash": "7b3186b7fe33","dependencies": {"dependencies": [{@@ -363,7 +363,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path # noqa: TC003\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data, dotdict\nfrom lfx.utils.util import transform_localhost_url\nfrom pydantic import ValidationError\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = (\n \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n )\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n StrInput(\n name=\"verify_ssl\",\n display_name=\"Verify SSL\",\n info=\"Whether to verify SSL certificates for Docling Serve.\",\n value=\"DOCLING_SERVE_VERIFY_SSL\",\n load_from_db=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(\n f\"The status requests got a http response {response.status_code} too many times.\"\n )\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if result.get(\"status\") == \"failure\" or result.get(\"errors\"):\n errors = result.get(\"errors\", [])\n err_msg_list = []\n for err in errors:\n if isinstance(err, dict) and \"error_message\" in err:\n err_msg_list.append(err[\"error_message\"])\n elif isinstance(err, str):\n err_msg_list.append(err)\n\n err_details = (\n \"; \".join(err_msg_list) if err_msg_list else \"Unknown Docling processing error\"\n )\n\n msg = f\"Docling processing failed: {err_details}\"\n raise ValueError(msg)\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _get_verify_ssl(self) -> bool:\n \"\"\"Determine whether to verify SSL certificates for Docling Serve.\n\n Returns:\n bool: True if SSL verification should be enforced, False otherwise.\n \"\"\"\n verify = getattr(self, \"verify_ssl\", None)\n if hasattr(verify, \"get_secret_value\"):\n verify = verify.get_secret_value()\n if hasattr(verify, \"text\"):\n verify = verify.text\n if isinstance(verify, bool):\n return verify\n val_str = str(verify or \"\").strip().lower()\n if val_str in (\"false\", \"0\", \"no\"):\n return False\n return True\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(\n self, file_list: list[BaseFileComponent.BaseFile]\n ) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(\n client: httpx.Client, file_path: Path, options: dict[str, Any]\n ) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [\n {\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}\n ],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = [None] * len(file_list)\n with (\n httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is not None:\n futures.append(\n (i, executor.submit(_convert_document, client, file.path, docling_options))\n )\n\n for idx, future in futures:\n try:\n processed_data[idx] = future.result()\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"+ "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path # noqa: TC003\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data, dotdict\nfrom lfx.utils.util import transform_localhost_url\nfrom pydantic import ValidationError\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = (\n \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n )\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n StrInput(\n name=\"verify_ssl\",\n display_name=\"Verify SSL\",\n info=\"Whether to verify SSL certificates for Docling Serve.\",\n value=\"DOCLING_SERVE_VERIFY_SSL\",\n load_from_db=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _raise_if_task_missing(self, response: httpx.Response, task_id: str) -> None:\n \"\"\"Raise a clear error when Docling Serve no longer knows the task.\n\n Docling Serve deletes a finished task some minutes after its result is\n first fetched, and forgets every task when it restarts. Polling such a\n task returns 404, which would otherwise surface as a bare HTTP error.\n \"\"\"\n if response.status_code != httpx.codes.NOT_FOUND:\n return\n msg = (\n f\"Docling task {task_id} was not found on the Docling Serve server. Its result \"\n \"may have expired (Docling Serve deletes finished results after a delay) or the \"\n \"server restarted. Re-ingest the file.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n self._raise_if_task_missing(response, task_id)\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(\n f\"The status requests got a http response {response.status_code} too many times.\"\n )\n return None\n continue\n\n self._raise_if_task_missing(response, task_id)\n response.raise_for_status()\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if result.get(\"status\") == \"failure\" or result.get(\"errors\"):\n errors = result.get(\"errors\", [])\n err_msg_list = []\n for err in errors:\n if isinstance(err, dict) and \"error_message\" in err:\n err_msg_list.append(err[\"error_message\"])\n elif isinstance(err, str):\n err_msg_list.append(err)\n\n err_details = (\n \"; \".join(err_msg_list) if err_msg_list else \"Unknown Docling processing error\"\n )\n\n msg = f\"Docling processing failed: {err_details}\"\n raise ValueError(msg)\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _get_verify_ssl(self) -> bool:\n \"\"\"Determine whether to verify SSL certificates for Docling Serve.\n\n Returns:\n bool: True if SSL verification should be enforced, False otherwise.\n \"\"\"\n verify = getattr(self, \"verify_ssl\", None)\n if hasattr(verify, \"get_secret_value\"):\n verify = verify.get_secret_value()\n if hasattr(verify, \"text\"):\n verify = verify.text\n if isinstance(verify, bool):\n return verify\n val_str = str(verify or \"\").strip().lower()\n if val_str in (\"false\", \"0\", \"no\"):\n return False\n return True\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(\n self, file_list: list[BaseFileComponent.BaseFile]\n ) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(\n client: httpx.Client, file_path: Path, options: dict[str, Any]\n ) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [\n {\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}\n ],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = [None] * len(file_list)\n with (\n httpx.Client(headers=self._process_headers(), verify=self._get_verify_ssl()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is not None:\n futures.append(\n (i, executor.submit(_convert_document, client, file.path, docling_options))\n )\n\n for idx, future in futures:\n try:\n processed_data[idx] = future.result()\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"},"delete_server_file_after_processing": {"_input_type": "BoolInput",diff --git a/frontend/app/api/queries/useDoclingHealthQuery.test.ts b/frontend/app/api/queries/useDoclingHealthQuery.test.tsnew file mode 100644index 000000000..0164d02ba--- /dev/null+++ b/frontend/app/api/queries/useDoclingHealthQuery.test.ts@@ -0,0 +1,85 @@+import { HttpResponse, http } from "msw";+import { describe, expect, it } from "vitest";+import { server } from "@/test-utils/msw/server";+import { createQueryWrapper, renderHook, waitFor } from "@/test-utils/render";+import { useDoclingHealthQuery } from "./useDoclingHealthQuery";++function renderDoclingHealth() {+ return renderHook(() => useDoclingHealthQuery({ retry: false }), {+ wrapper: createQueryWrapper(),+ });+}++describe("useDoclingHealthQuery", () => {+ it("reports healthy on a 200 healthy body", async () => {+ server.use(+ http.get("/api/docling/health", () =>+ HttpResponse.json({ status: "healthy", host: "localhost" }),+ ),+ );++ const { result } = renderDoclingHealth();++ await waitFor(() => expect(result.current.isSuccess).toBe(true));+ expect(result.current.data).toEqual({ status: "healthy" });+ });++ it("reports degraded when docling-serve is slow to respond", async () => {+ server.use(+ http.get("/api/docling/health", () =>+ HttpResponse.json({+ status: "degraded",+ message: "Docling Serve is slow to respond",+ host: "localhost",+ }),+ ),+ );++ const { result } = renderDoclingHealth();++ await waitFor(() => expect(result.current.isSuccess).toBe(true));+ expect(result.current.data).toEqual({+ status: "degraded",+ message: "Docling Serve is slow to respond",+ });+ });++ it("treats a 200 without a JSON body as healthy", async () => {+ server.use(+ http.get(+ "/api/docling/health",+ () => new HttpResponse(null, { status: 200 }),+ ),+ );++ const { result } = renderDoclingHealth();++ await waitFor(() => expect(result.current.isSuccess).toBe(true));+ expect(result.current.data).toEqual({ status: "healthy" });+ });++ it("reports unhealthy on a 503", async () => {+ server.use(+ http.get("/api/docling/health", () =>+ HttpResponse.json(+ { status: "unhealthy", message: "Connection timeout" },+ { status: 503 },+ ),+ ),+ );++ const { result } = renderDoclingHealth();++ await waitFor(() => expect(result.current.isSuccess).toBe(true));+ expect(result.current.data?.status).toBe("unhealthy");+ });++ it("reports backend-unavailable when the request itself fails", async () => {+ server.use(http.get("/api/docling/health", () => HttpResponse.error()));++ const { result } = renderDoclingHealth();++ await waitFor(() => expect(result.current.isSuccess).toBe(true));+ expect(result.current.data?.status).toBe("backend-unavailable");+ });+});diff --git a/frontend/app/api/queries/useDoclingHealthQuery.ts b/frontend/app/api/queries/useDoclingHealthQuery.tsindex 5e77eecc1..e9acb4ddb 100644--- a/frontend/app/api/queries/useDoclingHealthQuery.ts+++ b/frontend/app/api/queries/useDoclingHealthQuery.ts@@ -5,7 +5,9 @@ import {} from "@tanstack/react-query";export interface DoclingHealthResponse {- status: "healthy" | "unhealthy" | "backend-unavailable";+ // "degraded": docling-serve is up but slow to answer (busy converting), so+ // ingest still works and the "stopped" banner must not show.+ status: "healthy" | "degraded" | "unhealthy" | "backend-unavailable";message?: string;}@@ -19,6 +21,10 @@ async function checkDoclingHealth(): Promise<DoclingHealthResponse> {});if (response.ok) {+ const body = await response.json().catch(() => ({}));+ if (body?.status === "degraded") {+ return { status: "degraded", message: body.message };+ }return { status: "healthy" };} else if (response.status === 503) {return {@@ -53,8 +59,10 @@ export const useDoclingHealthQuery = (queryFn: checkDoclingHealth,retry: 1,refetchInterval: (query) => {- // If healthy, check every 30 seconds; otherwise check every 3 seconds- return query.state.data?.status === "healthy" ? 30000 : 3000;+ // Up (healthy or busy): check every 30 seconds; down: every 3 seconds.+ // Polling a busy docling-serve fast would only add to its load.+ const status = query.state.data?.status;+ return status === "healthy" || status === "degraded" ? 30000 : 3000;},refetchOnWindowFocus: false, // Disabled to reduce unnecessary calls on tab switchesrefetchOnMount: true,diff --git a/frontend/app/api/queries/useProviderHealthQuery.test.tsx b/frontend/app/api/queries/useProviderHealthQuery.test.tsxindex 60e582e0f..8587502a7 100644--- a/frontend/app/api/queries/useProviderHealthQuery.test.tsx+++ b/frontend/app/api/queries/useProviderHealthQuery.test.tsx@@ -112,6 +112,39 @@ describe("useProviderHealthQuery", () => {expect(result.current.data?.llm_error).toBe("Invalid API Key");});+ it("keeps the warnings a 503 carries beside its errors", async () => {+ const warning = {+ code: "stale_embedding_space",+ provider: "rhoai",+ message:+ "Documents are indexed with 'old', which this endpoint no longer serves",+ };+ server.use(+ http.get("/api/settings", () =>+ HttpResponse.json(makeSettings({ edited: true })),+ ),+ http.get(+ "/api/provider/health",+ () =>+ new HttpResponse(+ JSON.stringify({+ message: "Provider validation failed",+ embedding_error: "endpoint unreachable",+ warnings: [warning],+ }),+ { status: 503 },+ ),+ ),+ );++ const { result } = renderHook(() => useProviderHealthQuery(), {+ wrapper: wrapper(),+ });++ await waitFor(() => expect(result.current.data?.status).toBe("unhealthy"));+ expect(result.current.data?.warnings).toEqual([warning]);+ });+it("maps any other non-ok status to an error status", async () => {server.use(http.get("/api/settings", () =>diff --git a/frontend/app/api/queries/useProviderHealthQuery.ts b/frontend/app/api/queries/useProviderHealthQuery.tsindex a35b1a74f..f62992be9 100644--- a/frontend/app/api/queries/useProviderHealthQuery.ts+++ b/frontend/app/api/queries/useProviderHealthQuery.ts@@ -14,6 +14,20 @@ export interface ProviderHealthDetails {endpoint?: string | null;}+/**+ * Something degraded that is not a provider failure, reported beside the+ * verdict rather than in it — e.g. `stale_embedding_space`: documents indexed+ * with a model the provider no longer serves. The fix is in the corpus, not in+ * provider settings.+ */+export interface ProviderHealthWarning {+ code: string;+ provider: string;+ message: string;+ models?: string[];+ served?: string[];+}+export interface ProviderHealthResponse {status: "healthy" | "unhealthy" | "error" | "backend-unavailable";message: string;@@ -23,6 +37,7 @@ export interface ProviderHealthResponse {llm_error?: string | null;embedding_error?: string | null;details?: ProviderHealthDetails;+ warnings?: ProviderHealthWarning[];}export interface ProviderHealthParams {@@ -97,6 +112,7 @@ export const useProviderHealthQuery = (llm_error: errorData.llm_error,embedding_error: errorData.embedding_error,details: errorData.details,+ warnings: errorData.warnings,};} else {const errorData = await response.json().catch(() => ({}));diff --git a/frontend/components/layout-wrapper.tsx b/frontend/components/layout-wrapper.tsxindex f07cf2468..6ec84ba9e 100644--- a/frontend/components/layout-wrapper.tsx+++ b/frontend/components/layout-wrapper.tsx@@ -72,7 +72,8 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {});const { isUnhealthy: isDoclingUnhealthy } = useDoclingHealth();- const { isUnhealthy: isProviderUnhealthy } = useProviderHealth();+ const { isUnhealthy: isProviderUnhealthy, isDegraded: isProviderDegraded } =+ useProviderHealth();if (isAuthPage) {return <div className="h-full">{children}</div>;@@ -120,7 +121,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {{settings?.edited && isOnboardingComplete && (<AnimatedConditionalvertical- isOpen={isProviderUnhealthy}+ isOpen={isProviderUnhealthy || isProviderDegraded}className="w-full"><ProviderHealthBanner />diff --git a/frontend/components/provider-health-banner.test.tsx b/frontend/components/provider-health-banner.test.tsxindex a41f5a5cf..3055423d2 100644--- a/frontend/components/provider-health-banner.test.tsx+++ b/frontend/components/provider-health-banner.test.tsx@@ -67,4 +67,60 @@ describe("ProviderHealthBanner", () => {screen.getByText(/OpenAI error - Invalid API Key/i),).toBeInTheDocument();});++ const staleSpace = {+ code: "stale_embedding_space",+ provider: "rhoai",+ message:+ "Documents are indexed with 'old-model', which this endpoint no longer serves",+ };++ function mockHealth(data: object) {+ vi.mocked(useProviderHealthQuery).mockReturnValue({+ data,+ isLoading: false,+ isFetching: false,+ error: null,+ isError: false,+ } as unknown as ReturnType<typeof useProviderHealthQuery>);+ }++ it("renders a healthy provider with warnings as a warning that points at Knowledge", async () => {+ const user = userEvent.setup();+ mockPush.mockClear();+ mockHealth({ status: "healthy", message: "ok", warnings: [staleSpace] });++ render(<ProviderHealthBanner />);++ expect(+ screen.getByText(/warning - Documents are indexed with 'old-model'/i),+ ).toBeInTheDocument();+ expect(screen.queryByRole("button", { name: "Fix Setup" })).toBeNull();+ await user.click(screen.getByRole("button", { name: "Open Knowledge" }));+ expect(mockPush).toHaveBeenCalledWith("/knowledge");+ });++ it("lets an unhealthy verdict outrank a warning", () => {+ mockHealth({+ status: "unhealthy",+ provider: "openai",+ llm_error: "Invalid API Key",+ warnings: [staleSpace],+ });++ render(<ProviderHealthBanner />);++ expect(+ screen.getByText(/OpenAI error - Invalid API Key/i),+ ).toBeInTheDocument();+ expect(screen.queryByRole("button", { name: "Open Knowledge" })).toBeNull();+ });++ it("renders nothing for a healthy provider without warnings", () => {+ mockHealth({ status: "healthy", message: "ok", warnings: [] });++ const { container } = render(<ProviderHealthBanner />);++ expect(container).toBeEmptyDOMElement();+ });});diff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsxindex f026ec9b0..85e9772e4 100644--- a/frontend/components/provider-health-banner.tsx+++ b/frontend/components/provider-health-banner.tsx@@ -36,6 +36,9 @@ export function useProviderHealth() {health?.status === "unhealthy" || health?.status === "error";const isBackendUnavailable =health?.status === "backend-unavailable" || isError;+ // Serving, but something outside provider setup needs attention. An+ // unhealthy verdict outranks it, so the two banners never compete.+ const isDegraded = isHealthy && (health?.warnings?.length ?? 0) > 0;return {health,@@ -45,20 +48,61 @@ export function useProviderHealth() {isError,isHealthy,isUnhealthy,+ isDegraded,isBackendUnavailable,};}export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {- const { isLoading, isHealthy, isUnhealthy, health } = useProviderHealth();+ const { isLoading, isHealthy, isUnhealthy, isDegraded, health } =+ useProviderHealth();const router = useRouter();const isNarrow = useNarrowLayout();- // Only show banner when provider is unhealthy (not when backend is unavailable)- if (isLoading || isHealthy) {+ // Only show banner when provider is unhealthy or degraded (not when the+ // backend is unavailable)+ if (isLoading || (isHealthy && !isDegraded)) {return null;}+ if (isDegraded) {+ // A warning, not an error: the provider is serving and its setup is fine.+ // The remedy (re-ingest or delete) lives on the Knowledge page, so that is+ // where the action goes rather than to provider settings.+ const [warning] = health?.warnings ?? [];+ const providerTitle = getProviderChrome(warning.provider).name;++ return (+ <Banner+ className={cn(+ "bg-amber-50 dark:bg-amber-950 text-foreground border-accent-amber border-b w-full",+ isNarrow && "flex-wrap gap-y-1 py-2",+ className,+ )}+ >+ <BannerIcon+ className="text-accent-amber-foreground shrink-0"+ icon={AlertTriangle}+ />+ <BannerTitle+ className={cn(+ "font-medium flex items-center gap-2",+ isNarrow && "text-xs",+ )}+ >+ {`${providerTitle} warning - ${warning.message}`}+ </BannerTitle>+ <Button+ size="sm"+ className="shrink-0"+ onClick={() => router.push("/knowledge")}+ >+ Open Knowledge+ </Button>+ </Banner>+ );+ }+if (isUnhealthy) {const llmProvider = health?.llm_provider || health?.provider;const embeddingProvider = health?.embedding_provider;diff --git a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md b/kubernetes/operator/internal/controller/ENV_OVERRIDE.mdindex 359345b38..c9e699645 100644--- a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md+++ b/kubernetes/operator/internal/controller/ENV_OVERRIDE.md@@ -40,7 +40,7 @@ Environment variables are merged using the following priority (highest to lowest│ Priority Level 1 (Lowest): Hardcoded Defaults ││ ┌─────────────────────────────────────────────────────────┐ ││ │ env.go DefaultLangflowEnvVars: │ │-│ │ "LANGFLOW_LOG_LEVEL": "DEBUG" │ │+│ │ "LANGFLOW_LOG_LEVEL": "INFO" │ ││ │ "LANGFLOW_WORKERS": "4" │ ││ │ "LANGFLOW_AUTO_LOGIN": "true" │ ││ └─────────────────────────────────────────────────────────┘ │diff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.goindex f75cf5b8f..1f2575f32 100644--- a/kubernetes/operator/internal/controller/env.go+++ b/kubernetes/operator/internal/controller/env.go@@ -45,7 +45,7 @@ func NewEnvVarManager() *EnvVarManager {// Langflow runtime configuration"LANGFLOW_WORKERS": "4","LANGFLOW_CONFIG_DIR": "/tmp",- "LANGFLOW_LOG_LEVEL": "DEBUG",+ "LANGFLOW_LOG_LEVEL": "INFO","HIDE_GETTING_STARTED_PROGRESS": "true","LANGFLOW_ALEMBIC_LOG_TO_STDOUT": "true","LANGFLOW_DEACTIVATE_TRACING": "true",diff --git a/src/api/docling.py b/src/api/docling.pyindex 40c860b6a..dd405114f 100644--- a/src/api/docling.py+++ b/src/api/docling.py@@ -21,6 +21,10 @@ logger = get_logger(__name__)DOCLING_SERVICE_URL = DOCLING_SERVE_URLHOST_IP = DOCLING_HOST_IP+# Per-phase httpx budget, matching the console status check. A single-worker+# docling-serve converting documents can stall its HTTP endpoints for seconds.+_HEALTH_CHECK_TIMEOUT_S = 5.0+async def health(request: Request, user: Annotated[User | None, Depends(get_optional_user)] = None@@ -37,7 +41,9 @@ async def health(try:async with httpx.AsyncClient(verify=DOCLING_SERVE_VERIFY_SSL) as client:- response = await client.get(health_url, headers=headers, timeout=2.0)+ response = await client.get(+ health_url, headers=headers, timeout=_HEALTH_CHECK_TIMEOUT_S+ )if response.status_code == 200:return JSONResponse({"status": "healthy", "host": HOST_IP})@@ -54,12 +60,29 @@ async def health(status_code=503,)- except httpx.TimeoutException:- logger.warning("Docling health check timeout", url=health_url)+ except httpx.ConnectTimeout:+ logger.warning("Docling health check connect timeout", url=health_url)return JSONResponse({"status": "unhealthy", "message": "Connection timeout", "host": HOST_IP},status_code=503,)+ except httpx.TimeoutException as e:+ # The connection opened but the reply was slow: docling-serve is busy, not+ # stopped, and ingest still works. Answer 200 so the UI doesn't announce+ # that the service is stopped; the body says it is degraded.+ logger.warning(+ "Docling health check slow to respond",+ url=health_url,+ error_type=type(e).__name__,+ timeout_s=_HEALTH_CHECK_TIMEOUT_S,+ )+ return JSONResponse(+ {+ "status": "degraded",+ "message": "Docling Serve is slow to respond",+ "host": HOST_IP,+ }+ )except Exception as e:logger.error("Docling health check failed", url=health_url, error=str(e))return JSONResponse(diff --git a/src/api/provider_health.py b/src/api/provider_health.pyindex 579e989bd..5497bf151 100644--- a/src/api/provider_health.py+++ b/src/api/provider_health.py@@ -1,12 +1,17 @@"""Provider health check endpoint."""import asyncio+from typing import Anyimport httpxfrom fastapi import Dependsfrom fastapi.responses import JSONResponse-from api.provider_validation import sanitize_provider_error_content, validate_provider_setup+from api.provider_validation import (+ ProbeResult,+ sanitize_provider_error_content,+ validate_provider_setup,+)from config.settings import get_openrag_configfrom dependencies import require_permissionfrom services import provider_error_log@@ -17,6 +22,202 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)+#: `warnings[].code` for indexed vectors whose model the provider no longer serves.+STALE_EMBEDDING_SPACE = "stale_embedding_space"+++#: Ceiling on re-listing a provider's models from inside a health check. The+#: listing is only an input to a diagnostic, so a slow cluster costs this at+#: most; the probes report an unreachable one in their own words.+_MODEL_REFRESH_TIMEOUT_SECONDS = 5.0+++async def _refresh_live_models(provider: str) -> None:+ """Re-list what `provider` serves, if it can say, best effort.++ Only the provider being checked: the listing of any other would not be+ read, and waiting on its cluster would only slow this one's verdict. A+ provider with no enhancement, or one that cannot list its own models, has+ nothing to refresh, and is skipped without a call.++ TTL-guarded inside each enhancement, so this is a network call once every+ few minutes rather than once per poll — while the cluster answers. A failed+ listing is not cached, so an unreachable cluster is asked again on every+ poll; hence the timeout. A failure or timeout leaves the previous answer in+ place and must never fail the health check.+ """+ try:+ from enhancements.providers.registry import get as get_enhancement+ from services.model_catalog import refresh_live_models++ enhancement = get_enhancement(provider)+ if enhancement is None or not hasattr(enhancement, "fetch_models"):+ return+ await asyncio.wait_for(+ refresh_live_models(provider), timeout=_MODEL_REFRESH_TIMEOUT_SECONDS+ )+ except TimeoutError:+ logger.debug(+ "Timed out refreshing live model listing",+ provider=provider,+ timeout_seconds=_MODEL_REFRESH_TIMEOUT_SECONDS,+ )+ except Exception:+ logger.debug("Could not refresh live model listings", provider=provider, exc_info=True)+++#: Spaces per aggregation page — the same page size retrieval discovers them+#: with. A corpus has a handful: one per embedding model it was indexed with.+_EMBEDDING_SPACE_PAGE_SIZE = 100++#: Pages read before giving up. Far past any real corpus; it bounds what one+#: health poll can cost if the index is somehow full of distinct spaces.+_EMBEDDING_SPACE_MAX_PAGES = 10+++async def _indexed_spaces(provider: str) -> tuple[str, ...] | None:+ """Embedding models this corpus holds vectors for, under `provider`.++ None means unknown — OpenSearch unreachable, nothing indexed, or an answer+ that may be incomplete. A partial list is never returned as if it were the+ whole: a space missing from it is a stale space that goes unreported. So+ every page of the composite aggregation is read, shard failures raise+ rather than quietly dropping buckets, and running out of pages before the+ cursor does is treated as not knowing.++ Read with the admin client, not a user-scoped one, because this is a+ property of the corpus rather than of whoever is looking; DLS would+ silently shrink it.+ """+ try:+ from config.settings import clients, get_index_name+ from utils.embedding_fields import (+ build_embedding_space_aggregation,+ embedding_space_after_keys,+ embedding_spaces_from_aggregation,+ split_embedding_space_id,+ )++ client = clients.opensearch+ if client is None:+ return None+ space_ids: list[str] = []+ after: dict[str, Any] | None = None+ for _ in range(_EMBEDDING_SPACE_MAX_PAGES):+ result = await client.search(+ index=get_index_name(),+ body={+ "size": 0,+ # Qualified spaces only. A legacy space records no provider,+ # so it cannot be attributed to this one, and it is governed+ # by OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP rather than by what+ # the cluster serves today.+ "aggs": build_embedding_space_aggregation(+ size=_EMBEDDING_SPACE_PAGE_SIZE,+ qualified_after=after,+ include_legacy=False,+ ),+ },+ # A failed shard must fail the read, not shrink its buckets.+ params={"allow_partial_search_results": "false"},+ )+ if result.get("timed_out"):+ logger.debug("Reading indexed embedding spaces timed out")+ return None+ space_ids.extend(space.space_id for space in embedding_spaces_from_aggregation(result))+ # Raw buckets, not parsed spaces: the parser skips blank ids, and a+ # page it thinned out is not a short one.+ buckets = (+ result.get("aggregations", {}).get("embedding_spaces", {}).get("buckets") or []+ )+ next_after, _ = embedding_space_after_keys(result)+ # A short page is the last one; a composite aggregation still hands+ # back an `after_key` for it, and following it costs an empty read.+ if len(buckets) < _EMBEDDING_SPACE_PAGE_SIZE or not next_after or next_after == after:+ break+ after = next_after+ else:+ logger.debug(+ "Indexed embedding spaces exceed the read limit",+ pages=_EMBEDDING_SPACE_MAX_PAGES,+ page_size=_EMBEDDING_SPACE_PAGE_SIZE,+ )+ return None+ except Exception:+ logger.debug("Could not read indexed embedding spaces", exc_info=True)+ return None++ key = (provider or "").strip().lower()+ models = []+ for space_id in space_ids:+ space_provider, model = split_embedding_space_id(space_id)+ if space_provider == key and model:+ models.append(model)+ return tuple(dict.fromkeys(models)) or None+++async def _stale_embedding_spaces(provider: str) -> dict[str, Any] | None:+ """Indexed vector spaces whose model the provider has stopped serving.++ Nothing re-checks a corpus once it is indexed. A chunk records the space it+ was embedded in (`provider:model`), retrieval embeds the query once per+ space it finds there, and an `InferenceService` redeployed under a new+ `--served-model-name` leaves every earlier chunk pointing at a model that+ is gone. No save can be rejected over it and no probe reaches it — the+ *configured* model is fine. Retrieval skips a space it cannot embed for and+ carries on, so nothing fails outright: those documents quietly stop being+ found by meaning, and the only trace is a 502 quoting a model name that+ appears nowhere in Settings.++ A warning, never an error: the provider is serving and nothing in its setup+ is wrong. The remedy is in the corpus — re-ingest or delete — so reporting+ it as a provider failure would send the operator to the one screen that+ cannot fix it.++ Silent unless both halves are known: what the provider serves, and what the+ corpus holds. Neither absence is evidence of the other. "Known" is+ `served is not None`: an empty listing is a provider that answered and+ serves no embedding model, which makes every indexed space stale.+ """+ try:+ from enhancements.providers.registry import live_models_for++ served = live_models_for(provider, "embedding")+ except Exception:+ logger.debug("Could not read live models for %s", provider, exc_info=True)+ return None+ if served is None:+ return None++ indexed = await _indexed_spaces(provider)+ if not indexed:+ return None+ stale = [model for model in indexed if model not in served]+ if not stale:+ return None+ serving = f"it serves: {', '.join(served)}" if served else "it lists no embedding models at all"+ # Debug, not warning: the banner polls this every 30s per open tab, and the+ # condition lasts until someone re-ingests. The response carries it to the+ # user; the log is for tracing why.+ logger.debug(+ "Indexed embedding spaces are no longer served by the provider",+ provider=provider,+ stale=stale,+ served=list(served),+ )+ return {+ "code": STALE_EMBEDDING_SPACE,+ "provider": provider,+ "models": stale,+ "served": list(served),+ "message": (+ f"Documents are indexed with {', '.join(repr(model) for model in stale)}, which "+ f"this endpoint no longer serves ({serving}). Search skips "+ "those vectors, so the documents are found by keyword only until they are "+ "re-ingested or deleted."+ ),+ }+async def check_provider_health(provider: str | None = None,@@ -255,7 +456,7 @@ async def check_provider_health(# Validate LLM providertry:- await validate_provider_setup(+ llm_probe: ProbeResult = await validate_provider_setup(provider=provider,api_key=api_key,llm_model=llm_model,@@ -276,6 +477,23 @@ async def check_provider_health(except Exception as e:llm_error = sanitize_provider_error_content(e)logger.error(f"LLM provider ({provider}) validation failed: {llm_error}")+ else:+ # Without this the banner can only be cleared by chat traffic:+ # one failed turn latches it, the frontend then polls every 5s+ # with `test_completion` while it stays latched, and a provider+ # that has recovered keeps being reported broken until the entry+ # goes stale 15 minutes later.+ #+ # But a probe cannot reproduce the request that failed (see+ # `provider_error_log`), so only one that sent the same shape+ # of request may speak for it. A passing validation can mean a+ # deployment listing (Azure), no call at all (no model set), or+ # a tool-less completion (the LiteLLM probe) — none of which+ # reaches the tool-calling failures agent traffic actually hits.+ # Clearing on those would drop the banner while chat is still+ # broken, only for the next turn to raise it again.+ if test_completion and llm_probe.model_probed and llm_probe.tools_exercised:+ provider_error_log.record_success(provider, "chat")# Validate embedding provider# For WatsonX with test_completion=True, wait 2 seconds between completion and embedding tests@@ -291,7 +509,7 @@ async def check_provider_health(await asyncio.sleep(2)try:- await validate_provider_setup(+ embedding_probe: ProbeResult = await validate_provider_setup(provider=embedding_provider,api_key=embedding_api_key,embedding_model=embedding_model,@@ -316,6 +534,12 @@ async def check_provider_health(logger.error(f"Embedding provider ({embedding_provider}) validation failed: {embedding_error}")+ else:+ # An embedding request has no shape beyond model and input, so a+ # real call to the configured model is the same request traffic+ # makes. Anything short of one proves nothing about it.+ if test_completion and embedding_probe.model_probed:+ provider_error_log.record_success(embedding_provider, "embedding")# A real call beats a probe. The probe sends its own request, so it# hits its own failure: OpenAI checks request shape before billing,@@ -330,6 +554,14 @@ async def check_provider_health(or embedding_error)+ # Nothing above looks at the corpus, and a vector space outlives+ # the model that made it. Reported beside the verdict, never in it:+ # a stale space degrades search but says nothing about whether the+ # provider is serving, so it must not turn the status code.+ await _refresh_live_models(embedding_provider)+ stale_spaces = await _stale_embedding_spaces(embedding_provider)+ warnings = [stale_spaces] if stale_spaces else []+# Return combined statusif llm_error or embedding_error:errors = []@@ -349,6 +581,7 @@ async def check_provider_health("embedding_provider": embedding_provider,"llm_error": llm_error,"embedding_error": embedding_error,+ "warnings": warnings,},status_code=503,)@@ -362,6 +595,7 @@ async def check_provider_health("llm_model": llm_model,"embedding_model": embedding_model,},+ "warnings": warnings,}provider_health_cache.set_and_release(health_cache_key, healthy_payload)_health_leader_key = Nonediff --git a/src/api/provider_validation.py b/src/api/provider_validation.pyindex c9afe1fc1..dce10af92 100644--- a/src/api/provider_validation.py+++ b/src/api/provider_validation.py@@ -4,6 +4,7 @@ import asyncioimport jsonimport refrom collections.abc import Mapping+from dataclasses import dataclassfrom typing import Anyfrom urllib.parse import urlparse@@ -155,6 +156,87 @@ def is_provider_tls_error(text: str | BaseException | None) -> bool:return any(marker in lowered for marker in _PROVIDER_TLS_ERROR_MARKERS)+#: Markers for a response the connection dropped before it finished: the server+#: promised more body than arrived. Nearly always something between OpenRAG and+#: the model server closing a long response — an authenticating proxy with a+#: fixed request timeout (RHOAI's kube-rbac-proxy cuts at 30s), a load balancer+#: idle timeout. aiohttp names it with a "400" of its own, which reads like the+#: model rejecting the request.+_PROVIDER_TRUNCATED_RESPONSE_MARKERS = (+ "transferencodingerror",+ "response payload is not completed",+ "not enough data to satisfy transfer length",+ "incomplete chunked read",+)+++def is_provider_truncated_response_error(text: str | BaseException | None) -> bool:+ """True when the provider's response was cut off mid-transfer."""+ if text is None:+ return False+ lowered = (str(text) if not isinstance(text, str) else text).lower()+ return any(marker in lowered for marker in _PROVIDER_TRUNCATED_RESPONSE_MARKERS)+++#: How deep to follow `__cause__` / `__context__`. LiteLLM wraps the transport+#: error three times (its exception, the provider's, `httpx.ReadError`).+_MAX_EXCEPTION_CHAIN_DEPTH = 8+++def is_provider_stale_connection_error(exc: BaseException | None) -> bool:+ """True when the connection closed before any response to the request began.++ Nearly always a pooled keep-alive connection the model server had already+ closed: LiteLLM keeps idle connections for 120s, vLLM (uvicorn) closes them+ after 5s, and a request sent while the close is still in flight finds a+ dead socket. The server never saw the request, so sending it again on a+ fresh connection is safe. Matched on the transport's own exception type,+ not its text: a response cut off mid-body raises a different error+ (`is_provider_truncated_response_error`) and must not be replayed.+ """+ disconnect_types: tuple[type[BaseException], ...] = ()+ try:+ from aiohttp import ServerDisconnectedError+ except ImportError: # pragma: no cover - aiohttp ships with LiteLLM+ pass+ else:+ disconnect_types = (ServerDisconnectedError,)++ current = exc+ for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):+ if current is None:+ return False+ if isinstance(current, disconnect_types):+ return True+ # The same failure on LiteLLM's httpx transport (a provider handed its+ # own client). Other RemoteProtocolErrors can follow a partial response.+ if isinstance(current, httpx.RemoteProtocolError) and (+ "without sending a response" in str(current).lower()+ ):+ return True+ current = current.__cause__ or current.__context__+ return False+++#: Markers for a prompt that does not fit the model's context window. That is a+#: property of one request — a long conversation, a lot of retrieved text — and+#: says nothing about whether the provider is serving.+_CONTEXT_WINDOW_ERROR_MARKERS = (+ "maximum context length",+ "context_length_exceeded",+ "contextwindowexceeded",+ "prompt is too long",+)+++def is_context_window_error(text: str | BaseException | None) -> bool:+ """True when the call failed because the prompt exceeds the model's context window."""+ if text is None:+ return False+ lowered = (str(text) if not isinstance(text, str) else text).lower()+ return any(marker in lowered for marker in _CONTEXT_WINDOW_ERROR_MARKERS)++_GENERIC_UPSTREAM_ERROR_MARKERS = ("an unknown error occurred","an error occurred while generating a response",@@ -662,6 +744,26 @@ def is_azure_ai_foundry_endpoint(api_base: str | None) -> bool:return (urlparse(api_base).hostname or "").endswith(".services.ai.azure.com")+@dataclass(frozen=True)+class ProbeResult:+ """What a successful `validate_provider_setup()` call actually exercised.++ Passing validation is weaker evidence than it looks: depending on the+ provider and arguments it may have listed deployments, checked a key, or+ made no call at all. Callers deciding whether a provider is *serving* — as+ the health check does before erasing a real-traffic failure — need to know+ which.++ ``model_probed``: a real call was made to the model under test — the+ embedding model when one was given, otherwise the LLM.+ ``tools_exercised``: that call was a completion carrying tool definitions,+ the shape the agent's own traffic has.+ """++ model_probed: bool = False+ tools_exercised: bool = False++async def validate_provider_setup(provider: str,api_key: str = None,@@ -672,7 +774,8 @@ async def validate_provider_setup(test_completion: bool = False,credentials: dict[str, str] | None = None,stored_credentials: Mapping[str, Any] | None = None,-) -> None:+ verify_model: bool = False,+) -> ProbeResult:"""Validate provider setup by testing completion with tool calling and embedding.@@ -690,6 +793,15 @@ async def validate_provider_setup(Only a provider enhancement's lightweight check reads it: a provider withseparate chat and embedding endpoints has ``credentials`` narrowed to oneof them, and the check has to see both to probe both.+ verify_model: Call the given model through LiteLLM even without ``test_completion``.+ The settings save passes it: for a provider enhancement that real call+ is the only check that rejects a model the cluster does not serve+ before it is stored. Without it, and without ``test_completion``, an+ enhancement gets its model-free lightweight check like any other+ provider.++ Returns:+ ProbeResult: what the successful validation exercised.Raises:Exception: If validation fails, raises the original exception with the actual error message.@@ -708,17 +820,21 @@ async def validate_provider_setup(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})")- # watsonx.ai on-prem has no bespoke model probe of its own: a real call- # through LiteLLM *is* its model probe, and it is what makes switching to- # a model the cluster does not serve fail instead of saving silently. Its- # catalogue check is only for the case there is no model to probe with,- # which is the one that used to report "A model is required".+ # A provider enhancement has no bespoke model probe of its own: a real+ # call through LiteLLM *is* its model probe, and it is what makes+ # switching to a model the cluster does not serve fail instead of saving+ # silently. It runs only when asked for — on save (`verify_model`) or a+ # full check (`test_completion`). The polled health check asks for+ # neither and gets the model-free lightweight check, as every other+ # provider does: polling inference every 30s holds a GPU slot (RHOAI) or+ # bills (watsonx.ai on-prem), and its pooled connections race the+ # model server's keep-alive and fail with "Server disconnected".enhancement = get_provider_enhancement(provider_lower)azure_ai_foundry_endpoint = provider_lower == "azure" and is_azure_ai_foundry_endpoint(supplied.get("api_base"))probes_the_model = (- bool(embedding_model or llm_model)+ bool(embedding_model or llm_model) and (test_completion or verify_model)if enhancement is not Noneelse (provider_lower not in _NATIVELY_VALIDATED_PROVIDERS@@ -742,6 +858,10 @@ async def validate_provider_setup(embedding_model=embedding_model,llm_model=llm_model,)+ # The LiteLLM probe is a plain completion: it proves the model+ # answers, not that it can take the tool-calling request the agent+ # sends.+ result = ProbeResult(model_probed=True)elif test_completion:if provider_lower == "azure":# Azure deployments are user-defined, so a model completion is@@ -755,7 +875,7 @@ async def validate_provider_setup(credentials=supplied,stored_credentials=stored_credentials,)- return+ return ProbeResult()# Full validation with completion/embedding tests (consumes credits)if embedding_model:# Test embedding@@ -766,6 +886,7 @@ async def validate_provider_setup(endpoint=endpoint,project_id=project_id,)+ result = ProbeResult(model_probed=True)elif llm_model:# Test completion with tool callingawait test_completion_with_tools(@@ -775,6 +896,10 @@ async def validate_provider_setup(endpoint=endpoint,project_id=project_id,)+ result = ProbeResult(model_probed=True, tools_exercised=True)+ else:+ # No model to test with, so nothing was called at all.+ result = ProbeResult()else:# Lightweight validation (no credits consumed)await test_lightweight_health(@@ -785,8 +910,10 @@ async def validate_provider_setup(credentials=supplied,stored_credentials=stored_credentials,)+ result = ProbeResult()logger.info(f"Validation successful for provider: {provider_lower}")+ return resultexcept Exception as e:logger.error(f"Validation failed for provider {provider_lower}: {str(e)}")@@ -805,6 +932,7 @@ async def _test_litellm_provider("""Validate arbitrary providers through the same LiteLLM adapter used at runtime."""import litellm+ from services.llm_gateway import with_stale_connection_retryfrom services.model_catalog import litellm_provider_keymodel = embedding_model or llm_model@@ -813,26 +941,38 @@ async def _test_litellm_provider(# Same aliasing the gateway applies, so the probe hits the route the real# call will: `watsonx_onprem/<model>` is not a prefix LiteLLM can resolve.litellm_model = f"{litellm_provider_key(provider)}/{model}"+ # And the same retry: a dead pooled connection is not a provider failure,+ # and reporting one would block a save or raise the banner for nothing.if embedding_model:- await litellm.aembedding(+ await with_stale_connection_retry(+ lambda: litellm.aembedding(+ model=litellm_model,+ # A list, never a bare string. OpenAI accepts either and LiteLLM+ # forwards whatever it is given, so a string reaches watsonx.ai as+ # `"inputs": "..."` where it requires `[]string` — the cluster then+ # answers "Mismatch type []string with value string" and the+ # pre-save check fails for every embedding model, which leaves a+ # wrongly-chosen one impossible to change.+ input=["OpenRAG provider validation"],+ **credentials,+ **(runtime_kwargs or {}),+ ),+ provider=provider,model=litellm_model,- # A list, never a bare string. OpenAI accepts either and LiteLLM- # forwards whatever it is given, so a string reaches watsonx.ai as- # `"inputs": "..."` where it requires `[]string` — the cluster then- # answers "Mismatch type []string with value string" and the- # pre-save check fails for every embedding model, which leaves a- # wrongly-chosen one impossible to change.- input=["OpenRAG provider validation"],- **credentials,- **(runtime_kwargs or {}),+ kind="embedding",)return- await litellm.acompletion(+ await with_stale_connection_retry(+ lambda: litellm.acompletion(+ model=litellm_model,+ messages=[{"role": "user", "content": "Reply with OK."}],+ max_tokens=4,+ **credentials,+ **(runtime_kwargs or {}),+ ),+ provider=provider,model=litellm_model,- messages=[{"role": "user", "content": "Reply with OK."}],- max_tokens=4,- **credentials,- **(runtime_kwargs or {}),+ kind="chat",)diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.pyindex 2cd99dc3f..8f732d733 100644--- a/src/api/settings/endpoints.py+++ b/src/api/settings/endpoints.py@@ -101,6 +101,7 @@ from dependencies import (get_task_service,require_permission,)+from services import provider_error_logfrom services.docling_service import DoclingConfig, get_docling_preset_configsfrom services.model_catalog import secret_field_keysfrom services.rbac_service import is_rbac_enforced@@ -574,6 +575,9 @@ async def update_settings(submitted_credentials.get(llm_provider_key, {}),remove=removals_by_provider.get(llm_provider_key, set()),),+ # A model the cluster does not serve must fail the+ # save, not the first chat.+ verify_model=True,)logger.info(f"LLM provider validation successful for {llm_provider}")@@ -632,6 +636,7 @@ async def update_settings(submitted_credentials.get(embedding_provider_key, {}),remove=removals_by_provider.get(embedding_provider_key, set()),),+ verify_model=True,)logger.info(f"Embedding provider validation successful for {embedding_provider}"@@ -1111,6 +1116,10 @@ async def update_settings(return JSONResponse({"error": "Failed to save configuration"}, status_code=500)provider_health_cache.invalidate()+ if should_validate or provider_updated:+ # A recorded failure describes the provider setup that just changed.+ # Real traffic re-raises it if the new setup fails the same way.+ provider_error_log.clear()# Refresh patched client immediately so subsequent requests pick up latest config.await clients.refresh_patched_client()@@ -1609,6 +1618,7 @@ async def onboarding(if config_manager.save_config_file(current_config):provider_health_cache.invalidate()+ provider_error_log.clear()set_fields = [k for k, v in body.model_dump(exclude_unset=True).items()]logger.info("Onboarding configuration updated successfully",diff --git a/src/api/settings/langflow_sync.py b/src/api/settings/langflow_sync.pyindex 91dec1b29..775787381 100644--- a/src/api/settings/langflow_sync.py+++ b/src/api/settings/langflow_sync.py@@ -226,7 +226,10 @@ async def ensure_required_langflow_global_variables(config=None):if sync_value:patch_payload["value"] = target_valpatch_resp = await clients.langflow_request(- "PATCH", f"/api/v1/variables/{var_id}", json=patch_payload+ "PATCH",+ f"/api/v1/variables/{var_id}",+ json=patch_payload,+ idempotent=True,)if not (200 <= patch_resp.status_code < 300):raise RuntimeError(diff --git a/src/api/v1/llm.py b/src/api/v1/llm.pyindex 8a1331a8c..3b1c56ed1 100644--- a/src/api/v1/llm.py+++ b/src/api/v1/llm.py@@ -10,6 +10,7 @@ from fastapi import Depends, Requestfrom fastapi.responses import JSONResponse, StreamingResponsefrom dependencies import require_llm_proxy_any_permission, require_llm_proxy_permission+from services.langflow_llm_token_service import HOP_PURPOSE_CHATfrom services.llm_gateway import LlmGatewayError, chat_completions, embeddingsfrom services.model_catalog import (CATALOG_UNAVAILABLE_MESSAGE,@@ -138,8 +139,11 @@ async def embeddings_endpoint(body = await _read_json_body(request)if isinstance(body, JSONResponse):return body+ # A chat run's hop token marks query embeddings, which must not queue+ # behind a bulk ingest at a concurrency-limited provider.+ interactive = getattr(request.state, "llm_hop_purpose", None) == HOP_PURPOSE_CHATtry:- result = await embeddings(body)+ result = await embeddings(body, interactive=interactive)except LlmGatewayError as exc:return _gateway_error(exc)return JSONResponse(result)diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 3259a475b..61a2ab923 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -783,7 +783,10 @@ class ConfigManager:# user-selected model preference. It must remain overridable after the# settings file is marked edited so existing installations can resolve# legacy vector spaces without modifying persisted application state.- from config.settings import get_legacy_embedding_provider_map_json+ from config.settings import (+ get_legacy_embedding_provider_map_json,+ get_rhoai_env_credentials,+ )legacy_provider_map_json = get_legacy_embedding_provider_map_json()if legacy_provider_map_json:@@ -890,21 +893,15 @@ class ConfigManager:# lets a Helm (`llmProviders.rhoai.*`) or operator (`spec.rhoai`) install# come up configured with no human clicking through Settings, which is# the point on an air-gapped cluster.- rhoai_endpoint = os.getenv("RHOAI_ENDPOINT")- rhoai_embeddings_endpoint = os.getenv("RHOAI_EMBEDDINGS_ENDPOINT")- rhoai_api_key = os.getenv("RHOAI_API_KEY")- rhoai_tls_verify = os.getenv("RHOAI_TLS_VERIFY")- if rhoai_endpoint or rhoai_embeddings_endpoint or rhoai_api_key or rhoai_tls_verify:+ rhoai_env = get_rhoai_env_credentials()+ if any(rhoai_env.values()):+ # Field names only: `api_key` is a bearer token.+ logger.debug(+ "Seeding the rhoai provider from environment variables",+ fields=sorted(name for name, value in rhoai_env.items() if value),+ )self._seed_custom_provider_credentials(- config_data,- "rhoai",- {- "api_base": rhoai_endpoint,- "embedding_api_base": rhoai_embeddings_endpoint,- "api_key": rhoai_api_key,- "ssl_verify": rhoai_tls_verify,- },- required=("api_base", "api_key"),+ config_data, "rhoai", rhoai_env, required=("api_base", "api_key"))# Knowledge settingsdiff --git a/src/config/model_providers.yaml b/src/config/model_providers.yamlindex 51580e012..6d7e760d1 100644--- a/src/config/model_providers.yaml+++ b/src/config/model_providers.yaml@@ -140,7 +140,7 @@ providers:# These are `--served-model-name` values, not image tags: whatever the# InferenceService was started with is the id that has to be sent.models:- - granite-3.3-2b-instruct+ - gpt-oss-120bembedding_models:- granite-embedding-english-r2diff --git a/src/config/settings.py b/src/config/settings.pyindex 19cb8054f..e5c39494b 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -33,6 +33,23 @@ def get_legacy_embedding_provider_map_json() -> str | None:return os.getenv("OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP")+def get_rhoai_env_credentials() -> dict[str, str | None]:+ """Return the raw ``RHOAI_*`` first-boot seed values, keyed by credential field.++ Seeds the ``rhoai`` provider enhancement (see+ ``enhancements/providers/redhat/openshift_ai.py``). Values are passed through+ unvalidated: the enhancement owns parsing, and an invalid+ ``embedding_max_concurrency`` falls back to its default with a warning.+ """+ return {+ "api_base": os.getenv("RHOAI_ENDPOINT"),+ "embedding_api_base": os.getenv("RHOAI_EMBEDDINGS_ENDPOINT"),+ "api_key": os.getenv("RHOAI_API_KEY"),+ "ssl_verify": os.getenv("RHOAI_TLS_VERIFY"),+ "embedding_max_concurrency": os.getenv("RHOAI_EMBEDDING_MAX_CONCURRENCY"),+ }++def get_opensearch_index_name_override() -> str | None:"""Return the raw ``OPENSEARCH_INDEX_NAME`` env override.@@ -698,8 +715,57 @@ UPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)LANGFLOW_TIMEOUT = get_env_float("LANGFLOW_TIMEOUT", 2400.0) # 40 minutesLANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30 seconds# Retries for transient Langflow HTTP failures (disconnects, 502/503/504).+# Only idempotent requests are replayed after the server may have acted on+# them; see `_retryable_langflow_statuses`.LANGFLOW_REQUEST_RETRIES = get_env_int("LANGFLOW_REQUEST_RETRIES", 2)+#: RFC 9110 §9.2.2 idempotent methods: repeating one leaves the server in the+#: same state, so a failure after the request was sent can be replayed safely.+_IDEMPOTENT_HTTP_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})+#: Statuses retried for any method, because they arrive before the request is+#: processed: 429 is a rate-limit rejection, and a 503 from Langflow itself is+#: emitted before processing. RFC 9110 does not guarantee this for 503 in+#: general: a proxy in front of Langflow (for example Envoy's "reset before+#: headers") can send one after forwarding the request. This relies on the+#: backend reaching Langflow directly, as it does in compose and over a+#: Kubernetes Service with no service mesh.+_LANGFLOW_ALWAYS_RETRYABLE_STATUSES = frozenset({429, 503})+#: Statuses that can arrive *after* the server acted. Langflow answers 500 when+#: a flow run's graph fails, so replaying `POST /api/v1/run` re-executes the+#: whole flow (re-embedding, re-polling a Docling task that has since been+#: deleted) and the replay's error masks the original one.+_LANGFLOW_IDEMPOTENT_ONLY_STATUSES = frozenset({408, 500, 502, 504})+#: Transport errors raised before the request reached the server.+_LANGFLOW_PRE_SEND_ERRORS: tuple[type[httpx.RequestError], ...] = (+ httpx.ConnectError,+ httpx.ConnectTimeout,+ httpx.PoolTimeout,+)+++def _is_idempotent_request(method: str, idempotent: bool | None) -> bool:+ """Whether a request may be replayed after the server could have acted on it."""+ if idempotent is not None:+ return idempotent+ return (method or "").upper() in _IDEMPOTENT_HTTP_METHODS+++def _retryable_langflow_statuses(idempotent: bool) -> frozenset[int]:+ """HTTP statuses worth retrying for a request of the given idempotency."""+ if idempotent:+ return _LANGFLOW_ALWAYS_RETRYABLE_STATUSES | _LANGFLOW_IDEMPOTENT_ONLY_STATUSES+ return _LANGFLOW_ALWAYS_RETRYABLE_STATUSES+++def _is_retry_safe_transport_error(exc: httpx.RequestError, idempotent: bool) -> bool:+ """Whether a transport error may be retried without risking a double execution.++ A read timeout or dropped connection can happen after Langflow started a+ flow run; only connection-phase failures prove the request was never seen.+ """+ return idempotent or isinstance(exc, _LANGFLOW_PRE_SEND_ERRORS)++# Per-file processing timeout for document ingestion tasks (in seconds)# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete# Default: 3600 seconds (60 minutes)@@ -1479,13 +1545,32 @@ class AppClients:"""Alias for cleanup() for convenience."""await self.cleanup()- async def langflow_request(self, method: str, endpoint: str, **kwargs):+ async def langflow_request(+ self,+ method: str,+ endpoint: str,+ *,+ idempotent: bool | None = None,+ **kwargs,+ ):"""Central method for all Langflow API requests.- Retries transient transport/server errors and once with a fresh API key on- auth failures (401/403).+ Retries once with a fresh API key on auth failures (401/403), and retries+ transient failures according to whether the request is safe to repeat:++ - Any request is retried when it provably was not processed: connection+ errors before the request was sent, 429, and 503.+ - Only idempotent requests are also retried on 408/500/502/504 and on+ transport errors after sending (read timeouts, dropped connections).++ `idempotent` defaults to the method's RFC 9110 semantics (GET, HEAD,+ OPTIONS, PUT, DELETE). Pass `idempotent=True` for a POST/PATCH that sets+ absolute state and can be repeated harmlessly. Replaying a non-idempotent+ request such as `POST /api/v1/run/{flow}` re-executes the flow, and the+ replay's error hides the original failure."""- _TRANSIENT_STATUS_CODES = {408, 429, 500, 502, 503, 504}+ is_idempotent = _is_idempotent_request(method, idempotent)+ retryable_statuses = _retryable_langflow_statuses(is_idempotent)max_attempts = max(1, LANGFLOW_REQUEST_RETRIES + 1)last_error: Exception | None = Noneauth_retry_attempted = False@@ -1539,12 +1624,24 @@ class AppClients:except httpx.RequestError as exc:last_error = excif attempt + 1 < max_attempts:+ if not _is_retry_safe_transport_error(exc, is_idempotent):+ logger.warning(+ "Langflow request failed; not retrying non-idempotent request",+ method=method,+ endpoint=endpoint,+ attempt=attempt + 1,+ error_type=type(exc).__name__,+ error=str(exc),+ )+ raisedelay = min(2**attempt, 4)logger.warning("Langflow request transport error, retrying",+ method=method,endpoint=endpoint,attempt=attempt + 1,max_attempts=max_attempts,+ error_type=type(exc).__name__,error=str(exc),retry_in_seconds=delay,)@@ -1583,6 +1680,17 @@ class AppClients:except httpx.RequestError as exc:last_error = excif attempt + 1 < max_attempts:+ if not _is_retry_safe_transport_error(exc, is_idempotent):+ logger.warning(+ "Langflow auth retry failed; not retrying "+ "non-idempotent request",+ method=method,+ endpoint=endpoint,+ attempt=attempt + 1,+ error_type=type(exc).__name__,+ error=str(exc),+ )+ raisedelay = min(2**attempt, 4)logger.warning("Langflow auth retry transport error, retrying",@@ -1609,10 +1717,26 @@ class AppClients:await asyncio.sleep(delay)continue- if response.status_code in _TRANSIENT_STATUS_CODES and attempt + 1 < max_attempts:+ if (+ response.status_code in _LANGFLOW_IDEMPOTENT_ONLY_STATUSES+ and response.status_code not in retryable_statuses+ and attempt + 1 < max_attempts+ ):+ # Surface the first failure as-is: a replay would re-execute+ # the request (e.g. a whole flow run) and could mask the error.+ logger.warning(+ "Langflow request failed; not retrying non-idempotent request",+ method=method,+ endpoint=endpoint,+ status_code=response.status_code,+ attempt=attempt + 1,+ )++ if response.status_code in retryable_statuses and attempt + 1 < max_attempts:delay = min(2**attempt, 4)logger.warning("Langflow request returned transient status, retrying",+ method=method,endpoint=endpoint,status_code=response.status_code,attempt=attempt + 1,@@ -1787,7 +1911,10 @@ class AppClients:}patch_response = await self.langflow_request(- "PATCH", f"/api/v1/variables/{variable_id}", json=update_payload+ "PATCH",+ f"/api/v1/variables/{variable_id}",+ json=update_payload,+ idempotent=True,)if patch_response.status_code == 200:diff --git a/src/dependencies.py b/src/dependencies.pyindex 746bd1f68..0be0ff44f 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -197,11 +197,15 @@ async def get_llm_proxy_user(token = raw[7:].strip() if raw.lower().startswith("bearer ") else raw.strip()if token and not token.startswith("orag_"):try:- user = token_service.validate_token(token)+ claims = token_service.validate_claims(token)except ValueError:- user = None+ claims = Noneelse:+ user = token_service.user_from_claims(claims)request.state.user = user+ # Which kind of Langflow run minted it; the gateway gives chat+ # (query) embeddings their own lane beside bulk ingestion.+ request.state.llm_hop_purpose = claims.get("purpose")return userreturn await resolve_api_key_user(request, api_key_service, session_manager)diff --git a/src/services/chat_service.py b/src/services/chat_service.pyindex 6d9d020c8..dec8501cf 100644--- a/src/services/chat_service.py+++ b/src/services/chat_service.py@@ -88,6 +88,7 @@ class ChatService:# Pass the selected embedding model as a global variablefrom config.settings import get_openrag_config+ from services.langflow_llm_token_service import HOP_PURPOSE_CHATfrom utils.langflow_headers import (add_provider_credentials_to_headers,build_model_provider_headers,@@ -154,6 +155,7 @@ class ChatService:flows_service=self.flows_service,jwt_token=jwt_token,user_id=user_id,+ purpose=HOP_PURPOSE_CHAT,)# Get context variables for filters, limit, and thresholdfrom auth_context import (@@ -274,6 +276,7 @@ class ChatService:extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_tokenfrom config.settings import get_openrag_config+ from services.langflow_llm_token_service import HOP_PURPOSE_CHATfrom utils.langflow_headers import (add_provider_credentials_to_headers,build_model_provider_headers,@@ -289,6 +292,7 @@ class ChatService:flows_service=self.flows_service,jwt_token=jwt_token,user_id=user_id,+ purpose=HOP_PURPOSE_CHAT,)# Build the complete filter expression like the chat service does@@ -530,6 +534,7 @@ class ChatService:extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_tokenfrom config.settings import get_openrag_config+ from services.langflow_llm_token_service import HOP_PURPOSE_CHATfrom utils.langflow_headers import (add_provider_credentials_to_headers,build_model_provider_headers,@@ -596,6 +601,7 @@ class ChatService:flows_service=self.flows_service,jwt_token=jwt_token,user_id=user_id,+ purpose=HOP_PURPOSE_CHAT,)# Ensure the Langflow client exists; try lazy init if neededdiff --git a/src/services/flows_service.py b/src/services/flows_service.pyindex 825843643..d75976cc2 100644--- a/src/services/flows_service.py+++ b/src/services/flows_service.py@@ -518,7 +518,10 @@ class FlowsService:"""Unlock a flow before making changes."""try:response = await clients.langflow_request(- "PATCH", f"/api/v1/flows/{flow_id}", json={"locked": False}+ "PATCH",+ f"/api/v1/flows/{flow_id}",+ json={"locked": False},+ idempotent=True,)if response.status_code not in (200, 404):logger.warning(@@ -531,7 +534,10 @@ class FlowsService:"""Lock a flow after making changes."""try:response = await clients.langflow_request(- "PATCH", f"/api/v1/flows/{flow_id}", json={"locked": True}+ "PATCH",+ f"/api/v1/flows/{flow_id}",+ json={"locked": True},+ idempotent=True,)if response.status_code not in (200, 404):logger.warning(@@ -566,7 +572,10 @@ class FlowsService:# Make PATCH request to Langflow API to update the flow using shared clienttry:response = await clients.langflow_request(- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data+ "PATCH",+ f"/api/v1/flows/{flow_id}",+ json=flow_data,+ idempotent=True,)if response.status_code == 200:@@ -729,7 +738,10 @@ class FlowsService:# Update the flow via PATCH requestpatch_response = await clients.langflow_request(- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data+ "PATCH",+ f"/api/v1/flows/{flow_id}",+ json=flow_data,+ idempotent=True,)if patch_response.status_code != 200:@@ -1319,7 +1331,10 @@ class FlowsService:# PATCH the updated flowresponse = await clients.langflow_request(- "PATCH", f"/api/v1/flows/{flow_id}", json=flow_data+ "PATCH",+ f"/api/v1/flows/{flow_id}",+ json=flow_data,+ idempotent=True,)if response.status_code != 200:diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 87bd1d260..a530c377c 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -490,6 +490,7 @@ class LangflowFileService:# Get the current embedding model and provider credentials from configfrom config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE+ from services.langflow_llm_token_service import HOP_PURPOSE_INGESTfrom utils.langflow_headers import (add_provider_credentials_to_headers,build_model_provider_headers,@@ -570,6 +571,7 @@ class LangflowFileService:flows_service=self.flows_service,jwt_token=jwt_token,user_id=owner,+ purpose=HOP_PURPOSE_INGEST,)if self.ingest_token_service is None:await self._ensure_langflow_ingest_index(embedding_model)@@ -696,6 +698,7 @@ class LangflowFileService:tweaks = {}from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, get_openrag_config+ from services.langflow_llm_token_service import HOP_PURPOSE_INGESTfrom utils.langflow_headers import (add_provider_credentials_to_headers,build_model_provider_headers,@@ -758,6 +761,7 @@ class LangflowFileService:flows_service=self.flows_service,jwt_token=jwt_token,user_id=owner,+ purpose=HOP_PURPOSE_INGEST,)if self.ingest_token_service is None:await self._ensure_langflow_ingest_index(embedding_model)diff --git a/src/services/langflow_llm_token_service.py b/src/services/langflow_llm_token_service.pyindex 5226eace7..6af222dc8 100644--- a/src/services/langflow_llm_token_service.py+++ b/src/services/langflow_llm_token_service.py@@ -29,6 +29,13 @@ LANGFLOW_HOP_AUDIENCES = frozenset(})+#: What a Langflow run's hop token is for, carried as its `purpose` claim. The+#: gateway gives `chat` embedding calls (query retrieval) their own lane so they+#: are not queued behind a bulk ingest. Anything else, including no claim at+#: all, is treated as bulk traffic.+HOP_PURPOSE_CHAT = "chat"+HOP_PURPOSE_INGEST = "ingest"+def langflow_hop_audience(token: str,@@ -89,6 +96,7 @@ class LangflowLlmTokenService:user_id: str,email: str | None = None,name: str | None = None,+ purpose: str | None = None,) -> str:now = int(time.time())subject = (user_id or "").strip() or "anonymous"@@ -103,9 +111,15 @@ class LangflowLlmTokenService:"email": (email or subject),"name": (name or subject),}+ if purpose:+ payload["purpose"] = purposereturn jwt.encode(payload, self._signing_key, algorithm=self.algorithm)def validate_token(self, token: str) -> User:+ return self.user_from_claims(self.validate_claims(token))++ def validate_claims(self, token: str) -> dict[str, Any]:+ """Verified claims of a hop token; raises `ValueError` if it is not one."""try:payload = jwt.decode(token,@@ -127,7 +141,11 @@ class LangflowLlmTokenService:user_id = str(payload.get("user_id") or payload.get("sub") or "").strip()if not user_id:raise ValueError("Langflow LLM proxy token is missing user_id")+ return payload+ @staticmethod+ def user_from_claims(payload: dict[str, Any]) -> User:+ user_id = str(payload.get("user_id") or payload.get("sub") or "").strip()return User(user_id=user_id,email=str(payload.get("email") or user_id),diff --git a/src/services/langflow_mcp_service.py b/src/services/langflow_mcp_service.pyindex 826c30621..21522a560 100644--- a/src/services/langflow_mcp_service.py+++ b/src/services/langflow_mcp_service.py@@ -257,6 +257,7 @@ class LangflowMCPService:method="PATCH",endpoint=f"/api/v2/mcp/servers/{server_name}",json=payload,+ idempotent=True,)if response.status_code in (502, 503, 504):response.raise_for_status()diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.pyindex 3dc42ea41..f545c9de1 100644--- a/src/services/llm_gateway.py+++ b/src/services/llm_gateway.py@@ -7,10 +7,15 @@ routes by model prefix / configured provider. Callers never see upstream keys.from __future__ import annotations+import asyncio+import contextlib+import functoolsimport jsonimport re-from collections.abc import AsyncIterator, Mapping+from collections import deque+from collections.abc import AsyncIterator, Awaitable, Callable, Mappingfrom typing import Any, Literal+from uuid import uuid4from services import provider_error_logfrom services.model_catalog import is_known_provider, litellm_provider_key@@ -219,6 +224,222 @@ def _provider_runtime_kwargs(provider: str, config=None) -> dict[str, Any]:return runtime_kwargs_for(enhancement, stored)+#: Lanes of a provider's embedding limit. `interactive` is query embedding —+#: search and chat retrieval, where a person is waiting; `bulk` is everything+#: else, chiefly ingestion. One slot of the limit is reserved for queries, so a+#: folder upload cannot queue them behind hundreds of chunks.+_BULK_LANE = "bulk"+_INTERACTIVE_LANE = "interactive"+++def _lane_limit(limit: int, interactive: bool) -> tuple[str, int]:+ """The lane a call waits in and that lane's size, out of `limit` in total.++ The lanes split the operator's limit rather than add to it, so the endpoint+ never sees more than `limit` calls at once: `limit - 1` for bulk traffic+ and one for queries. A limit of 1 cannot be split without exceeding it, so+ both kinds of call then share the single slot.+ """+ if limit < 2:+ return _BULK_LANE, limit+ if interactive:+ return _INTERACTIVE_LANE, 1+ return _BULK_LANE, limit - 1+++class _EmbeddingLimiter:+ """Bounds one provider's in-flight embedding calls under a limit that can change.++ A pair of per-lane semaphores cannot be resized: replacing them when the+ operator changes the limit forgets the calls still holding the old ones, so+ the endpoint briefly sees more than the new limit. This keeps one count per+ lane for the provider's lifetime and admits a call only while the total is+ under the current limit *and* its lane is under its share (`_lane_limit`).++ A freed slot is handed straight to the next waiter (as `asyncio.Semaphore`+ does), queries first, so neither a new arrival nor a bulk chunk can take a+ slot a waiting query is entitled to. `limit is None` means unbounded: calls+ are still counted, so turning a limit back on measures it against them.++ Not thread-safe; it lives on one event loop (see `_embedding_limiters`).+ """++ def __init__(self, provider: str, limit: int | None, loop: asyncio.AbstractEventLoop):+ self.provider = provider+ self.limit = limit+ self.loop = loop+ self._active = {_BULK_LANE: 0, _INTERACTIVE_LANE: 0}+ self._waiters: dict[str, deque[asyncio.Future[None]]] = {+ _BULK_LANE: deque(),+ _INTERACTIVE_LANE: deque(),+ }++ @property+ def in_flight(self) -> int:+ return sum(self._active.values())++ def _admits(self, lane: str) -> bool:+ if self.limit is None:+ return True+ if self.in_flight >= self.limit:+ return False+ _, lane_limit = _lane_limit(self.limit, lane == _INTERACTIVE_LANE)+ return self._active[lane] < lane_limit++ def _wake(self) -> None:+ """Grant free slots to waiters, queries before bulk, FIFO within a lane."""+ for lane in (_INTERACTIVE_LANE, _BULK_LANE):+ waiters = self._waiters[lane]+ while waiters and self._admits(lane):+ waiter = waiters.popleft()+ if waiter.done(): # cancelled while queued+ continue+ self._active[lane] += 1+ waiter.set_result(None)++ async def acquire(self, lane: str, model: str = "") -> None:+ if not self._waiters[lane] and self._admits(lane):+ self._active[lane] += 1+ return+ waiter: asyncio.Future[None] = self.loop.create_future()+ self._waiters[lane].append(waiter)+ logger.debug(+ "Embedding call waiting for a free provider slot",+ provider=self.provider,+ model=model,+ lane=lane,+ max_concurrency=self.limit,+ in_flight=self.in_flight,+ )+ try:+ await waiter+ except asyncio.CancelledError:+ if waiter.done() and not waiter.cancelled():+ # Granted a slot in the same tick the caller was cancelled:+ # hand it on, or it is lost for the provider's lifetime.+ self.release(lane)+ else:+ with contextlib.suppress(ValueError):+ self._waiters[lane].remove(waiter)+ raise++ def release(self, lane: str) -> None:+ if self._active[lane] <= 0:+ logger.error(+ "Embedding slot released more often than acquired; ignoring",+ provider=self.provider,+ lane=lane,+ )+ self._active[lane] = 0+ else:+ self._active[lane] -= 1+ self._wake()++ def resize(self, limit: int | None) -> None:+ """Apply a limit changed in Settings, counting every call still in flight."""+ if limit == self.limit:+ return+ previous = self.limit+ self.limit = limit+ logger.info(+ "Resizing embedding concurrency limit for provider",+ provider=self.provider,+ previous_max_concurrency=previous,+ max_concurrency=limit,+ bulk_in_flight=self._active[_BULK_LANE],+ interactive_in_flight=self._active[_INTERACTIVE_LANE],+ )+ if limit is not None and self.in_flight > limit:+ logger.info(+ "More embedding calls in flight than the new limit allows; "+ "new calls wait until they finish",+ provider=self.provider,+ max_concurrency=limit,+ in_flight=self.in_flight,+ )+ self._wake()++ @contextlib.asynccontextmanager+ async def slot(self, lane: str, model: str = "") -> AsyncIterator[None]:+ """Hold one of the provider's embedding slots for the duration of a call."""+ await self.acquire(lane, model)+ try:+ yield+ finally:+ self.release(lane)+++#: Per-provider embedding bulkheads, keyed by provider key. The backend runs a+#: single worker (enforced in `app/lifespan.py`), so a process-local limiter+#: bounds the provider's traffic globally. Each limiter records its loop, and a+#: different running loop gets a new one, because asyncio futures must not be+#: shared across loops.+_embedding_limiters: dict[str, _EmbeddingLimiter] = {}+++def _embedding_limiter(provider: str, config=None) -> _EmbeddingLimiter | None:+ """The limiter bounding embedding calls to `provider`, if any.++ The limit comes from the provider enhancement (`embedding_max_concurrency`),+ so providers without one — and every provider whose enhancement sets no+ limit — are unaffected. Excess callers wait here instead of queueing at the+ upstream, where a slow model server lets a fronting proxy time them out+ (RHOAI's kube-rbac-proxy answers 502 after 30s). Interactive calls wait in+ their own lane (`_lane_limit`). A limit changed in Settings takes effect on+ the next call and is measured against every call still in flight, so a+ lowered limit admits nothing new until enough of them finish.+ """+ cfg = config or _get_config()+ prov = getattr(cfg, "providers", None)+ if prov is None or not hasattr(prov, "credential_values"):+ return None++ from enhancements.providers.registry import embedding_concurrency_for, get++ key = (provider or "").strip().lower()+ if get(key) is None:+ return None+ stored = (+ prov.stored_credentials(key)+ if hasattr(prov, "stored_credentials")+ else prov.credential_values(key)+ )+ limit = embedding_concurrency_for(key, stored)+ loop = asyncio.get_running_loop()+ cached = _embedding_limiters.get(key)+ if cached is not None and cached.loop is loop:+ # Resized in place (also to "no limit"), so the calls it is already+ # counting stay counted against whatever limit applies next.+ cached.resize(limit)+ return cached+ if limit is None:+ _embedding_limiters.pop(key, None)+ return None++ limiter = _EmbeddingLimiter(key, limit, loop)+ _embedding_limiters[key] = limiter+ logger.info(+ "Bounding concurrent embedding calls for provider",+ provider=key,+ max_concurrency=limit,+ bulk_max_concurrency=_lane_limit(limit, False)[1],+ interactive_max_concurrency=_lane_limit(limit, True)[1],+ )+ return limiter+++async def _embedding_slot(+ limiter: _EmbeddingLimiter | None, provider: str, model: str, lane: str = _BULK_LANE+) -> AsyncIterator[None]:+ """Hold one of the provider's embedding slots for the duration of a call."""+ if limiter is None:+ yield+ return+ async with limiter.slot(lane, model):+ yield++def resolve_call(model: str | None,*,@@ -276,6 +497,36 @@ _UPSTREAM_TLS_MESSAGE = ("The provider's TLS certificate is not trusted by this deployment. An operator needs to ""add the provider's CA certificate to OpenRAG's trust store.")+#: An upstream 502/504 with no body to quote — typically a proxy in front of+#: the model server (RHOAI's kube-rbac-proxy) giving up while the server is+#: still busy. LiteLLM then reports only its own wrapper names+#: ("BadGatewayError: Hosted_vllmException -"), which reads like a bug.+_UPSTREAM_GATEWAY_MESSAGE = (+ "The model server did not respond in time or is unavailable (upstream 502/504). It may "+ "be overloaded: check its capacity and any proxy timeout in front of it, or lower the "+ "provider's max concurrent requests."+)+#: The connection closed before the response finished. A proxy or load balancer+#: in front of the model server timing out a long response, not the model+#: rejecting anything — see `is_provider_truncated_response_error`.+_UPSTREAM_TRUNCATED_MESSAGE = (+ "The connection to the model server closed before the response finished. A proxy or "+ "load balancer in front of it likely timed out a long response: raise that timeout, "+ "or point the provider at an endpoint that bypasses it."+)+_GATEWAY_FAILURE_STATUSES = frozenset({502, 504})+#: LiteLLM exception class names that wrap an upstream error, e.g.+#: "litellm.BadGatewayError: BadGatewayError: Hosted_vllmException -".+_EXCEPTION_WRAPPER_RE = re.compile(r"(?:litellm\.)?[A-Za-z_]+(?:Error|Exception)\b\s*[:\-]?\s*")+++def _is_opaque_gateway_failure(upstream: str, exc: BaseException | None) -> bool:+ """True for an upstream 502/504 whose text is nothing but exception wrappers."""+ status = getattr(exc, "status_code", None)+ if not isinstance(status, int) or status not in _GATEWAY_FAILURE_STATUSES:+ return False+ remainder = _EXCEPTION_WRAPPER_RE.sub("", upstream or "").strip(" -:.")+ return not remainder#: Longest upstream explanation we will echo. Provider bodies can embed a whole@@ -449,6 +700,7 @@ def _upstream_client_message(is_generic_upstream_error,is_provider_credential_error,is_provider_tls_error,+ is_provider_truncated_response_error,)label = _call_label(provider, model)@@ -457,10 +709,17 @@ def _upstream_client_message(# someone to rotate a working API key wastes the trip.if is_provider_tls_error(detail):return f"{_UPSTREAM_TLS_MESSAGE} ({label})"+ # Also ahead of the credential branch: aiohttp reports the cut-off with a+ # "400" of its own, and LiteLLM's wrapper chain around it says nothing an+ # operator can act on.+ if is_provider_truncated_response_error(detail):+ return f"{_UPSTREAM_TRUNCATED_MESSAGE} ({label})"if is_provider_credential_error(detail):return f"{_UPSTREAM_CREDENTIAL_MESSAGE} ({label})"upstream = _provider_error_text(detail, exc)upstream = _sanitise_upstream_detail(upstream) if upstream else ""+ if _is_opaque_gateway_failure(upstream, exc):+ return f"{_UPSTREAM_GATEWAY_MESSAGE} ({label})"if not upstream or is_generic_upstream_error(upstream):return f"{_UPSTREAM_FAILURE_MESSAGE} ({label})"# lgtm[py/stack-trace-exposure] — provider error text only; traceback frames,@@ -468,6 +727,24 @@ def _upstream_client_message(return f"{label}: {upstream}"+def _is_request_scoped_failure(detail: str, exc: BaseException | None) -> bool:+ """True when the failure belongs to this request, not to the provider.++ A prompt that overflows the context window fails because of what this one+ call carried; the next, shorter one succeeds. Recording it in+ `provider_error_log` would latch the health banner against a provider that+ is serving fine — and on a provider whose probe cannot clear it, until the+ entry goes stale. The caller still gets the error.+ """+ from api.provider_validation import is_context_window_error++ if exc is not None and any(+ cls.__name__ == "ContextWindowExceededError" for cls in type(exc).__mro__+ ):+ return True+ return is_context_window_error(detail)++def _redact(message: str, credentials: Mapping[str, Any]) -> str:redacted = messagefor value in credentials.values():@@ -565,6 +842,118 @@ def _repair_tool_calls(tool_calls: Any, provider: str, model: str) -> int:return repaired+def _finalise_tool_call(call: dict[str, Any]) -> bool:+ """Make `call` something a client can execute. False when it cannot be.++ A call with no name names no tool, so nothing can run it and no result can+ ever come back for it; forwarding it only corrupts the assistant message the+ client will replay on its next turn. A missing `id` is recoverable — the id+ is only the handle a tool result is correlated by, so any stable string+ serves — and minting one keeps a real call alive that clients would+ otherwise reject for carrying `id: null`. Blank `arguments` are recoverable+ too: a named call streamed with no arguments fragment is a zero-argument+ call, and the contract is a JSON *object* — `""` does not parse as one.+ """+ function = call.get("function")+ if not isinstance(function, dict) or not function.get("name"):+ return False+ if not isinstance(call.get("id"), str) or not call["id"]:+ call["id"] = f"call_{uuid4().hex}"+ arguments = function.get("arguments")+ if arguments is None or (isinstance(arguments, str) and not arguments.strip()):+ function["arguments"] = "{}"+ logger.debug(+ "Defaulted blank tool call arguments to an empty object", tool=function["name"]+ )+ return True+++def _sanitise_messages(messages: Any) -> tuple[list[Any], int]:+ """`messages` with unusable tool calls repaired or dropped, and a repair count.++ The gateway forwards a client's conversation verbatim, and one malformed+ tool call in it is not a one-turn problem. Clients replay their whole+ history on every turn, so the same bad message goes back up again and again,+ and providers reject the *request* rather than the message that spoiled it —+ vLLM answers the lot with "Please ensure `tool_calls` are iterable of tool+ calls". The conversation is then wedged for good, with nothing in the UI+ pointing at the turn that broke it. A proxy sitting between many clients and+ many providers is the right place to stop that.++ Three shapes are handled, all of them things real clients emit: `arguments`+ sent as an object rather than the JSON *string* OpenAI's contract requires;+ a tool call carrying no id or no name, which nothing can execute and no+ result can be matched to; and the tool results left behind by one, which are+ dropped alongside it so no provider is handed a reply to a call that is no+ longer there.++ What survives must still be a sequence providers accept: a `tool` message+ is kept only when it answers a call kept on the assistant message just+ before it, and an assistant message left with neither tool calls nor+ content is dropped, since `content` is only optional beside `tool_calls`.+ """+ cleaned: list[Any] = []+ repairs = 0+ # Ids of the calls kept on the most recent assistant message: the only ones+ # a `tool` message may answer.+ live_ids: set[str] = set()+ for message in messages or []:+ if not isinstance(message, dict):+ cleaned.append(message)+ continue+ if message.get("role") == "tool":+ if message.get("tool_call_id") not in live_ids:+ repairs += 1+ continue+ cleaned.append(message)+ continue+ live_ids = set()+ if "tool_calls" not in message:+ cleaned.append(message)+ continue++ raw = message.get("tool_calls")+ kept: list[dict[str, Any]] = []+ for call in raw if isinstance(raw, list) else []:+ if not isinstance(call, dict):+ repairs += 1+ continue+ # The client controls this value, and `dict()` raises on a string or+ # a number. A non-mapping names no tool, so it becomes an empty one+ # and the call is dropped below like any other nameless call.+ function = call.get("function")+ call = {**call, "function": dict(function) if isinstance(function, dict) else {}}+ arguments, changed = _normalise_tool_arguments(call["function"].get("arguments"))+ if changed:+ call["function"]["arguments"] = arguments+ repairs += 1+ # Unlike the response path, a missing id is not minted here: the+ # result for this call is already somewhere in the history under the+ # id the client used, and inventing a different one would orphan it.+ if isinstance(call.get("id"), str) and call["id"] and call["function"].get("name"):+ kept.append(call)+ else:+ repairs += 1++ message = dict(message)+ if kept:+ message["tool_calls"] = kept+ live_ids = {call["id"] for call in kept}+ else:+ # `tool_calls: null` is itself a shape some providers iterate+ # without a None check, so the key goes rather than emptying.+ message.pop("tool_calls")+ # A list's entries were each counted above; anything else was+ # never iterable in the first place and is one repair on its own.+ repairs += bool(raw) and not isinstance(raw, list)+ if raw and message.get("role") == "assistant" and not message.get("content"):+ # The dropped calls were the whole message; what is left has+ # nothing to say, and no provider accepts it saying nothing.+ continue+ cleaned.append(message)+ return cleaned, repairs++def _repair_completion_payload(payload: dict[str, Any], provider: str, model: str) -> int:repaired = 0for choice in payload.get("choices") or []:@@ -655,6 +1044,36 @@ def _reasoning_off_retry(model: str, kwargs: dict[str, Any]) -> bool:return True+async def with_stale_connection_retry(+ call: Callable[[], Awaitable[Any]], *, provider: str, model: str, kind: str+) -> Any:+ """Run one LiteLLM call, sending it once more if a dead pooled connection ate it.++ LiteLLM keeps idle connections for 120s; vLLM closes them after 5s. A call+ that picks one up just as the server closes it fails with "Server+ disconnected" before the server reads a byte (it logs nothing), and the+ replay goes out on a fresh connection. Only that failure is retried — see+ `is_provider_stale_connection_error` — so a request the server may have+ acted on is never sent twice. For a stream only the call that opens it is+ wrapped, so nothing has reached the caller yet; a mid-stream failure is+ not retried.+ """+ from api.provider_validation import is_provider_stale_connection_error++ try:+ return await call()+ except Exception as exc:+ if not is_provider_stale_connection_error(exc):+ raise+ logger.warning(+ "Model server closed an idle connection before answering; retrying once",+ provider=provider,+ model=model,+ kind=kind,+ )+ return await call()++async def chat_completions(body: Mapping[str, Any], *, config=None) -> dict[str, Any] | AsyncIterator[str]:@@ -667,24 +1086,37 @@ async def chat_completions(if litellm_model in _TOOLS_NEED_REASONING_OFF:# Already learned about this model; do not spend a round-trip relearning._reasoning_off_retry(litellm_model, kwargs)+ messages, repairs = _sanitise_messages(body.get("messages"))+ if repairs:+ logger.warning(+ "Repaired malformed tool calls in the request's conversation",+ provider=provider,+ model=litellm_model,+ repairs=repairs,+ )async def _call() -> Any:import litellm- return await litellm.acompletion(+ return await with_stale_connection_retry(+ lambda: litellm.acompletion(+ model=litellm_model,+ messages=messages,+ stream=stream,+ # OpenAI-compatible clients send OpenAI's full parameter set, but+ # providers accept different subsets — watsonx rejects+ # `parallel_tool_calls`, `max_completion_tokens` and `logit_bias`,+ # and LiteLLM raises UnsupportedParamsError rather than ignoring+ # them. A proxy that fans out to many providers must degrade to+ # the provider's capabilities instead of failing the request.+ drop_params=True,+ **credentials,+ **runtime_kwargs,+ **kwargs,+ ),+ provider=provider,model=litellm_model,- messages=list(body.get("messages") or []),- stream=stream,- # OpenAI-compatible clients send OpenAI's full parameter set, but- # providers accept different subsets — watsonx rejects- # `parallel_tool_calls`, `max_completion_tokens` and `logit_bias`,- # and LiteLLM raises UnsupportedParamsError rather than ignoring- # them. A proxy that fans out to many providers must degrade to the- # provider's capabilities instead of failing the request.- drop_params=True,- **credentials,- **runtime_kwargs,- **kwargs,+ kind="chat",)try:@@ -716,7 +1148,8 @@ async def chat_completions(# The health banner otherwise reports whatever its own probe hit, which# is a different request and so often a different error. Hand it the# text this caller is being shown.- provider_error_log.record_failure(provider, "chat", message)+ if not _is_request_scoped_failure(detail, exc):+ provider_error_log.record_failure(provider, "chat", message)raise LlmGatewayError(message,_upstream_status_code(exc),@@ -783,6 +1216,57 @@ class _ToolCallBuffer:def __bool__(self) -> bool:return bool(self._order)+ def _open_call(self, choice_index: int) -> dict[str, Any] | None:+ """The call most recently opened on this choice, if there is one."""+ for key in reversed(self._order):+ if key[0] == choice_index:+ return self._calls[key]+ return None++ def _target(self, choice_index: int, index: int, raw: Mapping[str, Any]) -> dict[str, Any]:+ """The call `raw` belongs to: one already open, or a new one.++ A delta that names no call — no `id`, no `function.name` — cannot be the+ start of one, so it continues the call already open on this choice+ rather than opening another. vLLM's tool parsers can attribute a call's+ last `arguments` fragment, often the closing brace by itself, to the+ *next* index; taken at face value that does two kinds of damage at once.+ The real call loses the fragment and its arguments no longer parse, and+ a second call appears with no id and no name that nothing can execute.+ Neither survives the round trip: the client folds both into the+ assistant message it replays on the next turn, and the provider then+ rejects every later request in that conversation rather than the one+ message that is malformed.++ With nothing open to continue, the fragment instead opens a provisional+ call at its own index. Some providers send an `arguments` fragment ahead+ of the delta that carries the call's `id` and `name`; dropping it would+ truncate the arguments once that delta arrives at the same index and+ finds the call. A provisional call that is never named is discarded by+ `drain`, which logs it.+ """+ key = (choice_index, index)+ call = self._calls.get(key)+ if call is not None:+ return call+ function = raw.get("function")+ names_a_call = bool(raw.get("id")) or bool(+ isinstance(function, dict) and function.get("name")+ )+ if not names_a_call:+ open_call = self._open_call(choice_index)+ if open_call is not None:+ return open_call+ logger.debug(+ "Holding an unnamed tool-call fragment until its call is named",+ choice_index=choice_index,+ index=index,+ )+ call = {"index": index, "type": "function", "function": {"name": "", "arguments": ""}}+ self._calls[key] = call+ self._order.append(key)+ return call+def absorb(self, choice_index: int, tool_calls: Any) -> None:for position, raw in enumerate(tool_calls or []):if not isinstance(raw, dict):@@ -790,16 +1274,7 @@ class _ToolCallBuffer:index = raw.get("index")if not isinstance(index, int):index = position- key = (choice_index, index)- call = self._calls.get(key)- if call is None:- call = {- "index": index,- "type": "function",- "function": {"name": "", "arguments": ""},- }- self._calls[key] = call- self._order.append(key)+ call = self._target(choice_index, index, raw)if raw.get("id"):call["id"] = raw["id"]if raw.get("type"):@@ -825,9 +1300,18 @@ class _ToolCallBuffer:grouped.setdefault(choice_index, [])for key in self._order:grouped[key[0]].append(self._calls[key])- for calls in grouped.values():+ for choice_index, calls in grouped.items():calls.sort(key=lambda call: call.get("index", 0))- repaired += _repair_tool_calls(calls, provider, model)+ usable = [call for call in calls if _finalise_tool_call(call)]+ if len(usable) != len(calls):+ logger.warning(+ "Dropped tool calls that named no tool",+ provider=provider,+ model=model,+ dropped=len(calls) - len(usable),+ )+ grouped[choice_index] = usable+ repaired += _repair_tool_calls(usable, provider, model)self._calls.clear()self._order.clear()return grouped, repaired@@ -1007,7 +1491,8 @@ async def _stream_sse(tally.error = detaillogger.error("LLM chat stream failed", provider=provider, model=model, error=detail)message = _upstream_client_message(detail, provider, model, exc)- provider_error_log.record_failure(provider, "chat", message)+ if not _is_request_scoped_failure(detail, exc):+ provider_error_log.record_failure(provider, "chat", message)yield _error_frame(message, provider, model)finally:close = getattr(stream, "aclose", None) or getattr(stream, "close", None)@@ -1056,13 +1541,22 @@ def _embedding_input(value: Any) -> Any:return [value] if isinstance(value, str) else value-async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:- """OpenAI `POST /v1/embeddings`."""+async def embeddings(+ body: Mapping[str, Any], *, config=None, interactive: bool = False+) -> dict[str, Any]:+ """OpenAI `POST /v1/embeddings`.++ `interactive` marks a query embedding someone is waiting on (search, chat+ retrieval). At a provider with a concurrency limit it waits in its own+ lane instead of behind bulk ingestion; see `_embedding_limiter`.+ """cfg = config or _get_config()litellm_model, provider, credentials = resolve_call(body.get("model"), kind="embedding", config=cfg)runtime_kwargs = _provider_runtime_kwargs(provider, cfg)+ limiter = _embedding_limiter(provider, cfg)+ lane = _INTERACTIVE_LANE if interactive else _BULK_LANEembedding_input = _embedding_input(body.get("input"))should_batch = (provider == "watsonx_onprem"@@ -1074,12 +1568,18 @@ async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:import litellmif not should_batch:- result = await litellm.aembedding(- model=litellm_model,- input=embedding_input,- **credentials,- **runtime_kwargs,- )+ async with _embedding_slot(limiter, provider, litellm_model, lane):+ result = await with_stale_connection_retry(+ lambda: litellm.aembedding(+ model=litellm_model,+ input=embedding_input,+ **credentials,+ **runtime_kwargs,+ ),+ provider=provider,+ model=litellm_model,+ kind="embedding",+ )response = _to_openai_dict(result)else:response = {}@@ -1091,12 +1591,19 @@ async def embeddings(body: Mapping[str, Any], *, config=None) -> dict[str, Any]:_WATSONX_ONPREM_EMBEDDING_BATCH_SIZE,):batch = embedding_input[offset : offset + _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE]- result = await litellm.aembedding(- model=litellm_model,- input=batch,- **credentials,- **runtime_kwargs,- )+ async with _embedding_slot(limiter, provider, litellm_model, lane):+ result = await with_stale_connection_retry(+ functools.partial(+ litellm.aembedding,+ model=litellm_model,+ input=batch,+ **credentials,+ **runtime_kwargs,+ ),+ provider=provider,+ model=litellm_model,+ kind="embedding",+ )payload = _to_openai_dict(result)if not response:response = {diff --git a/src/services/model_catalog.py b/src/services/model_catalog.pyindex 2fceffd9f..8dc743c35 100644--- a/src/services/model_catalog.py+++ b/src/services/model_catalog.py@@ -436,17 +436,24 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:)-async def refresh_live_models() -> None:+async def refresh_live_models(provider: str | None = None) -> None:"""Re-list the models on any provider that can only answer for itself.Called by the catalogue routes before the payload is built. The providercaches with a TTL, so this is a network call once every few minutes ratherthan once per request, and a failure leaves the previous answer — or theconfigured fallback — in place.++ `provider` narrows it to that one provider. The catalogue needs every+ listing; a caller asking about one provider — the health check — should not+ wait on a slow cluster that belongs to another."""from config.settings import get_openrag_config+ only = (provider or "").strip().lower() or Nonefor enhancement in provider_enhancements():+ if only is not None and enhancement.PROVIDER_KEY != only:+ continueif enhancement.PROVIDER_KEY not in supported_provider_keys() or not hasattr(enhancement, "fetch_models"):diff --git a/src/services/provider_error_log.py b/src/services/provider_error_log.pyindex 424ffe481..2672f6654 100644--- a/src/services/provider_error_log.py+++ b/src/services/provider_error_log.py@@ -69,5 +69,9 @@ def latest_failure(provider: str | None, kind: CallKind) -> str | None:def clear() -> None:- """Drop every recorded failure. For tests."""+ """Drop every recorded failure.++ For settings saves that change a provider, model or credentials — the+ entries describe a setup that no longer exists — and for tests.+ """_failures.clear()diff --git a/src/services/search_service.py b/src/services/search_service.pyindex 9e16e01f3..762f4c671 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -430,8 +430,10 @@ class SearchService:# Use the same credential-aware gateway as Langflow.# Provider-qualified and legacy routes are resolved in# one place and upstream credentials never leave OpenRAG.+ # A query, so it must not queue behind bulk ingestion.resp = await gateway_embeddings(- {"model": space.route_model, "input": [query]}+ {"model": space.route_model, "input": [query]},+ interactive=True,)data = resp.get("data", [])embedding = data[0].get("embedding") if data else Nonediff --git a/src/services/status_checks.py b/src/services/status_checks.pyindex f3da68da4..690929902 100644--- a/src/services/status_checks.py+++ b/src/services/status_checks.py@@ -1,5 +1,7 @@from time import perf_counter+import httpx+from api.schemas.status import ComponentBuild, ComponentState, ComponentStatusfrom config.settings import DOCLING_SERVE_URL, LANGFLOW_URL, clients, get_openrag_configfrom services.component_logs import record_check_result@@ -8,7 +10,36 @@ from utils.version_utils import OPENRAG_VERSIONlogger = get_logger(__name__)-_CHECK_TIMEOUT_S = 2.0+# Per-phase httpx budget (connect, read, ... each), not a total deadline. Keep it well+# below status_service.CHECK_TIMEOUT_S, or that outer deadline cancels the check first and+# the specific diagnosis below is replaced by a generic UNKNOWN.+_CHECK_TIMEOUT_S = 5.0+++def _record_http_check_failure(+ component: str,+ exc: Exception,+ target_url: str,+ *,+ unreachable_message: str,+ slow_message: str,+) -> tuple[ComponentState, str, str]:+ """Classify a failed HTTP probe and record it; returns (state, message, last_error).++ A timeout after the connection opened means the service is busy, not down: a+ single-worker Langflow or docling-serve stalls for seconds under ingest load and+ then catches up. That reads as DEGRADED. A connect timeout or any other failure+ means nothing answered, which is UNHEALTHY.+ """+ if isinstance(exc, httpx.TimeoutException) and not isinstance(exc, httpx.ConnectTimeout):+ state, message = ComponentState.DEGRADED, slow_message+ # str() of an httpx timeout is usually empty; say how long we waited instead.+ detail = f"{type(exc).__name__} after {_CHECK_TIMEOUT_S}s — target: {target_url}"+ else:+ state, message = ComponentState.UNHEALTHY, unreachable_message+ detail = f"{type(exc).__name__}: {exc} — target: {target_url}"+ record_check_result(component, state, message, detail=detail)+ return state, message, detailasync def check_openrag_backend() -> ComponentStatus:@@ -109,16 +140,14 @@ async def check_docling() -> ComponentStatus:)last_error = messageexcept Exception as e:- logger.warning("Docling status check failed", error=str(e))- message = "Docling Serve unreachable"- status = ComponentState.UNHEALTHY- record_check_result(+ logger.warning("Docling status check failed", error_type=type(e).__name__, error=str(e))+ status, message, last_error = _record_http_check_failure("docling",- ComponentState.UNHEALTHY,- message,- detail=f"{type(e).__name__}: {e} — target: {target_url}",+ e,+ target_url,+ unreachable_message="Docling Serve unreachable",+ slow_message="Docling Serve is slow to respond",)- last_error = f"{type(e).__name__}: {e} — target: {target_url}"return ComponentStatus(name="docling",@@ -174,16 +203,14 @@ async def check_langflow() -> ComponentStatus:)last_error = messageexcept Exception as e:- logger.warning("Langflow status check failed", error=str(e))- message = "Langflow is unreachable"- status = ComponentState.UNHEALTHY- record_check_result(+ logger.warning("Langflow status check failed", error_type=type(e).__name__, error=str(e))+ status, message, last_error = _record_http_check_failure("langflow",- ComponentState.UNHEALTHY,- message,- detail=f"{type(e).__name__}: {e} — target: {target_url}",+ e,+ target_url,+ unreachable_message="Langflow is unreachable",+ slow_message="Langflow is slow to respond",)- last_error = f"{type(e).__name__}: {e} — target: {target_url}"return ComponentStatus(name="langflow",diff --git a/src/services/status_service.py b/src/services/status_service.pyindex 5033a8d39..975e26c72 100644--- a/src/services/status_service.py+++ b/src/services/status_service.py@@ -14,7 +14,11 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)-CHECK_TIMEOUT_S = 5.0+# Total wall-clock deadline per check. It must exceed status_checks._CHECK_TIMEOUT_S, which+# httpx applies per phase, so a slow service is diagnosed by its own check rather than+# cancelled here into UNKNOWN. It is also the only bound on the OpenSearch check, and it+# caps /status latency, since that endpoint waits for its slowest check.+CHECK_TIMEOUT_S = 8.0CHECK_SPECS = [check_openrag_backend, check_docling, check_langflow, check_opensearch]diff --git a/src/utils/langflow_headers.py b/src/utils/langflow_headers.pyindex b5059f25d..ca8ee071d 100644--- a/src/utils/langflow_headers.py+++ b/src/utils/langflow_headers.py@@ -73,6 +73,7 @@ async def add_provider_credentials_to_headers(flows_service=None,jwt_token: str = None,user_id: str | None = None,+ purpose: str | None = None,) -> None:"""Add Langflow global variables for the OpenRAG LLM proxy and infra URLs.@@ -81,7 +82,9 @@ async def add_provider_credentials_to_headers(speak OpenAI-compatible HTTP to OpenRAG (`OPENRAG_LLM_BASE_URL`) andauthenticate with a short-lived hop token as `OPENRAG_LLM_TOKEN` — samepattern as `OPENRAG_INGEST_TOKEN`, scoped to the LLM proxy only.- Chat and embeddings share that base URL and hop token.+ Chat and embeddings share that base URL and hop token. `purpose` is+ stamped on the token (`HOP_PURPOSE_CHAT` / `HOP_PURPOSE_INGEST`) so the+ gateway can tell a query embedding from a bulk ingest one.NOTE: `headers` may hold a JWT after this call. Never log it directly —use utils.logging_config.sanitize_headers() if a header dict must be logged.@@ -92,7 +95,7 @@ async def add_provider_credentials_to_headers(headers["X-LANGFLOW-GLOBAL-VAR-OPENRAG_LLM_BASE_URL"] = get_langflow_llm_base_url()subject = (user_id or "").strip() or "anonymous"- hop_token = LangflowLlmTokenService().create_token(user_id=subject)+ hop_token = LangflowLlmTokenService().create_token(user_id=subject, purpose=purpose)headers["X-LANGFLOW-GLOBAL-VAR-OPENRAG_LLM_TOKEN"] = hop_token# Stock Language/Embedding Model nodes still bind api_key to OPENAI_API_KEY.headers["X-LANGFLOW-GLOBAL-VAR-OPENAI_API_KEY"] = hop_tokendiff --git a/src/utils/langflow_utils.py b/src/utils/langflow_utils.pyindex 5c42ef861..492e8df48 100644--- a/src/utils/langflow_utils.py+++ b/src/utils/langflow_utils.py@@ -232,6 +232,7 @@ async def enable_mcp_none_for_project(project_id: str | None, max_attempts: int"PATCH",f"/api/v1/mcp/project/{project_id}",json={"settings": [], "auth_settings": {"auth_type": "none"}},+ idempotent=True,)if patch_resp.status_code in (200, 201):logger.info(diff --git a/tests/unit/api/test_docling_health.py b/tests/unit/api/test_docling_health.pynew file mode 100644index 000000000..0ca1b577f--- /dev/null+++ b/tests/unit/api/test_docling_health.py@@ -0,0 +1,84 @@+"""`GET /docling/health`: a busy docling-serve is degraded, not stopped."""++import json+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++from api import docling+++def _patch_client(monkeypatch, *, raises=None, status_code=200):+ client = MagicMock()+ if raises is not None:+ client.get = AsyncMock(side_effect=raises)+ else:+ response = MagicMock(spec=httpx.Response)+ response.status_code = status_code+ client.get = AsyncMock(return_value=response)+ client.__aenter__ = AsyncMock(return_value=client)+ client.__aexit__ = AsyncMock(return_value=False)+ monkeypatch.setattr(docling.httpx, "AsyncClient", MagicMock(return_value=client))+ return client+++async def _call():+ response = await docling.health(MagicMock(), user=None)+ return response.status_code, json.loads(response.body)+++async def test_healthy(monkeypatch):+ client = _patch_client(monkeypatch, status_code=200)++ status_code, body = await _call()++ assert status_code == 200+ assert body["status"] == "healthy"+ assert client.get.call_args.kwargs["timeout"] == docling._HEALTH_CHECK_TIMEOUT_S+++ "exc", [httpx.ReadTimeout(""), httpx.WriteTimeout(""), httpx.PoolTimeout("")]+)+async def test_slow_reply_is_degraded_not_unhealthy(monkeypatch, exc):+ _patch_client(monkeypatch, raises=exc)++ status_code, body = await _call()++ assert status_code == 200+ assert body["status"] == "degraded"+ assert "slow to respond" in body["message"]+++async def test_connect_timeout_is_unhealthy(monkeypatch):+ _patch_client(monkeypatch, raises=httpx.ConnectTimeout(""))++ status_code, body = await _call()++ assert status_code == 503+ assert body["status"] == "unhealthy"+ assert body["message"] == "Connection timeout"+++async def test_connection_refused_is_unhealthy(monkeypatch):+ _patch_client(monkeypatch, raises=httpx.ConnectError("refused"))++ status_code, body = await _call()++ assert status_code == 503+ assert body["status"] == "unhealthy"+++async def test_non_200_is_unhealthy(monkeypatch):+ _patch_client(monkeypatch, status_code=500)++ status_code, body = await _call()++ assert status_code == 503+ assert body["status"] == "unhealthy"diff --git a/tests/unit/api/test_provider_health_latch.py b/tests/unit/api/test_provider_health_latch.pynew file mode 100644index 000000000..6e47b0992--- /dev/null+++ b/tests/unit/api/test_provider_health_latch.py@@ -0,0 +1,171 @@+"""The health banner has to be able to clear itself.++`provider_error_log` is erased by the next call to a provider that succeeds,+and until this was wired up the only such call was chat traffic. One failed+turn latched the banner, the frontend then polled every 5s with+`test_completion` while it stayed latched, and a provider that was+demonstrably serving kept being reported broken — offering "Fix Setup" for a+setup that had just passed its own check — until the entry went stale a+quarter of an hour later.++It must not clear itself on weaker evidence than the failure it erases,+though. Only a probe that sent the same shape of request — a tool-calling+completion for chat, a real embedding call for embeddings — may do it.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from api import provider_health+from api.provider_validation import ProbeResult+from services import provider_error_log++#: What each kind of passing validation actually exercised.+TOOL_COMPLETION = ProbeResult(model_probed=True, tools_exercised=True)+PLAIN_COMPLETION = ProbeResult(model_probed=True)+NO_CALL = ProbeResult()++[email protected](autouse=True)+def _clean():+ provider_error_log.clear()+ yield+ provider_error_log.clear()+++def _config():+ providers = SimpleNamespace(+ credential_values=lambda *_a, **_k: {},+ stored_credentials=lambda *_a, **_k: {},+ get_provider_config=lambda *_a, **_k: SimpleNamespace(api_key="k"),+ )+ return SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="gpt-oss-120b"),+ knowledge=SimpleNamespace(embedding_provider="rhoai", embedding_model="granite-embedding"),+ get_llm_provider_config=lambda: SimpleNamespace(api_key="k"),+ get_embedding_provider_config=lambda: SimpleNamespace(api_key="k"),+ )+++def _probes(monkeypatch, *, chat: ProbeResult, embedding: ProbeResult) -> None:+ """Validation passes for both roles, having exercised what is given."""++ async def ok(**kwargs):+ return embedding if kwargs.get("embedding_model") else chat++ monkeypatch.setattr(provider_health, "validate_provider_setup", ok)+++def _healthy_probe(monkeypatch):+ """A provider whose probes all pass, with the caches out of the way."""+ monkeypatch.setattr(provider_health, "get_openrag_config", _config)+ monkeypatch.setattr(provider_health, "is_known_provider", lambda _p: True)++ _probes(monkeypatch, chat=TOOL_COMPLETION, embedding=PLAIN_COMPLETION)+ monkeypatch.setattr(provider_health.provider_health_cache, "cache_key", lambda **_k: "key")+ monkeypatch.setattr(provider_health.provider_health_cache, "get", lambda _k: None)+ monkeypatch.setattr(+ provider_health.provider_health_cache, "set_and_release", lambda *_a, **_k: None+ )+ monkeypatch.setattr(provider_health.provider_health_cache, "release_error", lambda *_a: None)++ async def acquire(_key):+ return True++ monkeypatch.setattr(provider_health.provider_health_cache, "acquire", acquire)++ # Keep the corpus diagnostics out of it: unstubbed, they read the real+ # config, can list models on a real cluster, and query OpenSearch.+ async def _no_refresh(_provider):+ return None++ async def _no_stale(_provider):+ return None++ monkeypatch.setattr(provider_health, "_refresh_live_models", _no_refresh)+ monkeypatch.setattr(provider_health, "_stale_embedding_spaces", _no_stale)+++async def test_a_passing_tool_calling_probe_clears_a_latched_chat_failure(_healthy_probe):+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert provider_error_log.latest_failure("rhoai", "chat") is None+++async def test_a_passing_embedding_call_clears_a_latched_embedding_failure(_healthy_probe):+ provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert provider_error_log.latest_failure("rhoai", "embedding") is None+++async def test_a_tool_less_completion_leaves_a_latched_chat_failure(monkeypatch, _healthy_probe):+ """The LiteLLM probe (RHOAI and friends) sends a plain completion.++ Agent traffic fails on tool calls, which that probe never makes, so it can+ pass while every chat turn still fails.+ """+ _probes(monkeypatch, chat=PLAIN_COMPLETION, embedding=PLAIN_COMPLETION)+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 503+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"+++async def test_a_validation_that_called_no_model_clears_nothing(monkeypatch, _healthy_probe):+ """Azure's deployment listing, or a role with no model set, makes no model call."""+ _probes(monkeypatch, chat=NO_CALL, embedding=NO_CALL)+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")+ provider_error_log.record_failure("rhoai", "embedding", "embeddings fell over")++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 503+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"+ assert provider_error_log.latest_failure("rhoai", "embedding") == "embeddings fell over"+++async def test_a_model_free_check_proves_too_little_to_clear_anything(_healthy_probe):+ """Without a completion there is no evidence the provider can serve a call."""+ provider_error_log.record_failure("rhoai", "chat", "tool_calls are not iterable")++ response = await provider_health.check_provider_health(test_completion=False, user=None)++ assert response.status_code == 503+ assert provider_error_log.latest_failure("rhoai", "chat") == "tool_calls are not iterable"+++async def test_a_failing_probe_leaves_the_recorded_failure_in_place(monkeypatch, _healthy_probe):+ """A provider that is genuinely broken keeps saying so, in its own words."""++ async def boom(**kwargs):+ if kwargs.get("llm_model"):+ raise RuntimeError("endpoint unreachable")+ return PLAIN_COMPLETION++ monkeypatch.setattr(provider_health, "validate_provider_setup", boom)+ provider_error_log.record_failure("rhoai", "chat", "the real traffic failure")++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 503+ # A real call still beats a probe when both have something to say.+ assert provider_error_log.latest_failure("rhoai", "chat") == "the real traffic failure"diff --git a/tests/unit/api/test_provider_health_stale_spaces.py b/tests/unit/api/test_provider_health_stale_spaces.pynew file mode 100644index 000000000..f37d5dea9--- /dev/null+++ b/tests/unit/api/test_provider_health_stale_spaces.py@@ -0,0 +1,446 @@+"""A vector space outlives the model that made it.++A chunk records the space it was embedded in (`provider:model`), retrieval+embeds the query once per space found in the corpus, and an `InferenceService`+redeployed under a new `--served-model-name` leaves every earlier chunk+pointing at a model that is gone. Nothing else in the health path looks at the+corpus: the *configured* model is still fine, so every probe passes and the+banner stays green while those documents quietly drop out of vector search.+These pin that it gets said instead — as a warning beside the verdict, not as a+provider failure, because the provider is serving and its setup is not what+needs fixing.+"""++from __future__ import annotations++import json+from types import SimpleNamespace++import pytest++from api import provider_health+from api.provider_validation import ProbeResult+from services import provider_error_log++SERVED = "redhataigranite-embedding-engl"+STALE = "granite-embedding-english-r2"++[email protected](autouse=True)+def _clean():+ provider_error_log.clear()+ yield+ provider_error_log.clear()+++def _config():+ providers = SimpleNamespace(+ credential_values=lambda *_a, **_k: {},+ stored_credentials=lambda *_a, **_k: {},+ get_provider_config=lambda *_a, **_k: SimpleNamespace(api_key="k"),+ )+ return SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="gpt-oss-120b"),+ knowledge=SimpleNamespace(embedding_provider="rhoai", embedding_model=SERVED),+ get_llm_provider_config=lambda: SimpleNamespace(api_key="k"),+ get_embedding_provider_config=lambda: SimpleNamespace(api_key="k"),+ )+++def _healthy_probe(monkeypatch):+ """Everything the probes touch passes, so only the corpus can speak."""+ monkeypatch.setattr(provider_health, "get_openrag_config", _config)+ monkeypatch.setattr(provider_health, "is_known_provider", lambda _p: True)++ async def ok(**_kwargs):+ return ProbeResult()++ monkeypatch.setattr(provider_health, "validate_provider_setup", ok)+ monkeypatch.setattr(provider_health.provider_health_cache, "cache_key", lambda **_k: "key")+ monkeypatch.setattr(provider_health.provider_health_cache, "get", lambda _k: None)+ monkeypatch.setattr(+ provider_health.provider_health_cache, "set_and_release", lambda *_a, **_k: None+ )+ monkeypatch.setattr(provider_health.provider_health_cache, "release_error", lambda *_a: None)++ async def acquire(_key):+ return True++ monkeypatch.setattr(provider_health.provider_health_cache, "acquire", acquire)++ async def no_refresh(_provider):+ return None++ monkeypatch.setattr(provider_health, "_refresh_live_models", no_refresh)+++def _served(monkeypatch, models):+ import enhancements.providers.registry as registry++ monkeypatch.setattr(registry, "live_models_for", lambda _provider, _kind: models)+++def _indexed(monkeypatch, models):+ async def spaces(_provider):+ return models++ monkeypatch.setattr(provider_health, "_indexed_spaces", spaces)+++def _body(response):+ return json.loads(response.body)+++async def test_a_space_the_endpoint_no_longer_serves_is_a_warning_not_a_failure(+ monkeypatch, _healthy_probe+):+ _served(monkeypatch, (SERVED,))+ _indexed(monkeypatch, (SERVED, STALE))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ # The provider is serving; only the corpus is out of date.+ assert response.status_code == 200+ body = _body(response)+ assert body["status"] == "healthy"+ [warning] = body["warnings"]+ assert warning["code"] == provider_health.STALE_EMBEDDING_SPACE+ assert warning["provider"] == "rhoai"+ assert warning["models"] == [STALE]+ assert warning["served"] == [SERVED]+ assert "re-ingested or deleted" in warning["message"]+++async def test_the_warning_says_search_degrades_rather_than_fails(monkeypatch, _healthy_probe):+ """Retrieval skips a space it cannot embed for; it does not fail the search."""+ _served(monkeypatch, (SERVED,))+ _indexed(monkeypatch, (STALE,))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ message = _body(response)["warnings"][0]["message"]+ assert "keyword only" in message+ assert "will fail" not in message+++async def test_a_corpus_that_matches_what_is_served_says_nothing(monkeypatch, _healthy_probe):+ _served(monkeypatch, (SERVED,))+ _indexed(monkeypatch, (SERVED,))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert _body(response)["warnings"] == []+++async def test_an_unknown_listing_is_not_evidence_of_a_stale_corpus(monkeypatch, _healthy_probe):+ """An unreachable cluster must not read as every document being stale."""+ _served(monkeypatch, None)+ _indexed(monkeypatch, (STALE,))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert _body(response)["warnings"] == []+++async def test_a_provider_serving_no_embedding_model_makes_every_space_stale(+ monkeypatch, _healthy_probe+):+ """An empty listing is an answer, not an unknown: the provider listed+ itself and serves nothing to embed a query with, so every indexed space+ has stopped being searchable by meaning."""+ _served(monkeypatch, ())+ _indexed(monkeypatch, (SERVED, STALE))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ [warning] = _body(response)["warnings"]+ assert warning["code"] == provider_health.STALE_EMBEDDING_SPACE+ assert warning["models"] == [SERVED, STALE]+ assert warning["served"] == []+ assert "lists no embedding models" in warning["message"]+ assert "it serves: )" not in warning["message"]+++async def test_an_empty_listing_with_an_empty_corpus_says_nothing(monkeypatch, _healthy_probe):+ _served(monkeypatch, ())+ _indexed(monkeypatch, None)++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert _body(response)["warnings"] == []+++async def test_an_empty_corpus_is_not_evidence_of_drift(monkeypatch, _healthy_probe):+ _served(monkeypatch, (SERVED,))+ _indexed(monkeypatch, None)++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 200+ assert _body(response)["warnings"] == []+++async def test_a_real_failure_keeps_its_own_words_and_the_warning_rides_beside_it(+ monkeypatch, _healthy_probe+):+ async def boom(**kwargs):+ if kwargs.get("embedding_model"):+ raise RuntimeError("endpoint unreachable")+ return ProbeResult()++ monkeypatch.setattr(provider_health, "validate_provider_setup", boom)+ _served(monkeypatch, (SERVED,))+ _indexed(monkeypatch, (STALE,))++ response = await provider_health.check_provider_health(test_completion=True, user=None)++ assert response.status_code == 503+ body = _body(response)+ # The failure is reported as itself, not diluted by corpus advice...+ assert body["embedding_error"] == "endpoint unreachable"+ # ...and the corpus advice is still there for when the provider recovers.+ assert body["warnings"][0]["models"] == [STALE]+++# --------------------------------------------------------------------------+# Refreshing what the provider serves+# --------------------------------------------------------------------------+++def _refreshes(monkeypatch):+ """Record which providers the catalogue is asked to re-list."""+ import services.model_catalog as model_catalog++ asked: list[str | None] = []++ async def refresh(provider=None):+ asked.append(provider)++ monkeypatch.setattr(model_catalog, "refresh_live_models", refresh)+ return asked+++async def test_only_the_checked_provider_is_refreshed(monkeypatch, _refreshes):+ import enhancements.providers.registry as registry++ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace(fetch_models=lambda _c: None))++ await provider_health._refresh_live_models("rhoai")++ assert _refreshes == ["rhoai"]+++async def test_a_provider_that_cannot_list_its_models_is_not_refreshed(monkeypatch, _refreshes):+ """openai and friends have no enhancement: nothing to re-list, no call made."""+ import enhancements.providers.registry as registry++ monkeypatch.setattr(registry, "get", lambda _p: None)+ await provider_health._refresh_live_models("openai")++ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace())+ await provider_health._refresh_live_models("listless")++ assert _refreshes == []+++async def test_a_slow_cluster_cannot_stall_the_health_check(monkeypatch):+ import asyncio++ import enhancements.providers.registry as registry+ import services.model_catalog as model_catalog++ monkeypatch.setattr(registry, "get", lambda _p: SimpleNamespace(fetch_models=lambda _c: None))++ async def hang(provider=None):+ await asyncio.sleep(60)++ monkeypatch.setattr(model_catalog, "refresh_live_models", hang)+ monkeypatch.setattr(provider_health, "_MODEL_REFRESH_TIMEOUT_SECONDS", 0.01)++ # Returns (quietly) instead of raising or waiting out the listing.+ await asyncio.wait_for(provider_health._refresh_live_models("rhoai"), timeout=1)+++async def test_the_health_check_refreshes_the_embedding_provider(monkeypatch, _healthy_probe):+ asked: list[str] = []++ async def refresh(provider):+ asked.append(provider)++ monkeypatch.setattr(provider_health, "_refresh_live_models", refresh)+ _served(monkeypatch, None)++ await provider_health.check_provider_health(test_completion=True, user=None)++ assert asked == ["rhoai"]+++# --------------------------------------------------------------------------+# Reading the corpus+# --------------------------------------------------------------------------+++def _aggregation(*space_ids):+ return {+ "aggregations": {+ "embedding_spaces": {+ "buckets": [{"key": {"space_id": space_id}} for space_id in space_ids]+ }+ }+ }+++def _opensearch(monkeypatch):+ """Swap the admin client for one that replays a canned aggregation."""++ def install(result, *, fails=False):+ class _Client:+ async def search(self, **_kwargs):+ if fails:+ raise RuntimeError("opensearch is down")+ return result++ monkeypatch.setattr(+ "config.settings.clients", SimpleNamespace(opensearch=_Client()), raising=False+ )+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents", raising=False)++ return install+++def _page(*space_ids, after=None, **extra):+ """One composite-aggregation page; `after` is the cursor it hands back."""+ result = _aggregation(*space_ids)+ if after is not None:+ result["aggregations"]["embedding_spaces"]["after_key"] = {"space_id": after}+ result.update(extra)+ return result+++def _paged_opensearch(monkeypatch):+ """An admin client that serves `pages` in order and records each request."""++ def install(*pages):+ requests: list[dict] = []+ remaining = list(pages)++ class _Client:+ async def search(self, **kwargs):+ requests.append(kwargs)+ return remaining.pop(0)++ monkeypatch.setattr(+ "config.settings.clients", SimpleNamespace(opensearch=_Client()), raising=False+ )+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents", raising=False)+ return requests++ return install+++def _after(request):+ return request["body"]["aggs"]["embedding_spaces"]["composite"].get("after")+++async def test_indexed_spaces_follows_the_cursor_past_the_first_page(+ monkeypatch, _paged_opensearch+):+ """A stale space on page two is still a stale space."""+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_PAGE_SIZE", 2)+ requests = _paged_opensearch(+ _page("openai:text-embedding-3-small", f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"),+ _page(f"rhoai:{STALE}"),+ )++ assert await provider_health._indexed_spaces("rhoai") == (SERVED, STALE)+ assert [_after(request) for request in requests] == [None, {"space_id": f"rhoai:{SERVED}"}]+++async def test_indexed_spaces_stops_at_a_short_page(_paged_opensearch):+ """The last page still carries an `after_key`; following it is a wasted read."""+ requests = _paged_opensearch(_page(f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"))++ assert await provider_health._indexed_spaces("rhoai") == (SERVED,)+ assert len(requests) == 1+++async def test_indexed_spaces_refuses_partial_results(_paged_opensearch):+ """A failed shard must raise, not hand back fewer buckets as if complete."""+ requests = _paged_opensearch(_page(f"rhoai:{SERVED}"))++ await provider_health._indexed_spaces("rhoai")++ assert requests[0]["params"] == {"allow_partial_search_results": "false"}+++async def test_indexed_spaces_is_unknown_when_the_read_timed_out(_paged_opensearch):+ _paged_opensearch(_page(f"rhoai:{SERVED}", timed_out=True))++ assert await provider_health._indexed_spaces("rhoai") is None+++async def test_indexed_spaces_is_unknown_past_the_page_limit(monkeypatch, _paged_opensearch):+ """Running out of pages before the cursor does is not knowing, not a full list."""+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_PAGE_SIZE", 1)+ monkeypatch.setattr(provider_health, "_EMBEDDING_SPACE_MAX_PAGES", 2)+ _paged_opensearch(+ _page(f"rhoai:{SERVED}", after=f"rhoai:{SERVED}"),+ _page(f"rhoai:{STALE}", after=f"rhoai:{STALE}"),+ )++ assert await provider_health._indexed_spaces("rhoai") is None+++async def test_indexed_spaces_returns_this_providers_models(_opensearch):+ _opensearch(_aggregation(f"rhoai:{SERVED}", f"rhoai:{STALE}"))++ assert await provider_health._indexed_spaces("rhoai") == (SERVED, STALE)+++async def test_indexed_spaces_ignores_other_providers(_opensearch):+ """A corpus can hold spaces from a provider that is not the one selected."""+ _opensearch(_aggregation("openai:text-embedding-3-small", f"rhoai:{SERVED}"))++ assert await provider_health._indexed_spaces("rhoai") == (SERVED,)+++async def test_indexed_spaces_is_unknown_when_opensearch_cannot_answer(_opensearch):+ _opensearch(None, fails=True)++ assert await provider_health._indexed_spaces("rhoai") is None+++async def test_indexed_spaces_is_unknown_when_nothing_is_indexed(_opensearch):+ _opensearch(_aggregation())++ assert await provider_health._indexed_spaces("rhoai") is Nonediff --git a/tests/unit/api/test_provider_validation_probe_result.py b/tests/unit/api/test_provider_validation_probe_result.pynew file mode 100644index 000000000..3182d26ed--- /dev/null+++ b/tests/unit/api/test_provider_validation_probe_result.py@@ -0,0 +1,152 @@+"""`validate_provider_setup` says what a passing validation actually exercised.++The health check erases a real-traffic failure only on evidence as strong as+that failure, so each branch has to report honestly: a deployment listing, a+key check, or a missing model is not a model call, and the LiteLLM probe's+plain completion is not a tool-calling one.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from api import provider_validation+from api.provider_validation import ProbeResult, validate_provider_setup+++def calls(monkeypatch):+ """Stub every probe the validator can reach, recording which one ran."""+ ran: list[str] = []++ def stub(name):+ async def probe(*_args, **_kwargs):+ ran.append(name)++ return probe++ for name in (+ "_test_litellm_provider",+ "test_lightweight_health",+ "test_embedding",+ "test_completion_with_tools",+ ):+ monkeypatch.setattr(provider_validation, name, stub(name))+ monkeypatch.setattr(provider_validation, "get_provider_enhancement", lambda _p: None)+ return ran+++async def test_a_native_tool_calling_completion_is_reported_as_one(calls):+ result = await validate_provider_setup(+ provider="openai", api_key="k", llm_model="gpt-4o", test_completion=True+ )++ assert calls == ["test_completion_with_tools"]+ assert result == ProbeResult(model_probed=True, tools_exercised=True)+++async def test_a_native_embedding_call_probes_the_model(calls):+ result = await validate_provider_setup(+ provider="openai",+ api_key="k",+ embedding_model="text-embedding-3-small",+ test_completion=True,+ )++ assert calls == ["test_embedding"]+ assert result == ProbeResult(model_probed=True)+++async def test_the_litellm_probe_calls_the_model_without_tools(monkeypatch, calls):+ """RHOAI and every other enhancement: a plain `Reply with OK.` completion."""+ monkeypatch.setattr(+ provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()+ )+ monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})++ result = await validate_provider_setup(+ provider="rhoai", llm_model="granite", test_completion=True, credentials={}+ )++ assert calls == ["_test_litellm_provider"]+ assert result == ProbeResult(model_probed=True, tools_exercised=False)+++def enhancement(monkeypatch):+ monkeypatch.setattr(+ provider_validation, "get_provider_enhancement", lambda _p: SimpleNamespace()+ )+ monkeypatch.setattr("enhancements.providers.registry.runtime_kwargs_for", lambda *_a, **_k: {})++[email protected]("model", [{"llm_model": "granite"}, {"embedding_model": "granite-emb"}])+async def test_a_polled_enhancement_check_calls_no_model(calls, enhancement, model):+ """The 30s health poll must not run inference on RHOAI / watsonx.ai on-prem.++ It held a GPU slot (or billed) on every poll, and its pooled connections+ raced vLLM's 5s keep-alive into "Server disconnected" banners.+ """+ result = await validate_provider_setup(provider="rhoai", credentials={}, **model)++ assert calls == ["test_lightweight_health"]+ assert result == ProbeResult()+++async def test_a_save_still_probes_the_enhancement_model(calls, enhancement):+ """`verify_model` keeps the pre-save check that rejects an unserved model."""+ result = await validate_provider_setup(+ provider="rhoai", llm_model="granite", credentials={}, verify_model=True+ )++ assert calls == ["_test_litellm_provider"]+ assert result == ProbeResult(model_probed=True, tools_exercised=False)+++async def test_verify_model_does_not_change_a_native_provider(calls):+ """Native providers keep their own lightweight check on save."""+ result = await validate_provider_setup(+ provider="openai", api_key="k", llm_model="gpt-4o", verify_model=True+ )++ assert calls == ["test_lightweight_health"]+ assert result == ProbeResult()+++async def test_azure_lists_deployments_and_calls_no_model(calls):+ result = await validate_provider_setup(+ provider="azure",+ api_key="k",+ endpoint="https://example.openai.azure.com",+ llm_model="my-deployment",+ test_completion=True,+ )++ assert calls == ["test_lightweight_health"]+ assert result == ProbeResult()+++async def test_no_model_means_no_call(calls):+ """Azure after #2384 leaves the model empty; nothing is probed then."""+ result = await validate_provider_setup(provider="openai", api_key="k", test_completion=True)++ assert calls == []+ assert result == ProbeResult()+++async def test_a_lightweight_check_calls_no_model(calls):+ result = await validate_provider_setup(provider="openai", api_key="k", llm_model="gpt-4o")++ assert calls == ["test_lightweight_health"]+ assert result == ProbeResult()diff --git a/tests/unit/api/test_settings_endpoints.py b/tests/unit/api/test_settings_endpoints.pyindex 95a9f107a..8acb27f34 100644--- a/tests/unit/api/test_settings_endpoints.py+++ b/tests/unit/api/test_settings_endpoints.py@@ -355,6 +355,53 @@ async def test_update_settings_validates_enhancement_credentials_before_saving()save.assert_not_called()+ "fields",+ [+ {"llm_provider": "rhoai", "llm_model": "granite-chat"},+ {"embedding_provider": "rhoai", "embedding_model": "granite-embedding"},+ ],+ ids=["llm", "embedding"],+)+async def test_update_settings_asks_for_a_real_model_probe(fields):+ """A model switch is verified for real, not with the polled lightweight check.++ For a provider enhancement only a real call rejects a model the cluster+ does not serve; without `verify_model` the validator would run the+ model-free check and the switch would save, failing later in chat.+ """+ from api.settings.endpoints import update_settings+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict({})+ config.edited = True+ rbac = MagicMock()+ rbac.has_permission = AsyncMock(return_value=True)++ with (+ patch("api.settings.endpoints.get_openrag_config", return_value=config),+ patch(+ "api.settings.endpoints.validate_provider_setup",+ new_callable=AsyncMock,+ side_effect=Exception("stop after validation arguments are captured"),+ ) as validate,+ patch("api.settings.endpoints.config_manager.save_config_file") as save,+ ):+ response = await update_settings(+ body=SettingsUpdateBody(**fields),+ session_manager=AsyncMock(),+ user=MagicMock(spec=User),+ models_service=MagicMock(),+ rbac=rbac,+ )++ assert response.status_code == 400+ validate.assert_awaited_once()+ assert validate.await_args.kwargs["verify_model"] is True+ save.assert_not_called()++@pytest.mark.asyncioasync def test_update_settings_skips_pre_save_check_for_litellm_only_providers():"""Providers without an enhancement have no model-free probe to run."""@@ -742,3 +789,75 @@ async def test_onboarding_caps_chunk_size_for_watsonx_onprem_embeddings():saved_config = save_config.call_args.args[0]assert saved_config.knowledge.chunk_size == 500assert response.chunk_size_adjusted_to == 500+++async def _save_settings(body: SettingsUpdateBody) -> None:+ from api.settings.endpoints import update_settings+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict({})+ config.edited = True+ rbac = MagicMock()+ rbac.has_permission = AsyncMock(return_value=True)++ with (+ patch("api.settings.endpoints.get_openrag_config", return_value=config),+ patch("api.settings.endpoints.validate_provider_setup", new_callable=AsyncMock),+ patch("api.settings.endpoints.config_manager.save_config_file", return_value=True),+ patch("api.settings.endpoints.clients.refresh_patched_client", new_callable=AsyncMock),+ patch(+ "api.settings.endpoints._run_async_post_save_langflow_updates",+ new_callable=AsyncMock,+ ),+ ):+ response = await update_settings(+ body=body,+ session_manager=AsyncMock(),+ user=MagicMock(spec=User),+ models_service=MagicMock(),+ rbac=rbac,+ )+ assert getattr(response, "status_code", 200) == 200+++ "body",+ [+ SettingsUpdateBody(llm_model="qwen2.5-1.5b-instruct"),+ SettingsUpdateBody(+ provider_credentials={+ "openai_like": {"api_base": "https://llm.example", "api_key": "k"}+ }+ ),+ ],+ ids=["model", "credentials"],+)+async def test_provider_settings_save_clears_recorded_failures(body):+ """A recorded failure describes the setup the save just replaced.++ Without this, a provider whose probe cannot clear the banner kept showing+ the old failure after the operator fixed the model, until it went stale.+ """+ from services import provider_error_log++ provider_error_log.clear()+ provider_error_log.record_failure("rhoai", "chat", "old failure")+ try:+ await _save_settings(body)+ assert provider_error_log.latest_failure("rhoai", "chat") is None+ finally:+ provider_error_log.clear()+++async def test_non_provider_settings_save_keeps_recorded_failures():+ from services import provider_error_log++ provider_error_log.clear()+ provider_error_log.record_failure("rhoai", "chat", "still failing")+ try:+ await _save_settings(SettingsUpdateBody(system_prompt="Be brief."))+ assert provider_error_log.latest_failure("rhoai", "chat") == "still failing"+ finally:+ provider_error_log.clear()diff --git a/tests/unit/api/test_v1_llm.py b/tests/unit/api/test_v1_llm.pyindex 19576494d..278a39c37 100644--- a/tests/unit/api/test_v1_llm.py+++ b/tests/unit/api/test_v1_llm.py@@ -2,6 +2,7 @@import inspectimport json+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -124,6 +125,36 @@ async def test_embeddings_delegates_to_gateway(monkeypatch):assert data["data"][0]["embedding"] == [0.2]+ ("purpose", "interactive"),+ [("chat", True), ("ingest", False), (None, False)],+)+async def test_embeddings_lane_follows_the_hop_token_purpose(monkeypatch, purpose, interactive):+ gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})+ monkeypatch.setattr(v1_llm, "embeddings", gateway)+ request = MagicMock()+ request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})+ request.state = SimpleNamespace(llm_hop_purpose=purpose)++ await v1_llm.embeddings_endpoint(request, user=_user())++ assert gateway.await_args.kwargs == {"interactive": interactive}+++async def test_embeddings_without_a_hop_token_take_the_bulk_lane(monkeypatch):+ gateway = AsyncMock(return_value={"object": "list", "data": [{"embedding": [0.2]}]})+ monkeypatch.setattr(v1_llm, "embeddings", gateway)+ request = MagicMock()+ request.json = AsyncMock(return_value={"model": "rhoai:granite", "input": ["x"]})+ request.state = SimpleNamespace() # API key / JWT: no hop purpose recorded++ await v1_llm.embeddings_endpoint(request, user=_user())++ assert gateway.await_args.kwargs == {"interactive": False}++@pytest.mark.asyncioasync def test_list_openai_models_hides_catalog_exception_text(monkeypatch):"""CodeQL py/stack-trace-exposure: the exception text must not reach the caller."""diff --git a/tests/unit/config/test_cors_allowed_origins.py b/tests/unit/config/test_cors_allowed_origins.pyindex 2496ca2ce..949d5b3f7 100644--- a/tests/unit/config/test_cors_allowed_origins.py+++ b/tests/unit/config/test_cors_allowed_origins.py@@ -64,8 +64,10 @@ def test_empty_string_yields_empty_list():def test_unset_defaults_to_localhost():env = _python_env({"PYTHONPATH": str(SRC)})env.pop("CORS_ALLOWED_ORIGINS", None)+ # bootstrap.load_env() would otherwise re-read a developer's local .env.+ no_dotenv = "import dotenv; dotenv.load_dotenv = lambda *a, **k: False; "result = subprocess.run(- [sys.executable, "-c", PRINT_ORIGINS],+ [sys.executable, "-c", no_dotenv + PRINT_ORIGINS],capture_output=True,text=True,check=True,diff --git a/tests/unit/config/test_generic_provider_config.py b/tests/unit/config/test_generic_provider_config.pyindex 50743217d..981bac317 100644--- a/tests/unit/config/test_generic_provider_config.py+++ b/tests/unit/config/test_generic_provider_config.py@@ -145,6 +145,7 @@ def _clear_rhoai_env(monkeypatch):"RHOAI_EMBEDDINGS_ENDPOINT","RHOAI_API_KEY","RHOAI_TLS_VERIFY",+ "RHOAI_EMBEDDING_MAX_CONCURRENCY","OPENAI_API_KEY","LLM_PROVIDER","EMBEDDING_PROVIDER",@@ -230,6 +231,78 @@ def test_rhoai_tls_env_alone_reaches_a_stored_config(monkeypatch, tmp_path):}+def test_rhoai_embedding_concurrency_env_is_seeded(monkeypatch, tmp_path):+ """The bulkhead limit ships with the rest of the provider's first-boot seed."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")+ monkeypatch.setenv("RHOAI_EMBEDDING_MAX_CONCURRENCY", "8")++ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ assert config.providers.custom["rhoai"].configured is True+ assert config.providers.stored_credentials("rhoai")["embedding_max_concurrency"] == "8"+++def test_rhoai_env_credentials_accessor_maps_env_names_to_fields(monkeypatch):+ """Each RHOAI_* variable lands under the credential field the enhancement reads."""+ from config.settings import get_rhoai_env_credentials++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")+ monkeypatch.setenv("RHOAI_EMBEDDING_MAX_CONCURRENCY", "6")++ assert get_rhoai_env_credentials() == {+ "api_base": "https://chat.svc:8443/v1",+ "embedding_api_base": "https://embed.svc:8443/v1",+ "api_key": "sha256~token",+ "ssl_verify": "/var/run/secrets/service-ca.crt",+ "embedding_max_concurrency": "6",+ }+++def test_rhoai_seed_is_read_through_settings_boundary(monkeypatch, tmp_path):+ """Seeding goes through `config.settings`, and stores values unvalidated: an+ invalid concurrency limit is the enhancement's to reject (default + warning)."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setattr(+ "config.settings.get_rhoai_env_credentials",+ lambda: {+ "api_base": "https://chat.svc:8443/v1",+ "embedding_api_base": None,+ "api_key": "sha256~token",+ "ssl_verify": None,+ "embedding_max_concurrency": "not-a-number",+ },+ )++ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ assert config.providers.custom["rhoai"].configured is True+ assert config.providers.stored_credentials("rhoai") == {+ "api_base": "https://chat.svc:8443/v1",+ "api_key": "sha256~token",+ "embedding_max_concurrency": "not-a-number",+ }+++def test_rhoai_is_not_seeded_when_no_env_is_set(monkeypatch, tmp_path):+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)++ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ assert "rhoai" not in config.providers.custom++def _clear_watsonx_onprem_env(monkeypatch):for name in ("WATSONX_ENDPOINT_ONPREM",diff --git a/tests/unit/config/test_langflow_request_retry.py b/tests/unit/config/test_langflow_request_retry.pynew file mode 100644index 000000000..517eaad32--- /dev/null+++ b/tests/unit/config/test_langflow_request_retry.py@@ -0,0 +1,191 @@+"""`AppClients.langflow_request` retries only what is safe to repeat.++Regression: a folder ingest failed on a real 502 from the embedding endpoint,+Langflow answered the flow run with 500, and the backend replayed+`POST /api/v1/run/{flow}`. The replay re-ran the Docling component against a+task docling-serve had already deleted, and its 404 replaced the real error.+"""++import httpx+import pytest++import config.settings as settings+from config.settings import AppClients++RUN_ENDPOINT = "/api/v1/run/ingest-flow"+++class _FakeClient:+ """Replays a scripted sequence of responses or transport errors."""++ def __init__(self, outcomes):+ self.outcomes = list(outcomes)+ self.calls: list[tuple[str, str]] = []++ async def request(self, *, method, url, headers, **kwargs):+ self.calls.append((method, url))+ outcome = self.outcomes.pop(0)+ if isinstance(outcome, Exception):+ raise outcome+ return httpx.Response(outcome, request=httpx.Request(method, url))+++def clients(monkeypatch):+ async def api_key(force_regenerate: bool = False):+ return "lf-key"++ async def no_sleep(_delay):+ return None++ monkeypatch.setattr(settings, "get_langflow_api_key", api_key)+ monkeypatch.setattr(settings, "LANGFLOW_REQUEST_RETRIES", 2)+ monkeypatch.setattr(settings.asyncio, "sleep", no_sleep)+ return AppClients()+++def _install(clients, monkeypatch, outcomes) -> _FakeClient:+ fake = _FakeClient(outcomes)+ monkeypatch.setattr(clients, "_ensure_langflow_http_client", lambda: fake)+ return fake+++def _request_error(cls):+ return cls("boom", request=httpx.Request("POST", "http://langflow"))+++async def test_a_failed_flow_run_is_not_replayed(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [500, 200])++ response = await clients.langflow_request("POST", RUN_ENDPOINT, json={})++ assert response.status_code == 500+ assert len(fake.calls) == 1++[email protected]("status", [408, 500, 502, 504])+async def test_post_is_not_retried_on_statuses_after_processing(clients, monkeypatch, status):+ fake = _install(clients, monkeypatch, [status, 200])++ response = await clients.langflow_request("POST", RUN_ENDPOINT)++ assert response.status_code == status+ assert len(fake.calls) == 1++[email protected]("status", [429, 503])+async def test_post_is_retried_when_the_server_did_not_process_it(clients, monkeypatch, status):+ fake = _install(clients, monkeypatch, [status, 200])++ response = await clients.langflow_request("POST", RUN_ENDPOINT)++ assert response.status_code == 200+ assert len(fake.calls) == 2++[email protected]("status", [408, 500, 502, 503, 504])+async def test_get_is_retried_on_transient_statuses(clients, monkeypatch, status):+ fake = _install(clients, monkeypatch, [status, status, 200])++ response = await clients.langflow_request("GET", "/api/v1/flows/x")++ assert response.status_code == 200+ assert len(fake.calls) == 3+++async def test_retries_stop_at_the_configured_budget(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [502, 502, 502, 200])++ response = await clients.langflow_request("GET", "/api/v1/flows/x")++ assert response.status_code == 502+ assert len(fake.calls) == 3+++async def test_patch_is_not_retried_by_default(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [500, 200])++ response = await clients.langflow_request("PATCH", "/api/v1/mcp/project/p")++ assert response.status_code == 500+ assert len(fake.calls) == 1++[email protected]("method", ["POST", "PATCH"])+async def test_an_explicitly_idempotent_request_is_retried(clients, monkeypatch, method):+ fake = _install(clients, monkeypatch, [500, 200])++ response = await clients.langflow_request(method, "/api/v1/flows/x", idempotent=True)++ assert response.status_code == 200+ assert len(fake.calls) == 2+++async def test_an_explicitly_non_idempotent_get_is_not_retried(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [500, 200])++ response = await clients.langflow_request("GET", "/api/v1/flows/x", idempotent=False)++ assert response.status_code == 500+ assert len(fake.calls) == 1++[email protected]("error_cls", [httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout])+async def test_post_is_retried_when_it_never_reached_langflow(clients, monkeypatch, error_cls):+ fake = _install(clients, monkeypatch, [_request_error(error_cls), 200])++ response = await clients.langflow_request("POST", RUN_ENDPOINT)++ assert response.status_code == 200+ assert len(fake.calls) == 2+++ "error_cls", [httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.ReadError]+)+async def test_post_is_not_retried_after_it_may_have_run(clients, monkeypatch, error_cls):+ fake = _install(clients, monkeypatch, [_request_error(error_cls), 200])++ with pytest.raises(error_cls):+ await clients.langflow_request("POST", RUN_ENDPOINT)++ assert len(fake.calls) == 1+++async def test_get_is_retried_after_a_read_timeout(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [_request_error(httpx.ReadTimeout), 200])++ response = await clients.langflow_request("GET", "/api/v1/flows/x")++ assert response.status_code == 200+ assert len(fake.calls) == 2+++async def test_auth_refresh_still_applies_to_post(clients, monkeypatch):+ fake = _install(clients, monkeypatch, [401, 200])++ response = await clients.langflow_request("POST", RUN_ENDPOINT)++ assert response.status_code == 200+ assert len(fake.calls) == 2+++def test_idempotency_follows_rfc_9110_by_default():+ for method in ("GET", "head", "OPTIONS", "PUT", "DELETE"):+ assert settings._is_idempotent_request(method, None) is True+ for method in ("POST", "PATCH", "patch"):+ assert settings._is_idempotent_request(method, None) is False+ assert settings._is_idempotent_request("POST", True) is True+ assert settings._is_idempotent_request("GET", False) is Falsediff --git a/tests/unit/dependencies/test_llm_proxy_user_purpose.py b/tests/unit/dependencies/test_llm_proxy_user_purpose.pynew file mode 100644index 000000000..dda507a04--- /dev/null+++ b/tests/unit/dependencies/test_llm_proxy_user_purpose.py@@ -0,0 +1,51 @@+"""`get_llm_proxy_user` records which kind of Langflow run minted a hop token.++The embeddings route reads it to give chat (query) embeddings their own lane+beside bulk ingestion at a concurrency-limited provider.+"""++from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++import dependencies+from services.langflow_llm_token_service import HOP_PURPOSE_CHAT, LangflowLlmTokenService++SECRET = "llm-hop-test-secret-with-32-bytes!!"+++def _request(token: str) -> SimpleNamespace:+ return SimpleNamespace(headers={"authorization": f"Bearer {token}"}, state=SimpleNamespace())++[email protected]("purpose", [HOP_PURPOSE_CHAT, None])+async def test_a_hop_token_records_its_purpose(purpose):+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)+ request = _request(service.create_token(user_id="alice", purpose=purpose))++ user = await dependencies.get_llm_proxy_user(+ request, token_service=service, api_key_service=None, session_manager=None+ )++ assert user.user_id == "alice"+ assert user.provider == "langflow_llm"+ assert request.state.user is user+ assert request.state.llm_hop_purpose == purpose+++async def test_a_non_hop_token_records_no_purpose(monkeypatch):+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)+ api_user = SimpleNamespace(user_id="bob")+ fallback = AsyncMock(return_value=api_user)+ monkeypatch.setattr(dependencies, "resolve_api_key_user", fallback)+ request = _request("not-a-hop-token")++ user = await dependencies.get_llm_proxy_user(+ request, token_service=service, api_key_service=None, session_manager=None+ )++ assert user is api_user+ assert not hasattr(request.state, "llm_hop_purpose")diff --git a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py b/tests/unit/enhancements/providers/redhat/test_openshift_ai.pyindex 0fb891931..ac084eedb 100644--- a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py+++ b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py@@ -98,11 +98,19 @@ def test_the_form_asks_for_two_endpoints_a_token_and_a_ca() -> None:"""vLLM serves one model per InferenceService, so one endpoint is not enough."""fields = {field["key"]: field for field in model_catalog.credential_fields(PROVIDER)}- assert set(fields) == {"api_base", "embedding_api_base", "api_key", "ssl_verify"}+ assert set(fields) == {+ "api_base",+ "embedding_api_base",+ "api_key",+ "ssl_verify",+ "embedding_max_concurrency",+ }assert fields["api_base"]["field_type"] == "text"assert fields["embedding_api_base"]["field_type"] == "text"assert fields["api_key"]["field_type"] == "password"assert fields["ssl_verify"]["field_type"] == "text"+ assert fields["embedding_max_concurrency"]["field_type"] == "text"+ assert fields["embedding_max_concurrency"]["required"] is Falsedef test_only_the_token_is_encrypted_at_rest() -> None:@@ -147,6 +155,67 @@ def test_the_embedding_url_never_reaches_litellm(kind) -> None:assert "embedding_api_base" not in rhoai.litellm_credentials(_stored(), kind=kind)[email protected]("kind", ["chat", "embedding"])+def test_the_concurrency_limit_never_reaches_litellm(kind) -> None:+ """The gateway enforces the limit; forwarded, it would land in the request body."""+ stored = _stored(embedding_max_concurrency="2")+ assert "embedding_max_concurrency" not in rhoai.litellm_credentials(stored, kind=kind)+++# ---------------------------------------------------------------------------+# Embedding concurrency limit+# ---------------------------------------------------------------------------+++ ("value", "expected"),+ [+ (None, rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),+ ("", rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),+ (" ", rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY),+ ("1", 1),+ (" 16 ", 16),+ (8, 8),+ ("0", None),+ ],+)+def test_the_embedding_concurrency_limit_is_parsed(value, expected) -> None:+ stored = _stored(embedding_max_concurrency=value)+ assert rhoai.embedding_max_concurrency(stored) == expected++[email protected]("value", ["four", "-1", "2.5"])+def test_an_invalid_limit_falls_back_to_the_default_not_to_unlimited(value, monkeypatch) -> None:+ """Failing open would reintroduce the overload the limit exists to prevent."""+ warnings: list[str] = []+ monkeypatch.setattr(rhoai.logger, "warning", lambda message, **_: warnings.append(message))++ stored = _stored(embedding_max_concurrency=value)+ assert rhoai.embedding_max_concurrency(stored) == rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY+ assert rhoai.embedding_max_concurrency(stored) == rhoai.DEFAULT_EMBEDDING_MAX_CONCURRENCY+ assert len(warnings) == 1+++def test_the_registry_exposes_the_limit() -> None:+ assert registry.embedding_concurrency_for(PROVIDER, _stored(embedding_max_concurrency="3")) == 3+ assert (+ registry.embedding_concurrency_for(PROVIDER, _stored(embedding_max_concurrency="0")) is None+ )+++def test_the_registry_reports_no_limit_for_providers_without_the_hook() -> None:+ assert registry.embedding_concurrency_for("openai", {}) is None+ assert registry.embedding_concurrency_for("watsonx_onprem", {}) is None+++def test_a_failing_hook_never_breaks_the_request(monkeypatch) -> None:+ def broken(stored):+ raise RuntimeError("boom")++ monkeypatch.setattr(rhoai, "embedding_max_concurrency", broken)+ assert registry.embedding_concurrency_for(PROVIDER, _stored()) is None++def test_no_endpoint_means_no_credentials_at_all() -> None:"""Not tidiness: LiteLLM falls back to HOSTED_VLLM_API_BASE from the processenvironment when `api_base` is absent, so returning a bare token could send a@@ -524,10 +593,17 @@ def test_a_kindless_enhancement_is_still_callable() -> None:class _Response:- def __init__(self, status_code: int, payload: Any = None, text: str = "") -> None:+ def __init__(+ self,+ status_code: int,+ payload: Any = None,+ text: str = "",+ headers: dict[str, str] | None = None,+ ) -> None:self.status_code = status_codeself._payload = payloadself.text = text+ self.headers = headers or {}def json(self) -> Any:if self._payload is None:@@ -987,6 +1063,181 @@ async def test_a_rejected_token_names_the_rolebinding(monkeypatch, status) -> Noawait rhoai.lightweight_health_check(_stored())+LOGIN_PAGE = (+ "<!DOCTYPE html>\n<html><head><title>Log in</title></head>"+ "<body><form action='/oauth/start'>Log in with OpenShift</form></body></html>"+)+LOGIN_REDIRECT = (+ "https://oauth-openshift.apps.cluster.example/oauth/authorize"+ "?client_id=system%3Aserviceaccount&state=abc123&redirect_uri=https%3A%2F%2Fx"+)+++def _serve(monkeypatch, responses: dict[str, Any]) -> None:+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses))+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)+++async def test_a_login_page_answered_with_200_is_not_healthy(monkeypatch) -> None:+ """An OAuth proxy or SSO gateway in front of the route answers 200 with its+ login page; reading that as healthy defers the failure to the first chat."""+ _serve(+ monkeypatch,+ {+ f"{CHAT_BASE}/models": _Response(+ 200, text=LOGIN_PAGE, headers={"content-type": "text/html; charset=utf-8"}+ )+ },+ )++ with pytest.raises(Exception, match="chat endpoint .* returned a web page") as error:+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))++ assert "text/html" in str(error.value)+ assert "<html" not in str(error.value).lower()+++async def test_a_login_page_without_a_content_type_is_still_recognised(monkeypatch) -> None:+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(200, text=f" \n{LOGIN_PAGE}")})++ with pytest.raises(Exception, match="returned a web page"):+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))+++ "body",+ [{"status": "ok"}, [], {"data": "not-a-list"}, "ok"],+ ids=["status-object", "bare-list", "data-not-list", "bare-string"],+)+async def test_a_200_that_is_not_a_model_listing_is_not_healthy(monkeypatch, body) -> None:+ _serve(+ monkeypatch,+ {f"{CHAT_BASE}/models": _Response(200, body, headers={"content-type": "application/json"})},+ )++ with pytest.raises(Exception, match="OpenAI-compatible model list"):+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))+++async def test_an_empty_model_listing_still_passes(monkeypatch) -> None:+ """Only the shape is checked: a vLLM pod still loading its weights is a+ real endpoint, and the check does not depend on a model being chosen."""+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(200, _models_body())})++ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))+++async def test_a_login_page_on_the_embedding_endpoint_names_that_endpoint(monkeypatch) -> None:+ _serve(+ monkeypatch,+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(+ 200, text=LOGIN_PAGE, headers={"content-type": "text/html"}+ ),+ },+ )++ with pytest.raises(Exception, match="embeddings endpoint .* returned a web page"):+ await rhoai.lightweight_health_check(_stored())++[email protected]("status", [301, 302, 303, 307, 308])+async def test_a_redirect_to_a_login_page_says_so(monkeypatch, status) -> None:+ """httpx does not follow redirects, so an OAuth proxy's 302 arrives as-is;+ the message names where it pointed, without the OAuth query noise."""+ _serve(+ monkeypatch,+ {+ f"{CHAT_BASE}/models": _Response(+ status,+ text=LOGIN_PAGE,+ headers={"content-type": "text/html", "location": LOGIN_REDIRECT},+ )+ },+ )++ with pytest.raises(Exception, match="redirected to") as error:+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))++ message = str(error.value)+ assert "https://oauth-openshift.apps.cluster.example/oauth/authorize" in message+ assert "state=" not in message+ assert "<html" not in message.lower()+++async def test_a_redirect_without_a_location_is_still_reported(monkeypatch) -> None:+ _serve(monkeypatch, {f"{CHAT_BASE}/models": _Response(302)})++ with pytest.raises(Exception, match="redirected to another page"):+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))+++async def test_an_html_error_page_is_described_not_quoted(monkeypatch) -> None:+ """The router's own 403 page is HTML; quoting it would put markup in the+ Settings error. The status-specific advice still applies."""+ _serve(+ monkeypatch,+ {+ f"{CHAT_BASE}/models": _Response(+ 403, text=LOGIN_PAGE, headers={"content-type": "text/html"}+ )+ },+ )++ with pytest.raises(Exception, match="inferenceservices") as error:+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))++ assert "HTML page" in str(error.value)+ assert "<html" not in str(error.value).lower()+++ "response",+ [+ _Response(200, text=LOGIN_PAGE, headers={"content-type": "text/html"}),+ _Response(200, {"status": "ok"}),+ _Response(302, headers={"location": LOGIN_REDIRECT}),+ ],+ ids=["login-page", "wrong-shape", "redirect"],+)+async def test_a_proxy_answer_is_not_mistaken_for_a_bad_token(monkeypatch, response) -> None:+ """The fix is the URL, not the token; a credential classification would+ tell the operator to rotate a token that works."""+ from api.provider_validation import is_provider_credential_error++ _serve(monkeypatch, {f"{CHAT_BASE}/models": response})++ with pytest.raises(Exception) as error:+ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))++ assert not is_provider_credential_error(error.value)+++ ("body", "expected"),+ [+ ({"object": "list", "data": []}, True),+ (_models_body(CHAT_MODEL), True),+ ({"data": None}, False),+ ({"object": "list"}, False),+ ([], False),+ (None, False),+ ],+)+def test_a_model_listing_is_recognised_by_its_shape(body, expected) -> None:+ assert rhoai.is_model_listing(body) is expected++@pytest.mark.asyncioasync def test_an_unreachable_endpoint_says_which_one(monkeypatch) -> None:import httpxdiff --git a/tests/unit/enhancements/providers/test_registry_live_models.py b/tests/unit/enhancements/providers/test_registry_live_models.pynew file mode 100644index 000000000..c55a9a312--- /dev/null+++ b/tests/unit/enhancements/providers/test_registry_live_models.py@@ -0,0 +1,142 @@+"""What a provider says it serves, read back from its own cache.++`live_models_for` is the half of model-drift detection that must never guess:+a provider that cannot list itself, or has not listed itself yet, is *unknown*,+and a caller that treats unknown as "serves nothing" would tell an operator to+change a setting that was right all along.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from enhancements.providers import registry+++class _Listing(SimpleNamespace):+ """Stands in for `ClusterModels`: one attribute per endpoint."""+++def _enhancement(monkeypatch, module):+ monkeypatch.setitem(registry._ENHANCEMENTS, "probe", module)+++def test_a_provider_with_no_enhancement_is_unknown():+ assert registry.live_models_for("openai", "chat") is None+ assert registry.live_models_for("", "chat") is None+++def test_an_enhancement_that_cannot_list_its_models_is_unknown(monkeypatch):+ _enhancement(monkeypatch, SimpleNamespace(PROVIDER_KEY="probe"))++ assert registry.live_models_for("probe", "chat") is None+++def test_nothing_cached_yet_is_unknown(monkeypatch):+ _enhancement(monkeypatch, SimpleNamespace(cached_models=lambda: None))++ assert registry.live_models_for("probe", "embedding") is None+++def test_a_cached_listing_is_split_by_endpoint(monkeypatch):+ _enhancement(+ monkeypatch,+ SimpleNamespace(+ cached_models=lambda: _Listing(+ chat=("gpt-oss-120b",), embedding=("redhataigranite-embedding-engl",)+ )+ ),+ )++ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)+ assert registry.live_models_for("probe", "embedding") == ("redhataigranite-embedding-engl",)+++def test_one_endpoint_can_be_known_while_the_other_is_not(monkeypatch):+ """A rolling embedding deployment must not make the chat list unknown too."""+ _enhancement(+ monkeypatch,+ SimpleNamespace(cached_models=lambda: _Listing(chat=("gpt-oss-120b",), embedding=None)),+ )++ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)+ assert registry.live_models_for("probe", "embedding") is None+++def test_the_provider_key_is_matched_the_way_it_is_stored(monkeypatch):+ _enhancement(+ monkeypatch,+ SimpleNamespace(cached_models=lambda: _Listing(chat=("m",), embedding=("m",))),+ )++ assert registry.live_models_for(" PROBE ", "chat") == ("m",)+++def test_a_listing_that_raises_is_unknown_rather_than_fatal(monkeypatch):+ def boom():+ raise RuntimeError("cache exploded")++ _enhancement(monkeypatch, SimpleNamespace(cached_models=boom))++ assert registry.live_models_for("probe", "chat") is None+++def test_a_known_empty_half_is_an_answer_not_unknown(monkeypatch):+ """A provider that listed itself and serves no embedding model has said so;+ folding that into unknown would hide every indexed space going stale."""+ _enhancement(+ monkeypatch,+ SimpleNamespace(cached_models=lambda: _Listing(chat=("gpt-oss-120b",), embedding=())),+ )++ assert registry.live_models_for("probe", "embedding") == ()+ assert registry.live_models_for("probe", "chat") == ("gpt-oss-120b",)+++def test_a_cached_list_is_handed_back_as_a_tuple(monkeypatch):+ _enhancement(+ monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=["a", "b"], embedding=[]))+ )++ assert registry.live_models_for("probe", "chat") == ("a", "b")+ assert registry.live_models_for("probe", "embedding") == ()++[email protected]("listed", ["gpt-oss-120b", b"gpt-oss-120b", 42])+def test_a_malformed_half_is_unknown_not_split_into_nonsense(monkeypatch, listed):+ """A bare string would iterate into one-letter model ids."""+ _enhancement(+ monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=listed, embedding=()))+ )++ assert registry.live_models_for("probe", "chat") is None+++def test_a_half_the_listing_does_not_have_is_unknown(monkeypatch):+ _enhancement(monkeypatch, SimpleNamespace(cached_models=lambda: _Listing(chat=("m",))))++ assert registry.live_models_for("probe", "embedding") is None+++def test_watsonx_onprem_with_no_embedding_model_reports_an_empty_half(monkeypatch):+ """The reviewer's case end to end: the on-prem enhancement caches a+ successful listing whose embedding half is empty, and that must survive+ the registry as `()`."""+ import time++ from enhancements.providers.watsonx import onprem++ monkeypatch.setattr(+ onprem,+ "_models_cache",+ {+ "key": "k",+ "at": time.monotonic(),+ "value": onprem.ClusterModels(chat=("ibm/granite-3-8b-instruct",), embedding=()),+ },+ )++ assert registry.live_models_for(onprem.PROVIDER_KEY, "embedding") == ()+ assert registry.live_models_for(onprem.PROVIDER_KEY, "chat") == ("ibm/granite-3-8b-instruct",)diff --git a/tests/unit/enhancements/providers/watsonx/test_onprem.py b/tests/unit/enhancements/providers/watsonx/test_onprem.pyindex 1861c806d..930397531 100644--- a/tests/unit/enhancements/providers/watsonx/test_onprem.py+++ b/tests/unit/enhancements/providers/watsonx/test_onprem.py@@ -719,7 +719,11 @@ async def test_switching_to_a_model_the_cluster_lacks_is_still_blocked(monkeypat)with pytest.raises(Exception, match="not supported"):await provider_validation.validate_provider_setup(- provider=PROVIDER, credentials=credentials, llm_model="ibm/nope", embedding_model=None+ provider=PROVIDER,+ credentials=credentials,+ llm_model="ibm/nope",+ embedding_model=None,+ verify_model=True,)assert probes == ["real-call"], "a model in hand must be probed for real"@@ -747,6 +751,7 @@ async def test_switching_an_embedding_model_is_probed_for_real(monkeypatch) -> Ncredentials=credentials,llm_model=None,embedding_model="ibm/slate-125m-english-rtrvr-v2",+ verify_model=True,)assert probes == ["watsonx/ibm/slate-125m-english-rtrvr-v2"]@@ -1005,6 +1010,7 @@ async def test_validating_an_embedding_model_sends_a_list_not_a_string(monkeypat),llm_model=None,embedding_model="ibm/slate-30m-english-rtrvr",+ verify_model=True,)assert isinstance(sent["input"], list), sent["input"]diff --git a/tests/unit/services/test_langflow_llm_token_service.py b/tests/unit/services/test_langflow_llm_token_service.pyindex 25338ffd8..2c54ea089 100644--- a/tests/unit/services/test_langflow_llm_token_service.py+++ b/tests/unit/services/test_langflow_llm_token_service.py@@ -10,6 +10,8 @@ import pytestfrom services.document_index_writer import DocumentIndexContextfrom services.langflow_ingest_token_service import LangflowIngestTokenServicefrom services.langflow_llm_token_service import (+ HOP_PURPOSE_CHAT,+ HOP_PURPOSE_INGEST,LANGFLOW_LLM_AUDIENCE,LangflowLlmTokenService,langflow_hop_audience,@@ -84,3 +86,20 @@ def test_wrong_scope_is_rejected():def test_orag_key_is_not_a_hop_token():assert langflow_hop_audience("orag_abc") is Noneassert langflow_hop_audience("") is None++[email protected]("purpose", [HOP_PURPOSE_CHAT, HOP_PURPOSE_INGEST])+def test_the_purpose_claim_round_trips(purpose):+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)+ token = service.create_token(user_id="alice", purpose=purpose)+ claims = service.validate_claims(token)+ assert claims["purpose"] == purpose+ assert service.user_from_claims(claims).user_id == "alice"+ # The user it identifies is the same either way.+ assert service.validate_token(token).provider == "langflow_llm"+++def test_a_token_minted_without_a_purpose_has_no_claim():+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)+ claims = service.validate_claims(service.create_token(user_id="alice"))+ assert "purpose" not in claimsdiff --git a/tests/unit/services/test_langflow_mcp_service.py b/tests/unit/services/test_langflow_mcp_service.pyindex 76ef6c7a7..b10640b49 100644--- a/tests/unit/services/test_langflow_mcp_service.py+++ b/tests/unit/services/test_langflow_mcp_service.py@@ -124,5 +124,6 @@ async def test_patch_mcp_server_url_preserves_auth_settings(monkeypatch):"url": "http://langflow:7860/api/v1/mcp/project/project-id/streamable","headers": expected_headers,},+ "idempotent": True,}]diff --git a/tests/unit/services/test_llm_gateway.py b/tests/unit/services/test_llm_gateway.pyindex 67b4d166c..cefbaab08 100644--- a/tests/unit/services/test_llm_gateway.py+++ b/tests/unit/services/test_llm_gateway.py@@ -576,7 +576,21 @@ async def test_stream_sse_counts_tool_calls_as_usable_output(monkeypatch):async def gen():yield {- "choices": [{"delta": {"tool_calls": [{"index": 0}]}, "finish_reason": "tool_calls"}]+ "choices": [+ {+ "delta": {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_1",+ "type": "function",+ "function": {"name": "search_documents", "arguments": "{}"},+ }+ ]+ },+ "finish_reason": "tool_calls",+ }+ ]}[line async for line in llm_gateway._stream_sse(gen(), "openai", "gpt-4o-mini")]@@ -1020,6 +1034,542 @@ async def test_stream_sse_logs_the_repair(monkeypatch):assert fields["tool_calls"] == 1+# --------------------------------------------------------------------------+# Tool calls a provider splits across the wrong index+# --------------------------------------------------------------------------+++def _orphaned_fragment_deltas():+ """The gpt-oss-on-vLLM symptom: the closing brace lands under the next index.++ Taken literally this truncates the real call's arguments *and* invents a+ second call with no id and no name — the exact pair that makes the assistant+ message the agent replays unacceptable to vLLM on every later turn.+ """+ opening = {+ "id": "chatcmpl-tool-1",+ "type": "function",+ "index": 0,+ "function": {"name": "search_documents", "arguments": ""},+ }+ fragments = ['{\n "query": "AMCOR', ' signatories"\n']+ chunks = [{"choices": [{"index": 0, "delta": {"tool_calls": [opening]}}]}]+ chunks += [+ {+ "choices": [+ {"index": 0, "delta": {"tool_calls": [{"index": 0, "function": {"arguments": f}}]}}+ ]+ }+ for f in fragments+ ]+ # The stray one: same content, wrong index, and nothing naming a call.+ chunks.append(+ {+ "choices": [+ {+ "index": 0,+ "delta": {"tool_calls": [{"index": 1, "function": {"arguments": "}"}}]},+ }+ ]+ }+ )+ chunks.append({"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]})+ return chunks+++async def test_stream_sse_rejoins_a_fragment_misfiled_under_the_next_index():+ from services import llm_gateway++ async def gen():+ for chunk in _orphaned_fragment_deltas():+ yield chunk++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ calls = _streamed_tool_calls(lines)+ assert len(calls) == 1, "the stray fragment must not become a second tool call"+ assert calls[0]["id"] == "chatcmpl-tool-1"+ assert calls[0]["function"]["name"] == "search_documents"+ # The closing brace is back where it belongs, so the arguments parse.+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "AMCOR signatories"}+++async def test_stream_sse_drops_a_buffered_call_that_names_no_tool(monkeypatch):+ """Backstop: a call with no name can never be executed, so it must not ship."""+ from services import llm_gateway++ recorder = _RecordingLogger()+ monkeypatch.setattr(llm_gateway, "logger", recorder)++ async def gen():+ yield {+ "choices": [+ {+ "index": 0,+ "delta": {+ "tool_calls": [+ {"id": "call_1", "index": 0, "function": {"arguments": "{}"}}+ ]+ },+ "finish_reason": "tool_calls",+ }+ ]+ }++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ assert _streamed_tool_calls(lines) == []+ assert any(call[0] == "warning" and "named no tool" in call[1] for call in recorder.calls)+++def _tool_call_chunk(*tool_calls, finish_reason=None):+ choice = {"index": 0, "delta": {"tool_calls": list(tool_calls)}}+ if finish_reason:+ choice["finish_reason"] = finish_reason+ return {"choices": [choice]}+++async def test_stream_sse_keeps_an_arguments_fragment_that_arrives_before_the_call_is_named():+ """Nothing is open yet, so the fragment must wait for its call, not vanish."""+ from services import llm_gateway++ async def gen():+ yield _tool_call_chunk({"index": 0, "function": {"arguments": '{"query": '}})+ yield _tool_call_chunk(+ {+ "id": "call_1",+ "type": "function",+ "index": 0,+ "function": {"name": "search_documents", "arguments": '"x"}'},+ }+ )+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ calls = _streamed_tool_calls(lines)+ assert len(calls) == 1+ assert calls[0]["id"] == "call_1"+ assert calls[0]["function"]["name"] == "search_documents"+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "x"}+++async def test_stream_sse_rejoins_both_a_leading_and_a_misfiled_trailing_fragment():+ """The provisional call must not undo the next-index rejoin, and vice versa."""+ from services import llm_gateway++ async def gen():+ yield _tool_call_chunk({"index": 0, "function": {"arguments": "{"}})+ yield _tool_call_chunk(+ {+ "id": "call_1",+ "type": "function",+ "index": 0,+ "function": {"name": "search_documents", "arguments": '"query": "x"'},+ }+ )+ yield _tool_call_chunk({"index": 1, "function": {"arguments": "}"}})+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ calls = _streamed_tool_calls(lines)+ assert len(calls) == 1, "neither stray fragment may become a call of its own"+ assert calls[0]["id"] == "call_1"+ assert json.loads(calls[0]["function"]["arguments"]) == {"query": "x"}+++async def test_stream_sse_drops_an_unnamed_fragment_no_call_ever_claims(monkeypatch):+ """A provisional call that never gets a name is still discarded at drain."""+ from services import llm_gateway++ recorder = _RecordingLogger()+ monkeypatch.setattr(llm_gateway, "logger", recorder)++ async def gen():+ yield _tool_call_chunk({"index": 0, "function": {"arguments": "{}"}})+ yield {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ assert _streamed_tool_calls(lines) == []+ assert any(call[0] == "warning" and "named no tool" in call[1] for call in recorder.calls)+ # Nothing reached the client, so it is told why rather than left hanging.+ assert any('"code": "upstream_error"' in line for line in lines)+ assert lines[-1] == "data: [DONE]\n\n"+++async def test_stream_sse_mints_an_id_for_a_named_call_that_has_none():+ """A named call is real work; only its correlation handle is missing."""+ from services import llm_gateway++ async def gen():+ yield {+ "choices": [+ {+ "index": 0,+ "delta": {+ "tool_calls": [+ {+ "index": 0,+ "function": {"name": "search_documents", "arguments": "{}"},+ }+ ]+ },+ "finish_reason": "tool_calls",+ }+ ]+ }++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ calls = _streamed_tool_calls(lines)+ assert len(calls) == 1+ assert isinstance(calls[0]["id"], str) and calls[0]["id"]++[email protected]("fragments", [[], [" "]], ids=["no-fragment", "whitespace-fragment"])+async def test_stream_sse_defaults_a_zero_argument_call_to_an_empty_object(fragments):+ """`""` is not a JSON object; clients that parse arguments must get `{}`."""+ from services import llm_gateway++ async def gen():+ for chunk in _tool_call_deltas(fragments, name="list_sources"):+ yield chunk++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "hosted_vllm/gpt-oss")]++ calls = _streamed_tool_calls(lines)+ assert len(calls) == 1+ assert calls[0]["id"] == "call_1"+ assert calls[0]["function"]["name"] == "list_sources"+ assert json.loads(calls[0]["function"]["arguments"]) == {}+++def test_finalise_tool_call_leaves_real_arguments_alone():+ from services.llm_gateway import _finalise_tool_call++ call = {"id": "call_1", "function": {"name": "search", "arguments": _WELL_FORMED_ARGUMENTS}}+ assert _finalise_tool_call(call) is True+ assert call["function"]["arguments"] == _WELL_FORMED_ARGUMENTS+++def test_finalise_tool_call_does_not_default_arguments_on_a_nameless_call():+ from services.llm_gateway import _finalise_tool_call++ call = {"id": "call_1", "function": {"name": "", "arguments": ""}}+ assert _finalise_tool_call(call) is False+ assert call["function"]["arguments"] == ""+++# --------------------------------------------------------------------------+# A poisoned conversation must not wedge every later turn+# --------------------------------------------------------------------------+++def test_sanitise_messages_drops_a_tool_call_naming_no_tool_and_its_result():+ """vLLM rejects the whole request over this one message, so it cannot go up."""+ from services import llm_gateway++ messages = [+ {"role": "user", "content": "For the AMCOR form, who are the signatories?"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "type": "function",+ "id": "call_1",+ "function": {"name": "search_documents", "arguments": '{"query": "AMCOR"}'},+ },+ # What langchain-openai echoes back for an unparseable call.+ {"type": "function", "id": None, "function": {"name": "", "arguments": "}"}},+ ],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": "…"},+ {"role": "tool", "tool_call_id": None, "content": "…"},+ ]++ cleaned, repairs = llm_gateway._sanitise_messages(messages)++ assert repairs == 2+ assert [m["role"] for m in cleaned] == ["user", "assistant", "tool"]+ assert [c["id"] for c in cleaned[1]["tool_calls"]] == ["call_1"]+++def test_sanitise_messages_encodes_object_arguments_as_the_json_string_openai_requires():+ from services import llm_gateway++ cleaned, repairs = llm_gateway._sanitise_messages(+ [+ {+ "role": "assistant",+ "tool_calls": [+ {+ "type": "function",+ "id": "call_1",+ "function": {"name": "search", "arguments": {"query": "AMCOR"}},+ }+ ],+ }+ ]+ )++ assert repairs == 1+ assert cleaned[0]["tool_calls"][0]["function"]["arguments"] == '{"query": "AMCOR"}'+++def test_sanitise_messages_removes_a_tool_calls_key_left_empty():+ """`tool_calls: null` is itself a shape providers iterate without a None check."""+ from services import llm_gateway++ cleaned, repairs = llm_gateway._sanitise_messages(+ [{"role": "assistant", "content": "hello", "tool_calls": None}]+ )++ assert "tool_calls" not in cleaned[0]+ assert cleaned[0]["content"] == "hello"+ assert repairs == 0++[email protected]("function", ["search", 42, ["name", "search"]])+def test_sanitise_messages_drops_a_call_whose_function_is_not_an_object(function):+ """Malformed history is the sanitiser's job to absorb, not to crash on."""+ from services import llm_gateway++ good = {"type": "function", "id": "call_1", "function": {"name": "search", "arguments": "{}"}}+ cleaned, repairs = llm_gateway._sanitise_messages(+ [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [good, {"type": "function", "id": "call_2", "function": function}],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": "kept"},+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},+ ]+ )++ assert repairs == 2+ assert [call["id"] for call in cleaned[0]["tool_calls"]] == ["call_1"]+ assert [m["content"] for m in cleaned[1:]] == ["kept"]+++async def test_chat_completions_absorbs_a_non_object_function_instead_of_raising(monkeypatch):+ """It runs before the call's error handling, so a raise here was a bare 500."""+ captured = {}++ async def fake_acompletion(**kwargs):+ captured.update(kwargs)+ return {"id": "1", "choices": [{"message": {"role": "assistant", "content": "hi"}}]}++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)++ await chat_completions(+ {+ "model": "gpt-4o-mini",+ "messages": [+ {"role": "user", "content": "hi"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"type": "function", "id": "call_1", "function": "oops"}],+ },+ ],+ },+ config=_config(),+ )++ assert captured["messages"] == [{"role": "user", "content": "hi"}]+++def test_sanitise_messages_drops_the_result_of_a_dropped_call_that_had_an_id():+ """The result carries a real id, but the call it answers is gone."""+ from services import llm_gateway++ cleaned, repairs = llm_gateway._sanitise_messages(+ [+ {"role": "user", "content": "q"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "type": "function",+ "id": "call_1",+ "function": {"name": "search", "arguments": "{}"},+ },+ {+ "type": "function",+ "id": "call_2",+ "function": {"name": "", "arguments": "{}"},+ },+ ],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": "kept"},+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},+ ]+ )++ assert repairs == 2+ assert [m.get("content") for m in cleaned if m["role"] == "tool"] == ["kept"]+++def test_sanitise_messages_drops_an_assistant_message_that_was_only_bad_calls():+ """`content` is optional only beside `tool_calls`; with neither, nothing is left."""+ from services import llm_gateway++ cleaned, repairs = llm_gateway._sanitise_messages(+ [+ {"role": "user", "content": "q"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "type": "function",+ "id": "call_2",+ "function": {"name": "", "arguments": "{}"},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "call_2", "content": "orphaned"},+ {"role": "user", "content": "again"},+ ]+ )++ assert repairs == 2+ assert cleaned == [{"role": "user", "content": "q"}, {"role": "user", "content": "again"}]+++def test_sanitise_messages_keeps_an_assistant_message_that_still_has_content():+ from services import llm_gateway++ cleaned, _ = llm_gateway._sanitise_messages(+ [+ {+ "role": "assistant",+ "content": "Let me look.",+ "tool_calls": [+ {+ "type": "function",+ "id": "call_2",+ "function": {"name": "", "arguments": "{}"},+ }+ ],+ }+ ]+ )++ assert cleaned == [{"role": "assistant", "content": "Let me look."}]+++def test_sanitise_messages_drops_a_result_not_answering_the_preceding_assistant_message():+ """A result only answers the calls of the assistant message just before it."""+ from services import llm_gateway++ call = {"type": "function", "id": "call_1", "function": {"name": "search", "arguments": "{}"}}+ cleaned, repairs = llm_gateway._sanitise_messages(+ [+ {"role": "assistant", "content": "", "tool_calls": [call]},+ {"role": "tool", "tool_call_id": "call_1", "content": "answer"},+ {"role": "user", "content": "next"},+ {"role": "tool", "tool_call_id": "call_1", "content": "replayed out of place"},+ ]+ )++ assert repairs == 1+ assert [m.get("content") for m in cleaned] == ["", "answer", "next"]+++def test_sanitise_messages_keeps_every_result_of_parallel_calls():+ from services import llm_gateway++ messages = [+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"type": "function", "id": "a", "function": {"name": "search", "arguments": "{}"}},+ {"type": "function", "id": "b", "function": {"name": "search", "arguments": "{}"}},+ ],+ },+ {"role": "tool", "tool_call_id": "b", "content": "second"},+ {"role": "tool", "tool_call_id": "a", "content": "first"},+ ]++ cleaned, repairs = llm_gateway._sanitise_messages(messages)++ assert repairs == 0+ assert cleaned == messages+++def test_sanitise_messages_leaves_a_healthy_conversation_untouched():+ from services import llm_gateway++ messages = [+ {"role": "system", "content": "be helpful"},+ {"role": "user", "content": "hi"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "type": "function",+ "id": "call_1",+ "function": {"name": "search", "arguments": '{"query": "x"}'},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": "result"},+ ]++ cleaned, repairs = llm_gateway._sanitise_messages(messages)++ assert repairs == 0+ assert cleaned == messages+++async def test_chat_completions_sends_the_sanitised_conversation_upstream(monkeypatch):+ """The repair has to reach the wire, not just exist as a helper."""+ captured = {}++ async def fake_acompletion(**kwargs):+ captured.update(kwargs)+ return {"id": "1", "choices": [{"message": {"role": "assistant", "content": "hi"}}]}++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)++ await chat_completions(+ {+ "model": "gpt-4o-mini",+ "messages": [+ {"role": "user", "content": "hi"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"type": "function", "id": None, "function": {"name": "", "arguments": "}"}}+ ],+ },+ ],+ },+ config=_config(),+ )++ # The assistant message held nothing but the unusable call, so it goes too.+ assert captured["messages"] == [{"role": "user", "content": "hi"}]++# --------------------------------------------------------------------------# Surfacing the real upstream failure# --------------------------------------------------------------------------@@ -1095,6 +1645,60 @@ async def test_chat_completions_masks_an_upstream_4xx_as_a_gateway_failure(monkeassert exc.value.status_code == 502+class _FakeMidStreamFallbackError(Exception):+ llm_provider = "hosted_vllm"+++_PROXY_CUTOFF = (+ "litellm.MidStreamFallbackError: litellm.APIConnectionError: APIConnectionError: "+ "Hosted_vllmException - Response payload is not completed: <TransferEncodingError: 400, "+ "message='Not enough data to satisfy transfer length header.'>"+)+++def test_a_truncated_response_names_the_proxy_not_the_wrapper_chain():+ """aiohttp's own "400" must not read as the model rejecting the request."""+ from services.llm_gateway import _upstream_client_message++ message = _upstream_client_message(+ f"MidStreamFallbackError: {_PROXY_CUTOFF}",+ "rhoai",+ "hosted_vllm/qwen2.5-0.5b-instruct",+ _FakeMidStreamFallbackError(_PROXY_CUTOFF),+ )++ assert "closed before the response finished" in message+ assert "timed out" in message+ assert "rhoai/qwen2.5-0.5b-instruct" in message+ for noise in ("MidStreamFallbackError", "TransferEncodingError", "400"):+ assert noise not in message+++async def test_a_mid_stream_cutoff_is_reported_and_still_latches_the_banner():+ """Unlike a context overflow, this fails every long request on that endpoint."""+ from services import llm_gateway, provider_error_log++ provider_error_log.clear()++ async def gen():+ raise _FakeMidStreamFallbackError(_PROXY_CUTOFF)+ yield # pragma: no cover - makes this an async generator++ try:+ lines = [+ line+ async for line in llm_gateway._stream_sse(+ gen(), "rhoai", "hosted_vllm/qwen2.5-0.5b-instruct"+ )+ ]+ error = json.loads(lines[-2][len("data: ") :])["error"]+ assert "closed before the response finished" in error["message"]+ assert provider_error_log.latest_failure("rhoai", "chat") == error["message"]+ finally:+ provider_error_log.clear()++def test_sanitise_upstream_detail_drops_interpreter_state():from services.llm_gateway import _sanitise_upstream_detail@@ -1273,6 +1877,99 @@ class TestRealFailuresReachTheBanner:assert provider_error_log.latest_failure("openai", "chat") is None+class TestRequestScopedFailuresStayOutOfTheBanner:+ """A prompt that overflows the context window is this request's problem.++ The provider is serving; the next, shorter prompt succeeds. Recording it+ latched the health banner against a healthy provider, and on one whose+ probe cannot clear it (every provider enhancement), it stayed latched.+ """++ _OVERFLOW = (+ "This model's maximum context length is 32768 tokens. However, you requested 0 "+ "output tokens and your prompt contains at least 32769 input tokens"+ )++ @pytest.fixture(autouse=True)+ def _clean(self, monkeypatch):+ from services import llm_gateway, provider_error_log++ provider_error_log.clear()+ monkeypatch.setattr(+ llm_gateway,+ "resolve_call",+ lambda *a, **k: ("hosted_vllm/qwen2.5-0.5b-instruct", "rhoai", {}),+ )+ yield+ provider_error_log.clear()++ def _raise(self, monkeypatch, exc: BaseException):+ async def _boom(**_kwargs):+ raise exc++ monkeypatch.setattr("litellm.acompletion", _boom)++ @pytest.mark.asyncio+ async def test_a_context_window_overflow_is_not_recorded(self, monkeypatch):+ from services import llm_gateway, provider_error_log++ class ContextWindowExceededError(Exception):+ llm_provider = "hosted_vllm"++ self._raise(monkeypatch, ContextWindowExceededError(self._OVERFLOW))++ with pytest.raises(llm_gateway.LlmGatewayError) as raised:+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})++ assert "maximum context length" in str(raised.value)+ assert provider_error_log.latest_failure("rhoai", "chat") is None++ @pytest.mark.asyncio+ async def test_an_overflow_does_not_erase_a_real_failure(self, monkeypatch):+ from services import llm_gateway, provider_error_log++ provider_error_log.record_failure("rhoai", "chat", "earlier real failure")+ self._raise(monkeypatch, RuntimeError(self._OVERFLOW))++ with pytest.raises(llm_gateway.LlmGatewayError):+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})++ assert provider_error_log.latest_failure("rhoai", "chat") == "earlier real failure"++ @pytest.mark.asyncio+ async def test_other_upstream_failures_are_still_recorded(self, monkeypatch):+ from services import llm_gateway, provider_error_log++ self._raise(+ monkeypatch,+ RuntimeError('{"error": {"message": "The model `qwen` does not exist."}}'),+ )++ with pytest.raises(llm_gateway.LlmGatewayError):+ await llm_gateway.chat_completions({"model": "rhoai:qwen", "messages": []})++ assert "does not exist" in provider_error_log.latest_failure("rhoai", "chat")++ @pytest.mark.asyncio+ async def test_a_mid_stream_overflow_is_reported_but_not_recorded(self):+ from services import llm_gateway, provider_error_log++ overflow = self._OVERFLOW++ class BadRequestError(Exception):+ llm_provider = "hosted_vllm"++ async def gen():+ raise BadRequestError(overflow)+ yield # pragma: no cover - makes this an async generator++ lines = [line async for line in llm_gateway._stream_sse(gen(), "rhoai", "qwen")]++ error = json.loads(lines[-2][len("data: ") :])["error"]+ assert "maximum context length" in error["message"]+ assert provider_error_log.latest_failure("rhoai", "chat") is None++class TestToolsBesideReasoningEffort:"""gpt-5.x refuses function tools next to its own default reasoning effort.diff --git a/tests/unit/services/test_llm_gateway_embedding_concurrency.py b/tests/unit/services/test_llm_gateway_embedding_concurrency.pynew file mode 100644index 000000000..375b43d3e--- /dev/null+++ b/tests/unit/services/test_llm_gateway_embedding_concurrency.py@@ -0,0 +1,539 @@+"""Per-provider embedding bulkhead and the opaque upstream 502/504 message.++A folder upload fans out to several Langflow ingest runs, each embedding one+chunk per request on eight threads. Against a slow (CPU-served) RHOAI model+the calls queued at the endpoint until its kube-rbac-proxy timed them out with+a body-less 502. The gateway now bounds concurrent embedding calls per+provider (limit supplied by the provider enhancement) and explains such a 502.++One slot of that limit is reserved for query embeddings (`interactive=True`:+search and chat retrieval) so a bulk ingest cannot queue them; the two lanes+split the limit rather than add to it.+"""++import asyncio+from types import SimpleNamespace++import pytest++from config.config_manager import (+ AnthropicConfig,+ GenericProviderConfig,+ OllamaConfig,+ OpenAIConfig,+ ProvidersConfig,+ WatsonXConfig,+)+from services import llm_gateway+from services.llm_gateway import LlmGatewayError, embeddings++RHOAI_MODEL = "rhoai:granite-embedding"+++def _rhoai_config(limit: str | None = None) -> SimpleNamespace:+ credentials = {+ "api_base": "https://chat.example.com/v1",+ "embedding_api_base": "https://embed.example.com/v1",+ "api_key": "token",+ }+ if limit is not None:+ credentials["embedding_max_concurrency"] = limit+ providers = ProvidersConfig(+ openai=OpenAIConfig(api_key="sk-openai", configured=True),+ anthropic=AnthropicConfig(),+ watsonx=WatsonXConfig(),+ ollama=OllamaConfig(),+ custom={"rhoai": GenericProviderConfig(credentials=credentials, configured=True)},+ )+ return SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_model="gpt-4o-mini", llm_provider="openai"),+ knowledge=SimpleNamespace(embedding_model="granite-embedding", embedding_provider="rhoai"),+ )+++class _InFlightTracker:+ """A fake `litellm.aembedding` that records peak concurrency."""++ def __init__(self, delay: float = 0.01):+ self.delay = delay+ self.in_flight = 0+ self.peak = 0+ self.calls: list[dict] = []++ async def __call__(self, **kwargs):+ self.calls.append(kwargs)+ self.in_flight += 1+ self.peak = max(self.peak, self.in_flight)+ try:+ await asyncio.sleep(self.delay)+ finally:+ self.in_flight -= 1+ return {+ "object": "list",+ "data": [{"object": "embedding", "embedding": [0.1], "index": 0}],+ "usage": {"prompt_tokens": 1, "total_tokens": 1},+ }++[email protected](autouse=True)+def _reset_limiters():+ llm_gateway._embedding_limiters.clear()+ yield+ llm_gateway._embedding_limiters.clear()+++async def _embed_concurrently(cfg, model: str, count: int, *, interactive=False) -> None:+ await asyncio.gather(+ *(+ embeddings({"model": model, "input": f"chunk {i}"}, config=cfg, interactive=interactive)+ for i in range(count)+ )+ )+++RHOAI = "rhoai"+++async def test_rhoai_embeddings_are_bounded_by_the_configured_limit(monkeypatch):+ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await _embed_concurrently(_rhoai_config("3"), RHOAI_MODEL, 10)++ assert len(tracker.calls) == 10+ # One of the three slots is held back for queries.+ assert tracker.peak == 2+++async def test_rhoai_embeddings_use_the_default_limit_when_unset(monkeypatch):+ from enhancements.providers.redhat.openshift_ai import DEFAULT_EMBEDDING_MAX_CONCURRENCY++ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await _embed_concurrently(_rhoai_config(), RHOAI_MODEL, 2 * DEFAULT_EMBEDDING_MAX_CONCURRENCY)++ assert tracker.peak == DEFAULT_EMBEDDING_MAX_CONCURRENCY - 1+++async def test_limit_zero_disables_the_bulkhead(monkeypatch):+ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 10)++ assert tracker.peak == 10+ assert RHOAI not in llm_gateway._embedding_limiters+++async def test_the_limit_never_reaches_litellm(monkeypatch):+ tracker = _InFlightTracker(delay=0)+ monkeypatch.setattr("litellm.aembedding", tracker)++ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config("3"))++ assert "embedding_max_concurrency" not in tracker.calls[0]+ assert tracker.calls[0]["api_base"] == "https://embed.example.com/v1"+++async def test_providers_without_an_enhancement_are_unbounded(monkeypatch):+ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await _embed_concurrently(_rhoai_config("1"), "openai:text-embedding-3-small", 6)++ assert tracker.peak == 6+ assert "openai" not in llm_gateway._embedding_limiters+++async def test_a_changed_limit_resizes_the_limiter_in_place():+ first = llm_gateway._embedding_limiter("rhoai", _rhoai_config("2"))+ same = llm_gateway._embedding_limiter("rhoai", _rhoai_config("2"))+ changed = llm_gateway._embedding_limiter("rhoai", _rhoai_config("5"))++ # One limiter for the provider's lifetime, so its in-flight count survives.+ assert first is same is changed+ assert llm_gateway._embedding_limiters[RHOAI].limit == 5+++async def test_a_cancelled_waiter_releases_its_slot(monkeypatch):+ gate = asyncio.Event()++ async def blocking_aembedding(**kwargs):+ await gate.wait()+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}++ monkeypatch.setattr("litellm.aembedding", blocking_aembedding)+ cfg = _rhoai_config("1")++ holder = asyncio.create_task(embeddings({"model": RHOAI_MODEL, "input": "a"}, config=cfg))+ waiter = asyncio.create_task(embeddings({"model": RHOAI_MODEL, "input": "b"}, config=cfg))+ await asyncio.sleep(0.01)+ waiter.cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter+ gate.set()+ await holder++ # Both slots are free again: a new call proceeds immediately.+ result = await asyncio.wait_for(+ embeddings({"model": RHOAI_MODEL, "input": "c"}, config=cfg), timeout=1+ )+ assert result["data"]+++async def test_a_failing_call_releases_its_slot(monkeypatch):+ async def boom(**kwargs):+ raise RuntimeError("boom")++ monkeypatch.setattr("litellm.aembedding", boom)+ cfg = _rhoai_config("1")+ for _ in range(3):+ with pytest.raises(LlmGatewayError):+ await asyncio.wait_for(+ embeddings({"model": RHOAI_MODEL, "input": "x"}, config=cfg), timeout=1+ )+++# --------------------------------------------------------------------------+# Interactive lane+# --------------------------------------------------------------------------+++class _Gate:+ """A fake `litellm.aembedding` whose calls block until released."""++ def __init__(self):+ self.release = asyncio.Event()+ self.in_flight = 0+ self.peak = 0+ self.started: list[str] = []++ async def __call__(self, **kwargs):+ self.started.append(kwargs["input"][0] if isinstance(kwargs["input"], list) else "")+ self.in_flight += 1+ self.peak = max(self.peak, self.in_flight)+ try:+ await self.release.wait()+ finally:+ self.in_flight -= 1+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}+++def _embed(cfg, text: str, *, interactive: bool = False) -> asyncio.Task:+ return asyncio.create_task(+ embeddings({"model": RHOAI_MODEL, "input": text}, config=cfg, interactive=interactive)+ )+++async def test_a_query_is_not_queued_behind_bulk_ingestion(monkeypatch):+ gate = _Gate()+ monkeypatch.setattr("litellm.aembedding", gate)+ cfg = _rhoai_config("4")++ bulk = [_embed(cfg, f"chunk {i}") for i in range(20)]+ await asyncio.sleep(0.01)+ assert gate.in_flight == 3 # the bulk lane is full, 17 chunks waiting++ query = _embed(cfg, "query", interactive=True)+ await asyncio.sleep(0.01)+ assert "query" in gate.started # started ahead of every waiting chunk+ assert gate.in_flight == 4++ gate.release.set()+ await asyncio.gather(query, *bulk)+ assert gate.peak == 4 # never more than the operator's limit in total+++async def test_queries_are_bounded_by_their_own_lane(monkeypatch):+ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await _embed_concurrently(_rhoai_config("4"), RHOAI_MODEL, 6, interactive=True)++ assert len(tracker.calls) == 6+ assert tracker.peak == 1+ assert llm_gateway._embedding_limiters[RHOAI].limit == 4+++async def test_a_limit_of_one_is_shared_by_queries_and_ingestion(monkeypatch):+ gate = _Gate()+ monkeypatch.setattr("litellm.aembedding", gate)+ cfg = _rhoai_config("1")++ chunk = _embed(cfg, "chunk")+ await asyncio.sleep(0.01)+ query = _embed(cfg, "query", interactive=True)+ await asyncio.sleep(0.01)++ # Splitting a single slot would exceed the limit, so the query waits.+ assert gate.started == ["chunk"]+ gate.release.set()+ await asyncio.gather(chunk, query)+ assert gate.peak == 1+++async def test_no_limit_throttles_neither_lane(monkeypatch):+ tracker = _InFlightTracker()+ monkeypatch.setattr("litellm.aembedding", tracker)++ await asyncio.gather(+ _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 5),+ _embed_concurrently(_rhoai_config("0"), RHOAI_MODEL, 5, interactive=True),+ )++ assert tracker.peak == 10+ assert not llm_gateway._embedding_limiters+++ ("limit", "interactive", "expected"),+ [+ (4, False, ("bulk", 3)),+ (4, True, ("interactive", 1)),+ (2, False, ("bulk", 1)),+ (2, True, ("interactive", 1)),+ (1, False, ("bulk", 1)),+ (1, True, ("bulk", 1)),+ ],+)+def test_lane_limits_split_the_configured_total(limit, interactive, expected):+ assert llm_gateway._lane_limit(limit, interactive) == expected+++# --------------------------------------------------------------------------+# Limit changes while calls are in flight+# --------------------------------------------------------------------------+++class _PerCallGate:+ """A fake `litellm.aembedding` whose calls are released one at a time."""++ def __init__(self):+ self.events: dict[str, asyncio.Event] = {}+ self.started: list[str] = []+ self.in_flight = 0++ def release(self, text: str) -> None:+ self.events.setdefault(text, asyncio.Event()).set()++ async def __call__(self, **kwargs):+ text = kwargs["input"][0]+ self.started.append(text)+ self.in_flight += 1+ try:+ await self.events.setdefault(text, asyncio.Event()).wait()+ finally:+ self.in_flight -= 1+ return {"object": "list", "data": [{"embedding": [0.1], "index": 0}]}+++async def _settle() -> None:+ await asyncio.sleep(0.01)+++async def test_a_lowered_limit_counts_calls_still_in_flight(monkeypatch):+ gate = _PerCallGate()+ monkeypatch.setattr("litellm.aembedding", gate)++ old = [_embed(_rhoai_config("3"), "old 1"), _embed(_rhoai_config("3"), "old 2")]+ await _settle()+ assert gate.in_flight == 2++ # Lowered to 2 (one bulk slot): the two old calls already exceed it.+ new = _embed(_rhoai_config("2"), "new")+ await _settle()+ assert "new" not in gate.started++ gate.release("old 1")+ await _settle()+ assert "new" not in gate.started # one old call still holds the bulk share++ gate.release("old 2")+ await _settle()+ assert "new" in gate.started+ gate.release("new")+ await asyncio.gather(*old, new)+++async def test_a_raised_limit_admits_queued_calls_at_once(monkeypatch):+ gate = _PerCallGate()+ monkeypatch.setattr("litellm.aembedding", gate)++ calls = [_embed(_rhoai_config("2"), f"chunk {i}") for i in range(3)]+ await _settle()+ assert gate.in_flight == 1 # bulk share of a limit of 2++ llm_gateway._embedding_limiter("rhoai", _rhoai_config("4"))+ await _settle()+ assert gate.in_flight == 3 # no call had to finish first++ for i in range(3):+ gate.release(f"chunk {i}")+ await asyncio.gather(*calls)+++async def test_lowering_to_one_counts_a_query_from_the_interactive_lane(monkeypatch):+ gate = _PerCallGate()+ monkeypatch.setattr("litellm.aembedding", gate)++ query = _embed(_rhoai_config("3"), "query", interactive=True)+ await _settle()++ chunk = _embed(_rhoai_config("1"), "chunk")+ await _settle()+ assert gate.started == ["query"]++ gate.release("query")+ await _settle()+ assert gate.started == ["query", "chunk"]+ gate.release("chunk")+ await asyncio.gather(query, chunk)+++async def test_turning_the_limit_off_and_on_keeps_counting(monkeypatch):+ gate = _PerCallGate()+ monkeypatch.setattr("litellm.aembedding", gate)++ first = _embed(_rhoai_config("2"), "first")+ await _settle()+ unbounded = [_embed(_rhoai_config("0"), f"free {i}") for i in range(2)]+ await _settle()+ assert gate.in_flight == 3 # "0" switches the limit off++ late = _embed(_rhoai_config("2"), "late")+ await _settle()+ assert "late" not in gate.started # the unbounded calls still count++ gate.release("first")+ gate.release("free 0")+ await _settle()+ assert "late" not in gate.started # total is 1 but the bulk share (1) is used+ gate.release("free 1")+ await _settle()+ assert "late" in gate.started+ gate.release("late")+ await asyncio.gather(first, *unbounded, late)+++async def test_a_limit_of_one_hands_the_slot_to_a_waiting_query_first(monkeypatch):+ gate = _PerCallGate()+ monkeypatch.setattr("litellm.aembedding", gate)+ cfg = _rhoai_config("1")++ calls = [_embed(cfg, "chunk 0")]+ await _settle()+ calls += [_embed(cfg, "chunk 1"), _embed(cfg, "chunk 2")]+ await _settle()+ calls.append(_embed(cfg, "query", interactive=True))+ await _settle()++ gate.release("chunk 0")+ await _settle()+ assert gate.started == ["chunk 0", "query"]++ for text in ("query", "chunk 1", "chunk 2"):+ gate.release(text)+ await asyncio.gather(*calls)+ assert gate.started == ["chunk 0", "query", "chunk 1", "chunk 2"]+++async def test_a_slot_granted_to_a_cancelled_waiter_is_handed_on():+ limiter = llm_gateway._EmbeddingLimiter("rhoai", 1, asyncio.get_running_loop())+ await limiter.acquire(llm_gateway._BULK_LANE)++ waiter = asyncio.create_task(limiter.acquire(llm_gateway._BULK_LANE))+ await _settle()+ limiter.release(llm_gateway._BULK_LANE) # grants the waiter's future...+ waiter.cancel() # ...but it is cancelled before it resumes+ with pytest.raises(asyncio.CancelledError):+ await waiter++ assert limiter.in_flight == 0+ await asyncio.wait_for(limiter.acquire(llm_gateway._BULK_LANE), timeout=1)+++async def test_an_extra_release_does_not_free_a_phantom_slot():+ limiter = llm_gateway._EmbeddingLimiter("rhoai", 1, asyncio.get_running_loop())+ limiter.release(llm_gateway._BULK_LANE)++ await limiter.acquire(llm_gateway._BULK_LANE)+ waiter = asyncio.create_task(limiter.acquire(llm_gateway._BULK_LANE))+ await _settle()+ assert not waiter.done() # the stray release did not leave a spare slot+ waiter.cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter+++# --------------------------------------------------------------------------+# Opaque upstream 502/504+# --------------------------------------------------------------------------+++class _FakeLiteLLMError(Exception):+ """Stands in for a LiteLLM exception, which is always provider-attributable."""++ def __init__(self, message: str, status_code: int):+ super().__init__(message)+ self.llm_provider = "hosted_vllm"+ self.status_code = status_code++[email protected]("status", [502, 504])+async def test_an_empty_upstream_gateway_failure_gets_an_actionable_message(monkeypatch, status):+ async def boom(**kwargs):+ raise _FakeLiteLLMError(+ "litellm.BadGatewayError: BadGatewayError: Hosted_vllmException - ", status+ )++ monkeypatch.setattr("litellm.aembedding", boom)+ with pytest.raises(LlmGatewayError) as exc:+ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config())++ assert exc.value.message.startswith(llm_gateway._UPSTREAM_GATEWAY_MESSAGE)+ assert "(rhoai/granite-embedding)" in exc.value.message+ assert "Hosted_vllmException" not in exc.value.message+ assert exc.value.status_code == status+++async def test_a_gateway_failure_with_provider_text_keeps_that_text(monkeypatch):+ async def boom(**kwargs):+ raise _FakeLiteLLMError(+ "litellm.BadGatewayError: BadGatewayError: upstream connect error", 502+ )++ monkeypatch.setattr("litellm.aembedding", boom)+ with pytest.raises(LlmGatewayError) as exc:+ await embeddings({"model": RHOAI_MODEL, "input": "x"}, config=_rhoai_config())++ assert "upstream connect error" in exc.value.message+ assert llm_gateway._UPSTREAM_GATEWAY_MESSAGE not in exc.value.message+++def test_an_empty_non_gateway_failure_is_not_relabelled():+ exc = _FakeLiteLLMError("litellm.InternalServerError: Hosted_vllmException - ", 500)+ message = llm_gateway._upstream_client_message(str(exc), "rhoai", "m", exc)+ assert llm_gateway._UPSTREAM_GATEWAY_MESSAGE not in messagediff --git a/tests/unit/services/test_llm_gateway_stale_connection_retry.py b/tests/unit/services/test_llm_gateway_stale_connection_retry.pynew file mode 100644index 000000000..7aca2b643--- /dev/null+++ b/tests/unit/services/test_llm_gateway_stale_connection_retry.py@@ -0,0 +1,216 @@+"""One retry when a dead pooled connection eats a LiteLLM call.++LiteLLM keeps idle connections for 120s; vLLM (uvicorn) closes them after 5s.+A call that reuses one while the server's close is still in flight fails with+"Server disconnected" before the server reads it, and surfaced as a provider+failure in chat, search and the health banner. Reproduced against RHOAI through+`oc port-forward`, where every reuse 4.95-6.0s after the last call failed.+"""++from types import SimpleNamespace++import aiohttp+import httpx+import pytest++from api.provider_validation import _test_litellm_provider, is_provider_stale_connection_error+from config.config_manager import (+ AnthropicConfig,+ GenericProviderConfig,+ OllamaConfig,+ OpenAIConfig,+ ProvidersConfig,+ WatsonXConfig,+)+from services import llm_gateway, provider_error_log+from services.llm_gateway import LlmGatewayError, chat_completions, embeddings+++def _stale_connection_error() -> Exception:+ """The chain LiteLLM 1.102 raises, as captured from a live RHOAI endpoint."""+ try:+ try:+ raise aiohttp.ServerDisconnectedError()+ except aiohttp.ServerDisconnectedError as disconnected:+ raise httpx.ReadError("Server disconnected") from disconnected+ except httpx.ReadError as read_error:+ error = Exception(+ "litellm.InternalServerError: InternalServerError: "+ "Hosted_vllmException - Server disconnected"+ )+ error.__cause__ = read_error+ return error+++def _rhoai_config() -> SimpleNamespace:+ providers = ProvidersConfig(+ openai=OpenAIConfig(api_key="sk-openai", configured=True),+ anthropic=AnthropicConfig(),+ watsonx=WatsonXConfig(),+ ollama=OllamaConfig(),+ custom={+ "rhoai": GenericProviderConfig(+ credentials={+ "api_base": "https://chat.example.com/v1",+ "embedding_api_base": "https://embed.example.com/v1",+ "api_key": "token",+ },+ configured=True,+ )+ },+ )+ return SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_model="granite-chat", llm_provider="rhoai"),+ knowledge=SimpleNamespace(embedding_model="granite-embedding", embedding_provider="rhoai"),+ )+++class _FailsThenAnswers:+ """A fake LiteLLM call that raises the given errors in turn, then answers."""++ def __init__(self, answer, *errors: Exception):+ self.answer = answer+ self.errors = list(errors)+ self.calls = 0++ async def __call__(self, **_kwargs):+ self.calls += 1+ if self.errors:+ raise self.errors.pop(0)+ return self.answer+++EMBEDDING = {+ "object": "list",+ "data": [{"object": "embedding", "embedding": [0.1], "index": 0}],+ "usage": {"prompt_tokens": 1, "total_tokens": 1},+}+COMPLETION = {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "choices": [{"message": {"role": "assistant", "content": "OK"}}],+}++[email protected](autouse=True)+def _clean_state():+ llm_gateway._embedding_limiters.clear()+ provider_error_log.clear()+ yield+ llm_gateway._embedding_limiters.clear()+ provider_error_log.clear()+++# --- classification ---------------------------------------------------------+++def test_litellm_wrapped_server_disconnect_is_a_stale_connection():+ assert is_provider_stale_connection_error(_stale_connection_error())+++def test_httpx_disconnect_before_any_response_is_a_stale_connection():+ """LiteLLM's httpx transport, used when a provider hands it its own client."""+ error = httpx.RemoteProtocolError("Server disconnected without sending a response.")++ assert is_provider_stale_connection_error(error)+++ "error",+ [+ # Sent after the response began: the server may have done the work.+ aiohttp.ClientPayloadError("Response payload is not completed"),+ httpx.RemoteProtocolError("peer closed connection without sending complete message body"),+ httpx.ReadError("Server disconnected"),+ # Text alone is not evidence; only the transport's own type is.+ Exception("Server disconnected"),+ Exception("401 Unauthorized"),+ None,+ ],+ ids=["payload-cut", "body-cut", "bare-read-error", "text-only", "auth", "none"],+)+def test_other_failures_are_not_stale_connections(error):+ assert not is_provider_stale_connection_error(error)+++# --- gateway ----------------------------------------------------------------+++async def test_an_embedding_on_a_dead_connection_is_sent_again(monkeypatch):+ fake = _FailsThenAnswers(EMBEDDING, _stale_connection_error())+ monkeypatch.setattr("litellm.aembedding", fake)++ result = await embeddings(+ {"model": "rhoai:granite-embedding", "input": "q"}, config=_rhoai_config()+ )++ assert fake.calls == 2+ assert result["data"][0]["embedding"] == [0.1]+ assert provider_error_log.latest_failure("rhoai", "embedding") is None+++async def test_a_completion_on_a_dead_connection_is_sent_again(monkeypatch):+ fake = _FailsThenAnswers(COMPLETION, _stale_connection_error())+ monkeypatch.setattr("litellm.acompletion", fake)++ result = await chat_completions(+ {"model": "rhoai:granite-chat", "messages": [{"role": "user", "content": "hi"}]},+ config=_rhoai_config(),+ )++ assert fake.calls == 2+ assert result["choices"][0]["message"]["content"] == "OK"+ assert provider_error_log.latest_failure("rhoai", "chat") is None+++async def test_only_one_retry_is_made(monkeypatch):+ """A server that keeps dropping connections is a real failure, reported as one."""+ fake = _FailsThenAnswers(EMBEDDING, _stale_connection_error(), _stale_connection_error())+ monkeypatch.setattr("litellm.aembedding", fake)++ with pytest.raises(LlmGatewayError):+ await embeddings({"model": "rhoai:granite-embedding", "input": "q"}, config=_rhoai_config())++ assert fake.calls == 2+ assert provider_error_log.latest_failure("rhoai", "embedding") is not None+++async def test_other_failures_are_not_retried(monkeypatch):+ """The server may have acted on these; sending them again could double the work."""+ fake = _FailsThenAnswers(COMPLETION, httpx.ReadTimeout("timed out"))+ monkeypatch.setattr("litellm.acompletion", fake)++ with pytest.raises(LlmGatewayError):+ await chat_completions(+ {"model": "rhoai:granite-chat", "messages": [{"role": "user", "content": "hi"}]},+ config=_rhoai_config(),+ )++ assert fake.calls == 1+++# --- health / pre-save probe -------------------------------------------------++[email protected]("kind", ["embedding", "chat"])+async def test_the_provider_probe_retries_a_dead_connection(monkeypatch, kind):+ """Otherwise a dead socket raises the health banner or blocks a settings save."""+ answer = EMBEDDING if kind == "embedding" else COMPLETION+ fake = _FailsThenAnswers(answer, _stale_connection_error())+ monkeypatch.setattr(f"litellm.a{'embedding' if kind == 'embedding' else 'completion'}", fake)++ await _test_litellm_provider(+ provider="rhoai",+ credentials={"api_base": "https://example.com/v1", "api_key": "token"},+ runtime_kwargs={},+ embedding_model="granite-embedding" if kind == "embedding" else None,+ llm_model="granite-chat" if kind == "chat" else None,+ )++ assert fake.calls == 2diff --git a/tests/unit/services/test_model_catalog.py b/tests/unit/services/test_model_catalog.pyindex 55ee9fb97..67f8ee302 100644--- a/tests/unit/services/test_model_catalog.py+++ b/tests/unit/services/test_model_catalog.py@@ -524,3 +524,32 @@ async def test_live_model_failure_does_not_block_other_provider_refreshes(monkeyawait model_catalog.refresh_live_models()assert calls == ["broken", "healthy"]+++async def test_refresh_live_models_can_be_narrowed_to_one_provider(monkeypatch):+ """A caller asking about one provider must not wait on another's cluster."""+ calls: list[str] = []++ def lister(name):+ async def fetch(_credentials):+ calls.append(name)++ return fetch++ first = SimpleNamespace(PROVIDER_KEY="first", fetch_models=lister("first"))+ second = SimpleNamespace(PROVIDER_KEY="second", fetch_models=lister("second"))+ providers = SimpleNamespace(stored_credentials=lambda _provider: {"api_key": "secret"})++ monkeypatch.setattr(model_catalog, "provider_enhancements", lambda: (first, second))+ monkeypatch.setattr(+ model_catalog, "supported_provider_keys", lambda: frozenset({"first", "second"})+ )+ monkeypatch.setattr(+ "config.settings.get_openrag_config",+ lambda: SimpleNamespace(providers=providers),+ )++ await model_catalog.refresh_live_models(" Second ")++ assert calls == ["second"]diff --git a/tests/unit/services/test_status_checks.py b/tests/unit/services/test_status_checks.pyindex fcb89400e..240b4e422 100644--- a/tests/unit/services/test_status_checks.py+++ b/tests/unit/services/test_status_checks.py@@ -403,3 +403,80 @@ async def test_last_error_populated_on_failure(monkeypatch):r = await check_langflow()assert r.last_error is not Noneassert len(r.last_error) > 0+++# Busy-vs-down classification: a timeout after the connection opened is DEGRADED+++ argnames="client_attr,check,target",+ argvalues=[+ ("langflow_http_client", check_langflow, "/api/v1/version"),+ ("docling_http_client", check_docling, "/version"),+ ],+)+async def test_read_timeout_is_degraded(monkeypatch, client_attr, check, target):+ monkeypatch.setattr(+ clients, client_attr, _mock_http(raises=httpx.ReadTimeout("")), raising=False+ )++ r = await check()++ assert r.status == ComponentState.DEGRADED+ assert "slow to respond" in (r.message or "")+ assert r.last_error is not None+ assert "ReadTimeout after" in r.last_error+ assert target in r.last_error+ assert r.version is None+++ argnames="client_attr,check",+ argvalues=[+ ("langflow_http_client", check_langflow),+ ("docling_http_client", check_docling),+ ],+)+async def test_connect_timeout_is_unhealthy(monkeypatch, client_attr, check):+ monkeypatch.setattr(+ clients, client_attr, _mock_http(raises=httpx.ConnectTimeout("")), raising=False+ )++ r = await check()++ assert r.status == ComponentState.UNHEALTHY+ assert "unreachable" in (r.message or "").lower()+ assert "ConnectTimeout" in (r.last_error or "")+++async def test_langflow_read_timeout_records_one_warning_then_recovers(monkeypatch):+ monkeypatch.setattr(+ clients,+ "langflow_http_client",+ _mock_http(raises=httpx.ReadTimeout("")),+ raising=False,+ )+ await check_langflow()+ await check_langflow()++ entries = _cl.get_entries("langflow", 10)+ assert [e["level"] for e in entries] == ["warning"]+ assert "ReadTimeout" in (entries[0]["detail"] or "")+ assert "target:" in (entries[0]["detail"] or "")++ monkeypatch.setattr(clients, "langflow_http_client", _mock_http(200), raising=False)+ r = await check_langflow()++ assert r.status == ComponentState.HEALTHY+ entries = _cl.get_entries("langflow", 10)+ assert entries[-1]["level"] == "info"+ assert entries[-1]["message"].startswith("recovered:")+++def test_check_timeout_stays_below_outer_deadline():+ from services import status_service++ assert status_checks._CHECK_TIMEOUT_S < status_service.CHECK_TIMEOUT_Sdiff --git a/tests/unit/test_backend_router.py b/tests/unit/test_backend_router.pyindex dc5845502..a4301ad20 100644--- a/tests/unit/test_backend_router.py+++ b/tests/unit/test_backend_router.py@@ -44,6 +44,7 @@ def test_callback_url_uses_backend_when_disabled(monkeypatch):def test_llm_base_url_uses_router_when_enabled(monkeypatch):+ monkeypatch.delenv("OPENRAG_LLM_PROXY_URL", raising=False)monkeypatch.setattr(settings, "OPENRAG_BACKEND_ROUTER_ENABLE", True)monkeypatch.setattr(settings, "OPENRAG_BACKEND_ROUTER_URL", "http://router:8100")assert settings.get_langflow_llm_base_url() == "http://router:8100"diff --git a/tests/unit/test_docling_remote_task_not_found.py b/tests/unit/test_docling_remote_task_not_found.pynew file mode 100644index 000000000..8905a6032--- /dev/null+++ b/tests/unit/test_docling_remote_task_not_found.py@@ -0,0 +1,131 @@+"""Docling Serve component: a 404 on a task poll is reported as an expired task.++docling-serve deletes a finished task some minutes after its result is first+fetched. When a flow run was replayed after that, the component surfaced a bare+"Client error '404 Not Found' for url .../status/poll/<id>", which hid the real+failure. It now raises an explicit "task not found / expired" error, both on+the first poll and on a poll inside the wait loop (which previously fell+through to a `KeyError` on the 404 body).++`lfx` and docling-core are not in the backend venv, so they are stubbed for the+duration of each test only.+"""++import importlib.util+import sys+import types+from pathlib import Path++import httpx+import pytest++COMPONENT_PATH = (+ Path(__file__).resolve().parents[2] / "custom_components" / "openrag" / "docling_remote.py"+)+BASE_URL = "http://docling:5001/v1"+TASK_ID = "df980024-2298-48f9-84e6-eadc046221cc"+++class _BaseFileComponent:+ class BaseFile: # referenced in annotations only+ pass++ @staticmethod+ def get_base_inputs():+ return []++ @staticmethod+ def get_base_outputs():+ return []+++def component(monkeypatch):+ def module(name, **attrs):+ mod = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(mod, key, value)+ monkeypatch.setitem(sys.modules, name, mod)+ return mod++ def factory(*args, **kwargs):+ return None++ module("lfx")+ module("lfx.base")+ module("lfx.base.data", BaseFileComponent=_BaseFileComponent)+ module(+ "lfx.inputs",+ IntInput=factory,+ NestedDictInput=factory,+ StrInput=factory,+ TableInput=factory,+ )+ module("lfx.inputs.inputs", FloatInput=factory)+ module("lfx.schema", Data=dict, dotdict=dict)+ module("lfx.utils")+ module("lfx.utils.util", transform_localhost_url=lambda url: url)+ module("docling_core")+ module("docling_core.types")+ module("docling_core.types.doc", DoclingDocument=object)++ spec = importlib.util.spec_from_file_location("_docling_remote_under_test", COMPONENT_PATH)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ monkeypatch.setattr(mod.time, "sleep", lambda _seconds: None)++ instance = object.__new__(mod.DoclingRemoteComponent)+ instance.logs = []+ instance.log = instance.logs.append+ instance.max_poll_timeout = 60+ return instance+++def _client(statuses: list[tuple[int, dict]]) -> tuple[httpx.Client, list[str]]:+ seen: list[str] = []+ queue = list(statuses)++ def handler(request: httpx.Request) -> httpx.Response:+ seen.append(request.url.path)+ status, body = queue.pop(0)+ return httpx.Response(status, json=body)++ return httpx.Client(transport=httpx.MockTransport(handler)), seen+++def test_a_missing_task_on_the_first_poll_is_reported_as_expired(component):+ client, seen = _client([(404, {"detail": "Task not found."})])++ with pytest.raises(RuntimeError, match="not found on the Docling Serve server") as exc:+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)++ assert TASK_ID in str(exc.value)+ assert "expired" in str(exc.value)+ assert seen == [f"/v1/status/poll/{TASK_ID}"]+ assert component.logs and TASK_ID in component.logs[0]+++def test_a_task_that_disappears_while_polling_is_reported_as_expired(component):+ client, seen = _client(+ [+ (200, {"task_status": "started"}),+ (404, {"detail": "Task not found."}),+ ]+ )++ with pytest.raises(RuntimeError, match="not found on the Docling Serve server"):+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)++ assert len(seen) == 2+++def test_other_client_errors_while_polling_still_raise_http_errors(component):+ client, _ = _client(+ [+ (200, {"task_status": "started"}),+ (403, {"detail": "Forbidden"}),+ ]+ )++ with pytest.raises(httpx.HTTPStatusError):+ component._poll_and_fetch_result(client, BASE_URL, TASK_ID)diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.pyindex 33219183d..3ba1cea8c 100644--- a/tests/unit/test_encryption.py+++ b/tests/unit/test_encryption.py@@ -6,11 +6,17 @@ from config.config_manager import ConfigManagerfrom connectors.connection_manager import ConnectionManagerfrom utils.encryption import decrypt_secret, encrypt_secret+@pytest.fixture(autouse=True)def setup_encryption_env(monkeypatch):import utils.encryption+utils.encryption._cached_master_secret = None- monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"0123456789abcdef0123456789abcdef").decode("ascii"))+ monkeypatch.setenv(+ "OPENRAG_ENCRYPTION_KEY",+ base64.b64encode(b"0123456789abcdef0123456789abcdef").decode("ascii"),+ )+def test_encryption_utility():print("Testing encryption utility...")@@ -19,82 +25,87 @@ def test_encryption_utility():assert isinstance(payload, dict)assert payload["algorithm"] == "AES-256-GCM"assert payload["tenant_id"] == "tenant-1"-+# Passing explicitly matches original AADdecrypted = decrypt_secret(payload, expected_tenant_id="tenant-1")assert decrypted == plaintext-+# Missing explicit bound works generically using payload lookupdecrypted_fallback = decrypt_secret(payload)assert decrypted_fallback == plaintext-+# Spoofing the identity dynamically rejects the AES-GCM tags!- try:+ with pytest.raises(ValueError):decrypt_secret(payload, expected_tenant_id="wrong-tenant-id")- assert False, "Should have thrown ValueError from AESGCM AAD mismatch"- except ValueError:- passprint("OK")+def test_config_manager(tmp_path):print("Testing config manager encryption...")# Create an initial config manager with a temporary filetest_yaml = tmp_path / "test_openrag_config.yaml"if test_yaml.exists():test_yaml.unlink()-+cm = ConfigManager(str(test_yaml))config = cm.get_config()config.providers.openai.api_key = "openai-api-key-plaintext"-+# Save the configcm.save_config_file(config)-+import yaml- with open(test_yaml, "r") as f:++ with open(test_yaml) as f:saved_data = yaml.safe_load(f)-+# Verify it was encrypted on diskassert isinstance(saved_data["providers"]["openai"]["api_key"], dict)assert saved_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"-+# Verify it can be loaded correctlycm_new = ConfigManager(str(test_yaml))config_new = cm_new.get_config()assert config_new.providers.openai.api_key == "openai-api-key-plaintext"print("OK")+@pytest.mark.asyncioasync def test_connection_manager(tmp_path):print("Testing connection manager encryption...")test_json = tmp_path / "test_openrag_connections.json"if test_json.exists():test_json.unlink()-+cm = ConnectionManager(str(test_json))await cm.create_connection(connector_type="google_drive",name="Test Drive",config={"client_secret": "my-client-secret-plaintext", "other_setting": "not-secret"},- user_id="user-1"+ user_id="user-1",)await cm.create_connection(connector_type="aws_s3",name="Test S3",config={"secret_key": "aws-secret", "access_key": "aws-access", "bucket_names": ["foo"]},- user_id="user-1"+ user_id="user-1",)await cm.create_connection(connector_type="ibm_cos",name="Test IBM",- config={"api_key": "ibm-api-key", "service_instance_id": "ibm-service", "bucket_names": ["bar"]},- user_id="user-1"+ config={+ "api_key": "ibm-api-key",+ "service_instance_id": "ibm-service",+ "bucket_names": ["bar"],+ },+ user_id="user-1",)# Should be saved encryptedimport json- with open(test_json, "r") as f:++ with open(test_json) as f:data = json.load(f)-+found_gd = Falsefound_s3 = Falsefound_ibm = False@@ -116,11 +127,11 @@ async def test_connection_manager(tmp_path):assert c["config"]["api_key"]["algorithm"] == "AES-256-GCM"assert isinstance(c["config"]["service_instance_id"], dict)assert found_gd and found_s3 and found_ibm-+# Reloading should decryptcm2 = ConnectionManager(str(test_json))await cm2.load_connections()-+found_gd = Falsefound_s3 = Falsefound_ibm = False@@ -140,23 +151,25 @@ async def test_connection_manager(tmp_path):assert found_gd and found_s3 and found_ibmprint("OK")-def test_auto_upgrade_features(tmp_path):++def test_auto_upgrade_features(tmp_path, monkeypatch):+ # A developer's OPENAI_API_KEY would seed the config and mask the file value.+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)test_yaml = tmp_path / "test_openrag_config_upgrade.yaml"import yaml+# Write purely plaintext configwith open(test_yaml, "w") as f:- yaml.dump({- "providers": {- "openai": {"api_key": "raw-unencrypted-openai-key-from-past"}- }- }, f)-+ yaml.dump({"providers": {"openai": {"api_key": "raw-unencrypted-openai-key-from-past"}}}, f)+cm = ConfigManager(str(test_yaml))cm.get_config()# Upon loading, the auto-upgrade should save the file over itself with the encrypted key.- with open(test_yaml, "r") as f:+ with open(test_yaml) as f:upgraded_data = yaml.safe_load(f)- assert isinstance(upgraded_data["providers"]["openai"]["api_key"], dict), "Failed to auto-upgrade config"+ assert isinstance(upgraded_data["providers"]["openai"]["api_key"], dict), (+ "Failed to auto-upgrade config"+ )assert upgraded_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"assert cm.get_config().providers.openai.api_key == "raw-unencrypted-openai-key-from-past"print("Auto-upgrade OK")diff --git a/tests/unit/test_langflow_global_variables.py b/tests/unit/test_langflow_global_variables.pyindex 23d8cd215..b82858f55 100644--- a/tests/unit/test_langflow_global_variables.py+++ b/tests/unit/test_langflow_global_variables.py@@ -135,7 +135,8 @@ async def test_update_langflow_global_variable_patches_when_type_matches():"value": "new-secret","default_fields": [],"type": "Credential",- }+ },+ "idempotent": True,},),]@@ -206,7 +207,8 @@ async def test_ensure_required_langflow_global_variables_removes_apply_to_fields"name": "OPENAI_API_KEY","default_fields": [],"type": "Credential",- }+ },+ "idempotent": True,},)@@ -359,7 +361,8 @@ async def test_update_langflow_global_variable_overwrites_redacted_credential():"value": "None","default_fields": [],"type": "Credential",- }+ },+ "idempotent": True,},),]diff --git a/tests/unit/test_langflow_utils.py b/tests/unit/test_langflow_utils.pyindex d0d818c46..6de5c06e2 100644--- a/tests/unit/test_langflow_utils.py+++ b/tests/unit/test_langflow_utils.py@@ -326,6 +326,7 @@ async def test_enable_mcp_none_for_project():"PATCH","/api/v1/mcp/project/proj-456",json={"settings": [], "auth_settings": {"auth_type": "none"}},+ idempotent=True,)@@ -346,6 +347,7 @@ async def test_enable_mcp_none_for_project_failure():"PATCH","/api/v1/mcp/project/proj-456",json={"settings": [], "auth_settings": {"auth_type": "none"}},+ idempotent=True,)diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.pyindex 2f872ee4d..dbd63ff2e 100644--- a/tests/unit/test_provider_error_formatting.py+++ b/tests/unit/test_provider_error_formatting.py@@ -8,8 +8,10 @@ import pytestfrom api.provider_validation import (format_provider_error_message,+ is_context_window_error,is_generic_upstream_error,is_provider_credential_error,+ is_provider_truncated_response_error,looks_like_provider_error_content,resolve_chat_stream_error_message,sanitize_provider_error_content,@@ -514,3 +516,52 @@ async def test_probe_checks_non_selected_providers_with_keys(monkeypatch):from api.provider_validation import probe_provider_credential_errorassert await probe_provider_credential_error() == "Provided API key is Invalid."+++ "text",+ [+ "This model's maximum context length is 32768 tokens. However, you requested 0 "+ "output tokens and your prompt contains at least 32769 input tokens",+ '{"error": {"code": "context_length_exceeded"}}',+ "ContextWindowExceededError: litellm.ContextWindowExceededError: ...",+ "prompt is too long: 210000 tokens > 200000 maximum",+ ],+)+def test_is_context_window_error(text):+ assert is_context_window_error(text)+++ "text",+ [None, "", "Incorrect API key provided", "The model `x` does not exist", "rate limit"],+)+def test_is_context_window_error_rejects_other_failures(text):+ assert not is_context_window_error(text)+++_KUBE_RBAC_PROXY_CUTOFF = (+ "MidStreamFallbackError: litellm.MidStreamFallbackError: litellm.APIConnectionError: "+ "APIConnectionError: Hosted_vllmException - Response payload is not completed: "+ "<TransferEncodingError: 400, message='Not enough data to satisfy transfer length header.'>"+)+++ "text",+ [+ _KUBE_RBAC_PROXY_CUTOFF,+ "ClientPayloadError: Response payload is not completed",+ "httpx.RemoteProtocolError: peer closed connection (incomplete chunked read)",+ ],+)+def test_is_provider_truncated_response_error(text):+ assert is_provider_truncated_response_error(text)+++ "text",+ [None, "", "Incorrect API key provided", "Connection refused", "maximum context length"],+)+def test_is_provider_truncated_response_error_rejects_other_failures(text):+ assert not is_provider_truncated_response_error(text)diff --git a/tests/unit/test_search_service_embedding_spaces.py b/tests/unit/test_search_service_embedding_spaces.pyindex 616972b66..76b02cbcd 100644--- a/tests/unit/test_search_service_embedding_spaces.py+++ b/tests/unit/test_search_service_embedding_spaces.py@@ -64,7 +64,9 @@ async def test_backend_search_queries_exact_and_legacy_embedding_providers(monkeservice.models_service = Noneroutes = []- async def create_embedding(body):+ async def create_embedding(body, *, interactive=False):+ # Query embeddings take the interactive lane, never the bulk one.+ assert interactive is Trueroutes.append(body["model"])return {"data": [{"embedding": [0.1, 0.2]}]}diff --git a/tests/unit/test_search_service_highlights.py b/tests/unit/test_search_service_highlights.pyindex e3b23981a..387268c8b 100644--- a/tests/unit/test_search_service_highlights.py+++ b/tests/unit/test_search_service_highlights.py@@ -62,7 +62,7 @@ def _make_service(opensearch: _OpenSearch, monkeypatch) -> SearchService:)service.models_service = None- async def _no_embed(body):+ async def _no_embed(body, **_kwargs):return {"data": [{"embedding": [0.1, 0.2]}]}monkeypatch.setattr("services.search_service.gateway_embeddings", _no_embed)