OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Integrate Red Hat OpenShift AI as a model provider (#2358)
Mike Pawlowskib0773c1c19a3
diff --git a/.env.example b/.env.exampleindex 501d46b9f..b1dd6dfd7 100644--- a/.env.example+++ b/.env.example@@ -264,6 +264,62 @@ WATSONX_PROJECT_ID=# OPTIONAL: Delay time = base backoff seconds for the jittered exponential backoff (default 0.5).# WATSONX_DELAY_TIME=0.5+# --- Red Hat OpenShift AI (RHOAI) ------------------------------------------+# Models served by KServe + vLLM inside your own OpenShift cluster, reached over+# the OpenAI-compatible API. vLLM serves ONE model per InferenceService, so chat+# and embeddings are two separate endpoints and both are listed below.+#+# The provider is hidden in every run mode as shipped. To offer it, copy+# src/config/model_providers.yaml, set `rhoai` to true for your run mode, and+# point OPENRAG_MODEL_PROVIDERS_CONFIG (above) at the copy. A backend restart is+# required -- that file is not hot-reloaded.+#+# WARNING: these values are read only while OpenRAG has no stored config yet.+# In the default OPENRAG_STORAGE_MODE=db, provider environment variables (these,+# and OPENAI_API_KEY / WATSONX_* / AZURE_OPENAI_* alike) are applied exactly+# once, when the database is first populated on the first boot; after that the+# stored config is authoritative and the environment is not consulted again.+# In files/hybrid mode the same happens the first time anyone saves in the+# Settings UI (config.edited becomes true). Either way: seed BEFORE the first+# boot, and afterwards change the provider through Settings, not the environment.+#+# Chat endpoint -- the predictor Service of your chat InferenceService, ending+# in /v1. In-cluster:+# https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1+# RHOAI_ENDPOINT=+#+# Embeddings endpoint -- a second InferenceService. Leave unset only if a single+# endpoint serves both. In-cluster:+# https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1+# RHOAI_EMBEDDINGS_ENDPOINT=+#+# Kubernetes bearer token used for BOTH endpoints. Its subject needs `get` on+# inferenceservices in the model namespace:+# oc create token openrag-inference -n openrag --duration=8760h+# Put the real value in a Secret, never in a committed file.+# RHOAI_API_KEY=+#+# TLS trust for the endpoints' certificates. An in-cluster Service is signed by+# OpenShift's service-serving CA, which OpenShift projects into every pod:+# RHOAI_TLS_VERIFY=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt+# Unlike SSL_CERT_FILE this is scoped to this provider only, so it cannot break+# OpenAI or Anthropic in the same deployment.+# Set to `false` to skip verification. That is for ONE case -- an oc port-forward+# to localhost, whose certificate cannot match -- and must never be set on a+# deployed environment.+# RHOAI_TLS_VERIFY=+#+# Selecting the models. The ids are each InferenceService's --served-model-name,+# not an image tag. OpenRAG lists the real ones from each endpoint's /v1/models.+# LLM_PROVIDER=rhoai+# LLM_MODEL=granite-3.3-2b-instruct+# EMBEDDING_PROVIDER=rhoai+# EMBEDDING_MODEL=granite-embedding-english-r2+#+# NOTE: the chat InferenceService must be started with --enable-auto-tool-choice+# and a --tool-call-parser (e.g. granite). OpenRAG's agent is tool-calling+# driven, and without them vLLM rejects every tool call with a 400.+# LLM Provider configuration. Providers can be "anthropic", "watsonx", "ibm" or "ollama".LLM_PROVIDER=LLM_MODEL=diff --git a/Makefile b/Makefileindex 20e2a4634..ab60a079a 100644--- a/Makefile+++ b/Makefile@@ -364,6 +364,20 @@ help_local: ## Show local development commands@echo " $(PURPLE)make instana-agent-up$(NC) - Start the local Instana host agent (forwards to INSTANA_AGENT_ENDPOINT)"@echo " $(PURPLE)make instana-agent-down$(NC) - Stop the local Instana host agent"@echo ''+ @echo "$(PURPLE)Red Hat OpenShift AI (RHOAI) models:$(NC)"+ @echo " Set these in $(ENV_FILE); they are exported to $(CYAN)make backend$(NC) and passed"+ @echo " through to the backend container by $(CYAN)make dev$(NC) / $(CYAN)make dev-cpu$(NC)."+ @echo " $(CYAN)RHOAI_ENDPOINT$(NC) - chat InferenceService URL, ending in /v1"+ @echo " $(CYAN)RHOAI_EMBEDDINGS_ENDPOINT$(NC) - embedding InferenceService URL (vLLM serves one model each)"+ @echo " $(CYAN)RHOAI_API_KEY$(NC) - ServiceAccount bearer token for both endpoints"+ @echo " $(CYAN)RHOAI_TLS_VERIFY$(NC) - CA bundle path, or 'false' for a port-forward (dev only)"+ @echo " In-cluster, use the predictor Service DNS and the projected service CA:"+ @echo " $(CYAN)/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt$(NC)"+ @echo " From a laptop, port-forward each InferenceService and set RHOAI_TLS_VERIFY=false:"+ @echo " $(CYAN)oc port-forward -n <ns> svc/<isvc>-predictor 8443:8443$(NC)"+ @echo " The provider is hidden in every run mode until $(CYAN)OPENRAG_MODEL_PROVIDERS_CONFIG$(NC)"+ @echo " points at a model_providers.yaml enabling it (see .env.example)."+ @echo ''@echo "$(PURPLE)Installation:$(NC)"@echo " $(PURPLE)make install$(NC) - Install all dependencies"@echo " $(PURPLE)make install-be$(NC) - Install backend dependencies (uv)"@@ -993,7 +1007,9 @@ test: ## Run all backend teststest-unit: ## Run unit tests only@echo "$(YELLOW)Running unit tests...$(NC)"- uv run pytest tests/unit/ -v+ @# A stale module that fails to import must not abort the whole suite;+ @# the collection error is still reported and still fails the target.+ uv run pytest tests/unit/ -v --continue-on-collection-errors@echo "$(PURPLE)Unit tests complete.$(NC)"test-integration: ## Run integration tests (requires infrastructure)diff --git a/docker-compose.yml b/docker-compose.ymlindex b5303facf..8c6c963c2 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -87,6 +87,13 @@ services:- WATSONX_API_KEY=${WATSONX_API_KEY}- WATSONX_ENDPOINT=${WATSONX_ENDPOINT}- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID}+ # Red Hat OpenShift AI. Two endpoints because vLLM serves one model per+ # InferenceService; RHOAI_TLS_VERIFY is a CA bundle path (or `false` for a+ # port-forward in development only).+ - RHOAI_ENDPOINT=${RHOAI_ENDPOINT:-}+ - RHOAI_EMBEDDINGS_ENDPOINT=${RHOAI_EMBEDDINGS_ENDPOINT:-}+ - RHOAI_API_KEY=${RHOAI_API_KEY:-}+ - RHOAI_TLS_VERIFY=${RHOAI_TLS_VERIFY:-}- OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT}- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex 0ba30ca7e..5828a918d 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -63,6 +63,10 @@ These variables are initially set during the [application onboarding process](/i| `WATSONX_API_KEY` | Not set | API key for the IBM watsonx.ai model provider. || `WATSONX_ENDPOINT` | Not set | Custom provider endpoint for the IBM watsonx.ai model provider. || `WATSONX_PROJECT_ID` | Not set | Project ID for the IBM watsonx.ai model provider. |+| `RHOAI_ENDPOINT` | Not set | Chat endpoint for the Red Hat OpenShift AI model provider: the predictor Service of the chat `InferenceService`, ending in `/v1`. |+| `RHOAI_EMBEDDINGS_ENDPOINT` | Not set | Embeddings endpoint for the Red Hat OpenShift AI model provider, a second `InferenceService`. Leave unset only if one endpoint serves both chat and embeddings. |+| `RHOAI_API_KEY` | Not set | Kubernetes bearer token for the Red Hat OpenShift AI model provider, sent to both endpoints. Its subject needs `get` on `inferenceservices` in the model namespace. |+| `RHOAI_TLS_VERIFY` | Not set | TLS trust for the Red Hat OpenShift AI endpoints. Set to a CA bundle path, such as the service-serving CA OpenShift projects into every pod (`/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`), or `false` to skip verification for a local `oc port-forward` only. Scoped to this provider, so it does not affect other providers. |Newly indexed chunks keep the model name in `embedding_model` for compatibility and also store`embedding_provider` plus the unambiguous `embedding_space_id` (`provider:model`). Retrieval queries@@ -104,6 +108,7 @@ The shipped defaults are:| `anthropic` | Yes | Yes | Yes || `azure_ai` | Yes | Yes | Yes || `azure` | Yes | Yes | Yes |+| `rhoai` | No | No | No |LiteLLM keys Azure twice, and the two are not interchangeable: `azure_ai` isthe Azure AI Foundry catalogue (Llama, Phi, Mistral, Cohere, and the gpt-5.x@@ -111,6 +116,14 @@ entries Foundry serves), while `azure` is Azure OpenAI Service — the only oneof the two that carries the gpt-4.1 family. Offer both unless a deploymentuses just one.+`rhoai` (Red Hat OpenShift AI) is hidden in every run mode as shipped, because+its endpoints and token are specific to one OpenShift cluster. To offer it,+copy `model_providers.yaml`, set `rhoai` to `true` for your run mode, point+`OPENRAG_MODEL_PROVIDERS_CONFIG` at the copy, and restart the backend. The+`RHOAI_*` variables above seed its credentials on first boot; the `models` and+`embedding_models` rows on that entry are the fallback pickers until the+cluster's own `/v1/models` listing has been read.+To change what a deployment offers, edit the `modes` values and restart, orpoint `OPENRAG_MODEL_PROVIDERS_CONFIG` at your own YAML file. The file is readonce per process, so the restart is what applies the edit; reload the consolediff --git a/enhancements/providers/redhat/__init__.py b/enhancements/providers/redhat/__init__.pynew file mode 100644index 000000000..f19b71dd3--- /dev/null+++ b/enhancements/providers/redhat/__init__.py@@ -0,0 +1 @@+"""Red Hat AI provider enhancements."""diff --git a/enhancements/providers/redhat/openshift_ai.py b/enhancements/providers/redhat/openshift_ai.pynew file mode 100644index 000000000..d153d34d8--- /dev/null+++ b/enhancements/providers/redhat/openshift_ai.py@@ -0,0 +1,771 @@+"""Red Hat OpenShift AI (RHOAI) — models served in the customer's own cluster.++RHOAI serves models through `KServe` + `vLLM`, and `vLLM` speaks the OpenAI HTTP+API. So there is no new transport here: the provider routes through LiteLLM's+`hosted_vllm` key, the same one any self-hosted vLLM endpoint uses. What differs+is everything around the call:++- **One endpoint per model.** `vLLM` serves a single model per deployment, so a+ RAG stack needs *two* `InferenceService`s — one for chat, one for embeddings —+ at two different URLs. That is the whole reason this module exists rather than+ a plain `hosted_vllm` row in `model_providers.yaml`: LiteLLM takes one+ `api_base` per call, so something has to choose which one, and only the caller+ knows whether it is embedding or completing. `litellm_credentials` therefore+ takes a `kind`, which is the optional half of the enhancement contract (see+ `credentials_for` in `enhancements/providers/registry.py`).+- **Authentication is a Kubernetes bearer token.** An `InferenceService`+ annotated `security.opendatahub.io/enable-auth: "true"` is fronted by a+ `kube-rbac-proxy` that runs a `TokenReview` and then a `SubjectAccessReview`:+ the token's subject needs `get` on `inferenceservices` in the model namespace.+ A `ServiceAccount` token is what an operator has in hand, and it goes out as+ an ordinary `Authorization: Bearer`, which is exactly what `api_key` becomes.+- **Models are whatever the operator deployed**, so the catalogue ids come from+ each endpoint's own `GET /v1/models` rather than LiteLLM's price table, which+ knows no `hosted_vllm` models at all.++The provider key is OpenRAG's own; `LITELLM_PROVIDER` is what it routes as.+Nothing here imports OpenRAG config — `config_manager`, `model_catalog` and+`llm_gateway` all read this module, so it has to stay a leaf.++Verified end to end against an `RHOAI` 3.5.0 cluster (`vLLM 0.24.0+rhaiv.9`,+CPU x86 runtime): model listing, chat and embeddings on both endpoints, with the+`ServiceAccount` token and the projected service CA.++Why `hosted_vllm` and not `openai_like`+---------------------------------------+Both are OpenAI-compatible passthrough keys, and either would carry the traffic.+`hosted_vllm` is the one that carries the **TLS setting** with it. Verified+against litellm 1.84 and a live cluster, both chat and embeddings:++- `hosted_vllm` dispatches through `base_llm_http_handler`, which reads+ `litellm_params["ssl_verify"]` — a per-call `ssl_verify=` is honoured.+- `openai_like` dispatches through `OpenAILikeChatHandler` /+ `OpenAILikeEmbeddingHandler`, neither of which mentions `ssl_verify`. The+ kwarg is silently dropped and the call fails with+ `ssl:True [SSLCertVerificationError: ... self-signed certificate in+ certificate chain]` against a cluster CA — or, worse, would quietly verify+ against the system store when the operator believed they had configured trust.++That difference is the whole reason TLS trust can be a per-provider field here+rather than the process-wide `SSL_CERT_FILE` that watsonx.ai on-prem is stuck+with (see `enhancements/providers/watsonx/onprem.py`). Do not "simplify" this to+`openai_like`.++TLS: trusting the cluster's service-serving CA+----------------------------------------------+An in-cluster `Service` presents a certificate signed by OpenShift's+service-serving CA, which is not in any container's default trust store.+OpenShift projects that CA into every pod, so the fix is a path, not a mount::++ RHOAI_TLS_VERIFY=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt++`ssl_verify` rides in the credential bag and is scoped to this provider only —+unlike `SSL_CERT_FILE`, which *replaces* certifi's roots process-wide and would+break every public provider in the same deployment.++`RHOAI_TLS_VERIFY=false` disables verification and exists for one case: an+`oc port-forward` to `localhost`, where the certificate cannot match. It is+development-only and must never be set on a deployed environment.+"""++from __future__ import annotations++import asyncio+import time+from collections.abc import Mapping+from typing import Any, Literal, NamedTuple+from urllib.parse import urlsplit, urlunsplit++from utils.logging_config import get_logger++logger = get_logger(__name__)++#: OpenRAG's key for the provider: the name in `model_providers.yaml`, in the+#: settings payload, and in a `rhoai:<model>` id.+PROVIDER_KEY = "rhoai"++#: The LiteLLM provider it is routed as. `rhoai/<model>` is not a prefix LiteLLM+#: can resolve, so the gateway swaps in this key when it builds the model+#: string. See the module docstring for why this is `hosted_vllm`.+LITELLM_PROVIDER = "hosted_vllm"++#: The two kinds of call the gateway makes. `litellm_credentials` picks an+#: endpoint from this, which is the one thing a generic provider never has to do.+CallKind = Literal["chat", "embedding"]++#: Credential form for the settings dialog. LiteLLM publishes a `hosted_vllm`+#: form, but it asks for a single `api_base` — which is exactly the assumption+#: that does not hold here, since chat and embeddings are separate deployments.+CREDENTIAL_FIELDS: list[dict[str, Any]] = [+ {+ "key": "api_base",+ "label": "Chat endpoint",+ "placeholder": "https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1",+ "tooltip": "URL of the InferenceService serving your chat model, ending in /v1. "+ "In-cluster this is the predictor Service; from a laptop it is an oc port-forward "+ "or a Route.",+ "required": True,+ "field_type": "text",+ "options": None,+ "default_value": None,+ },+ {+ "key": "embedding_api_base",+ "label": "Embeddings endpoint",+ "placeholder": "https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1",+ "tooltip": "URL of the InferenceService serving your embedding model, ending in /v1. "+ "vLLM serves one model per deployment, so this is a second InferenceService. Leave "+ "blank only if one endpoint serves both.",+ "required": False,+ "field_type": "text",+ "options": None,+ "default_value": None,+ },+ {+ "key": "api_key",+ "label": "ServiceAccount token",+ "placeholder": None,+ "tooltip": "A Kubernetes bearer token whose subject has `get` on inferenceservices in "+ "the model namespace, e.g. `oc create token openrag-inference -n openrag`. Both "+ "endpoints are authenticated with the same token.",+ "required": True,+ "field_type": "password",+ "options": None,+ "default_value": None,+ },+ {+ "key": "ssl_verify",+ "label": "TLS certificate authority",+ "placeholder": "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt",+ "tooltip": "Path to the CA bundle that signs the endpoint's certificate. In-cluster "+ "this is OpenShift's projected service CA. Set to `false` to skip verification — "+ "development only, for a port-forward whose certificate cannot match localhost.",+ "required": False,+ "field_type": "text",+ "options": None,+ "default_value": None,+ },+]++#: Fields an operator fills in that are not LiteLLM kwargs. The embedding+#: endpoint is selected *into* `api_base` per call; forwarding it as well would+#: land it in the request body, since LiteLLM passes kwargs it does not+#: recognise straight through.+_LOCAL_ONLY_FIELDS = frozenset({"embedding_api_base"})++#: LiteLLM kwargs that must not become request-body fields. `ssl_verify` is+#: read for TLS *and* — on the `hosted_vllm` chat path, verified against litellm+#: 1.84 with a capturing proxy — copied verbatim into the JSON sent to the+#: endpoint (`{"model": ..., "messages": ..., "ssl_verify": false}`). vLLM+#: ignores the unknown field; a stricter OpenAI-compatible server may 400 on+#: it, and either way a client TLS setting has no business on the wire.+#: `additional_drop_params` strips it from the body while leaving the TLS+#: effect intact, on chat, streaming and embeddings alike.+_BODY_DROP_PARAMS: tuple[str, ...] = ("ssl_verify",)++#: Values of `ssl_verify` that mean "do not verify". Spelled out rather than+#: parsed loosely: this switch turns off certificate checking, so an unrecognised+#: value must fail closed (be treated as a CA path) rather than open.+_FALSE_VALUES = frozenset({"false", "0", "no", "off"})+_TRUE_VALUES = frozenset({"true", "1", "yes", "on"})++#: TLS settings already reported. `resolve_ssl_verify` runs on every request, so+#: without this the "verification is disabled" warning — which has to be a+#: warning, since it is the one setting that silently weakens the deployment —+#: would be emitted once per chat message and per embedded chunk.+_reported_tls_settings: set[str] = set()+++def _warn_once(setting: str, message: str, **fields: Any) -> None:+ """Report a TLS (or plain-HTTP) setting the first time this process sees it."""+ if setting in _reported_tls_settings:+ return+ _reported_tls_settings.add(setting)+ logger.warning(message, **fields)+++def forget_tls_warnings() -> None:+ """Allow the TLS warnings to be emitted again. For tests."""+ _reported_tls_settings.clear()+++#: The path segment every vLLM OpenAI endpoint is served under. Appended when an+#: operator pastes the Service URL without it — the single most common cause of+#: "works with curl, fails from OpenRAG", because curl was given the full path.+_API_SUFFIX = "/v1"+++def normalized_api_base(value: Any) -> str:+ """An endpoint URL as LiteLLM needs it: no trailing slash, ending in `/v1`.++ Appends the suffix rather than rejecting the URL. An operator reads the+ predictor `Service` address off `oc get svc`, which has no path, and the+ alternative is a 404 from the endpoint's own router that says nothing about+ what is wrong.+ """+ raw = str(value or "").strip().rstrip("/")+ if not raw:+ return ""+ parts = urlsplit(raw)+ path = parts.path.rstrip("/")+ if path.endswith(_API_SUFFIX) or f"{_API_SUFFIX}/" in f"{path}/":+ return raw+ normalized = urlunsplit((parts.scheme, parts.netloc, f"{path}{_API_SUFFIX}", "", ""))+ logger.debug(+ "Appended the OpenAI path to the OpenShift AI endpoint",+ original=raw,+ normalized=normalized,+ )+ return normalized+++def resolve_ssl_verify(value: Any) -> bool | str:+ """The TLS setting for one endpoint: `True`, `False`, or a CA bundle path.++ Blank means "verify normally". A recognised false-y word disables+ verification. Anything else is taken as a filesystem path — including a path+ that does not exist, which is *warned about and passed through* so the call+ fails on the missing CA instead of quietly succeeding against the system+ trust store, which is not what the operator asked for.+ """+ raw = str(value if value is not None else "").strip()+ if not raw:+ return True+ lowered = raw.lower()+ if lowered in _FALSE_VALUES:+ _warn_once(+ "disabled",+ "TLS verification is disabled for OpenShift AI. This is a development-only "+ "setting for a port-forwarded endpoint; never use it on a deployed environment.",+ )+ return False+ if lowered in _TRUE_VALUES:+ return True++ import os++ if not os.path.exists(raw):+ _warn_once(+ f"missing:{raw}",+ "The OpenShift AI CA bundle path does not exist; TLS verification will fail "+ "until it does. In a pod the projected service CA is at "+ "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt.",+ path=raw,+ )+ return raw+++def _values(stored: Mapping[str, Any] | None) -> dict[str, str]:+ """Credential values as trimmed strings, with blanks dropped.++ Accepts either the stored form or the LiteLLM one `litellm_credentials`+ builds from it — the health check and model discovery are handed both by+ different callers. That is why a bool is spelled out rather than tested for+ truth: a translated `ssl_verify=False` must come back as `"false"`, not+ vanish as a blank, or verification is quietly switched back on. Lists+ (`additional_drop_params`) are not credentials and are skipped.+ """+ values: dict[str, str] = {}+ for name, value in (stored or {}).items():+ if isinstance(value, bool):+ text = "true" if value else "false"+ elif value is None or isinstance(value, (list, tuple, dict, set)):+ continue+ else:+ text = str(value).strip()+ if text:+ values[str(name)] = text+ return values+++def _warn_if_cleartext(label: str, api_base: str) -> None:+ """Flag an `http://` endpoint once: the ServiceAccount token goes out in+ the clear on every call to it.++ A warning, not a rejection. A raw-deployment predictor on the cluster+ network with auth disabled, or an `oc port-forward` to localhost, are+ legitimately plain HTTP — but pasting the wrong scheme for a real Service+ is silent otherwise, and it is the strictly weaker cousin of the disabled+ TLS verification this module already warns about.+ """+ if not api_base:+ return+ parts = urlsplit(api_base)+ if parts.scheme.lower() != "http":+ return+ _warn_once(+ f"http:{parts.netloc}",+ "The OpenShift AI endpoint uses plain HTTP; the ServiceAccount token will be "+ "sent in cleartext on every request. Use https:// for anything beyond a "+ "port-forward to localhost or a cluster-local endpoint with auth disabled.",+ endpoint=label,+ host=parts.netloc,+ )+++def endpoints(stored: Mapping[str, Any] | None) -> tuple[str, str]:+ """`(chat_base, embedding_base)`, both normalized, either possibly empty.++ The embedding endpoint falls back to the chat one: a deployment that serves+ both from a single endpoint is unusual for `vLLM` but perfectly valid for+ another OpenAI-compatible gateway fronting the cluster, and it keeps the+ second field optional.+ """+ values = _values(stored)+ chat = normalized_api_base(values.get("api_base"))+ embedding = normalized_api_base(values.get("embedding_api_base")) or chat+ _warn_if_cleartext("chat", chat)+ if embedding != chat:+ _warn_if_cleartext("embedding", embedding)+ return chat, embedding+++def ssl_verify_for(stored: Mapping[str, Any] | None) -> bool | str:+ """The TLS setting this module's own httpx calls should use.++ Model discovery and the health check talk to the endpoint directly rather+ than through LiteLLM, so without this they could fail on a certificate the+ gateway accepts, or the reverse.+ """+ return resolve_ssl_verify(_values(stored).get("ssl_verify"))+++def litellm_credentials(stored: Mapping[str, Any], *, kind: CallKind = "chat") -> dict[str, Any]:+ """Stored form values as LiteLLM kwargs for `hosted_vllm`, for one call kind.++ This is the only place the two-endpoint shape collapses into the single+ `api_base` LiteLLM takes. `kind` is supplied by the gateway, the health check+ and the pre-save validator, so all three exercise the same endpoint the real+ traffic will.++ Returns nothing at all when no endpoint is configured. That is deliberate+ and not merely tidiness: LiteLLM falls back to `HOSTED_VLLM_API_BASE` from+ the process environment when `api_base` is absent, so handing it a bare+ `api_key` could send the cluster's ServiceAccount token to an unrelated host+ that happens to be configured there.+ """+ values = _values(stored)+ chat_base, embedding_base = endpoints(values)+ api_base = embedding_base if kind == "embedding" else chat_base+ if not api_base:+ return {}++ credentials: dict[str, Any] = {+ name: value+ for name, value in values.items()+ if name not in _LOCAL_ONLY_FIELDS and name not in {"api_base", "ssl_verify"}+ }+ credentials["api_base"] = api_base+ credentials["ssl_verify"] = resolve_ssl_verify(values.get("ssl_verify"))+ credentials["additional_drop_params"] = list(_BODY_DROP_PARAMS)+ return credentials+++# --------------------------------------------------------------------------+# Model discovery+# --------------------------------------------------------------------------++#: How long a fetched model list is reused. The set of served models changes+#: when an administrator deploys an InferenceService, not per request.+MODELS_TTL_SECONDS = 300++#: Guard against a hung endpoint holding up a catalogue request. The catalogue+#: has a configured fallback, so failing fast is better than waiting.+MODELS_TIMEOUT_SECONDS = 10.0++#: Same, for the pre-save health check, which a human is waiting on.+HEALTH_TIMEOUT_SECONDS = 10.0++#: Retries for the health check. Fewer than the catalogue path: one retry still+#: rides out a pod mid-rollout, and a human is waiting on the answer.+HEALTH_MAX_RETRIES = 1++#: Upper bound on the whole pre-save check, both endpoints, retries and backoff+#: included. Without it two unreachable endpoints × attempts × timeout adds up+#: to a minute during which the settings save just hangs.+HEALTH_TOTAL_BUDGET_SECONDS = 25.0++#: The listing path, relative to an `api_base` that already ends in `/v1`.+MODELS_PATH = "/models"+++class ClusterModels(NamedTuple):+ """What the cluster is known to serve, one list per endpoint.++ A half is None when that endpoint could not be asked and nothing fresh is+ remembered for it. The catalogue then keeps its configured fallback for+ that half alone — a rolling embedding deployment must not empty the+ embedding picker, let alone for the whole cache TTL.+ """++ chat: tuple[str, ...] | None+ embedding: tuple[str, ...] | None+++class _Listing(NamedTuple):+ """One endpoint's answer and when it was fetched."""++ models: tuple[str, ...]+ at: float+++#: Each endpoint is remembered on its own clock, so a failed listing leaves+#: that half on its previous answer (until it ages out) instead of stamping+#: an empty list with a fresh timestamp.+_models_cache: dict[str, Any] = {"key": None, "chat": None, "embedding": None}+++def _fresh(listing: _Listing | None) -> tuple[str, ...] | None:+ if listing is None or time.monotonic() - listing.at > MODELS_TTL_SECONDS:+ return None+ return listing.models+++def cached_models() -> ClusterModels | None:+ """The last model lists fetched from the cluster, where still fresh.++ None when neither endpoint has a fresh answer; otherwise a half that has+ aged out is None on its own, and the caller falls back for just that one.+ """+ models = ClusterModels(+ chat=_fresh(_models_cache["chat"]), embedding=_fresh(_models_cache["embedding"])+ )+ if models.chat is None and models.embedding is None:+ return None+ return models+++def forget_models() -> None:+ """Drop the cached lists. For tests and for a credential change."""+ _models_cache.update(key=None, chat=None, embedding=None)+++def _cache_key(stored: Mapping[str, Any] | None) -> str:+ """Identifies the endpoints and the token a cached list was fetched with."""+ values = _values(stored)+ chat, embedding = endpoints(values)+ return f"{chat}|{embedding}|{values.get('api_key', '')}"+++def models_url(api_base: str) -> str:+ """The model-listing URL on `api_base`, however the operator typed it."""+ return f"{normalized_api_base(api_base)}{MODELS_PATH}"+++def _auth_headers(api_key: str) -> dict[str, str]:+ return {"Authorization": f"Bearer {api_key}", "Accept": "application/json"}+++def _error_details(response: Any) -> str:+ """The concise error an OpenAI-compatible endpoint returned."""+ try:+ body = response.json()+ except (ValueError, TypeError):+ return str(getattr(response, "text", ""))[:500]+ if isinstance(body, Mapping):+ error = body.get("error")+ if isinstance(error, Mapping):+ return str(error.get("message") or error.get("code") or "")[:500]+ return (+ str(body.get("message") or body.get("detail") or "")[:500]+ or str(getattr(response, "text", ""))[:500]+ )+ return str(getattr(response, "text", ""))[:500]+++async def _http_request_with_retry(+ method: str,+ url: str,+ *,+ client: Any,+ max_retries: int = 2,+ backoff_factor: float = 0.5,+ **kwargs: Any,+) -> Any:+ """Retry endpoint reads without depending on API validation helpers.++ A `vLLM` pod that is still loading weights answers 503, and a `ModelCar`+ cold start takes minutes — so a single attempt would report a healthy+ deployment as broken during a rollout.+ """+ import httpx++ for attempt in range(max_retries + 1):+ try:+ response = await client.request(method, url, **kwargs)+ if response.status_code not in (429, 500, 502, 503, 504) or attempt == max_retries:+ return response+ except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as error:+ if attempt == max_retries:+ raise error+ await asyncio.sleep(backoff_factor * (2**attempt))+ raise RuntimeError(f"OpenShift AI request to {url} failed after retries")+++def model_ids(body: Any) -> tuple[str, ...]:+ """Model ids out of one OpenAI `GET /v1/models` body.++ Order is preserved and duplicates dropped. Anything that is not a list of+ objects with an `id` yields nothing, so a proxy answering with an error page+ leaves that half on its configured fallback rather than putting garbage in+ a picker.+ """+ if not isinstance(body, Mapping):+ return ()+ data = body.get("data")+ if not isinstance(data, list):+ return ()+ ids: list[str] = []+ for entry in data:+ if not isinstance(entry, Mapping):+ continue+ model_id = str(entry.get("id") or "").strip()+ if model_id and model_id not in ids:+ ids.append(model_id)+ return tuple(ids)+++async def _list_models(client: Any, api_base: str, api_key: str) -> tuple[str, ...] | None:+ """The models one endpoint serves, or None if it could not be asked."""+ url = models_url(api_base)+ logger.debug("Listing models on the OpenShift AI endpoint", url=url)+ try:+ response = await _http_request_with_retry(+ "GET",+ url,+ client=client,+ headers=_auth_headers(api_key),+ timeout=MODELS_TIMEOUT_SECONDS,+ )+ except Exception as error:+ logger.warning(+ "Could not reach the OpenShift AI endpoint to list models",+ url=url,+ error=str(error),+ )+ return None+ if response.status_code != 200:+ logger.warning(+ "The OpenShift AI endpoint rejected the model listing",+ url=url,+ status_code=response.status_code,+ details=_error_details(response),+ )+ return None+ try:+ models = model_ids(response.json())+ except (ValueError, TypeError) as error:+ logger.warning(+ "The OpenShift AI endpoint returned a model listing that could not be read",+ url=url,+ error=str(error),+ )+ return None+ logger.debug("Listed models on the OpenShift AI endpoint", url=url, models=len(models))+ return models+++async def fetch_models(credentials: Mapping[str, Any]) -> ClusterModels | None:+ """Ask each configured endpoint which models it actually serves.++ An `InferenceService` serves whatever its operator deployed, so this is the+ only truthful source for the pickers — LiteLLM's bundled table has no+ `hosted_vllm` models at all, and the `models:` rows in+ `model_providers.yaml` are the fallback for when the cluster cannot be+ reached.++ Chat and embedding are told apart by *which endpoint answered*, which needs+ no capability sniffing and cannot be wrong. When one endpoint serves both+ (no separate embedding URL), its ids go into both lists: `GET /v1/models`+ does not say what a model is for, and emptying a picker on a guess is worse+ than offering an id that fails loudly if it is picked for the wrong job.++ Each endpoint is listed and remembered on its own: only a half whose cached+ answer has aged out is asked again, and a half that fails to answer keeps+ the answer it had, or is None so the caller falls back for that half only.+ Returns None when nothing is known at all, so a catalogue request never+ fails because the cluster is unreachable; the caller keeps whatever it had.+ """+ import httpx++ values = _values(credentials)+ chat_base, embedding_base = endpoints(values)+ api_key = values.get("api_key", "")+ if not chat_base or not api_key:+ logger.warning(+ "Not listing models on OpenShift AI: credentials are incomplete. A chat endpoint "+ "and a ServiceAccount token are needed; the configured model list is used "+ "until then.",+ has_chat_endpoint=bool(chat_base),+ has_token=bool(api_key),+ )+ return None++ key = _cache_key(values)+ if _models_cache["key"] != key:+ # Remembered for other endpoints or another token: none of it applies.+ forget_models()+ known = cached_models() or ClusterModels(chat=None, embedding=None)+ if known.chat is not None and known.embedding is not None:+ return known++ logger.info(+ "Listing models on OpenShift AI",+ chat_endpoint=chat_base,+ embedding_endpoint=embedding_base,+ shared_endpoint=chat_base == embedding_base,+ )+ listed: dict[str, tuple[str, ...] | None] = {}+ try:+ async with httpx.AsyncClient(+ verify=ssl_verify_for(values), timeout=MODELS_TIMEOUT_SECONDS+ ) as client:+ if chat_base == embedding_base:+ shared = await _list_models(client, chat_base, api_key)+ listed = {"chat": shared, "embedding": shared}+ else:+ if known.chat is None:+ listed["chat"] = await _list_models(client, chat_base, api_key)+ if known.embedding is None:+ listed["embedding"] = await _list_models(client, embedding_base, api_key)+ except Exception as error:+ logger.warning(+ "Could not list models on OpenShift AI; keeping the configured list",+ error=str(error),+ )+ return cached_models()++ if not any(listed.values()) and not any(known):+ # Nothing usable from any endpoint, and nothing remembered. An empty+ # listing is far more likely one we could not interpret than an+ # endpoint serving nothing; emptying both pickers on that guess would+ # be worse than showing the fallback, so nothing is cached.+ if any(models is not None for models in listed.values()):+ logger.warning("OpenShift AI listed no usable models; keeping the configured list")+ else:+ logger.warning("Could not list models on OpenShift AI; keeping the configured list")+ return None++ now = time.monotonic()+ for half, models in listed.items():+ # A half that failed or listed nothing keeps its previous answer and its+ # older timestamp, so it is asked again next time rather than parked for+ # a whole TTL — and an empty answer never replaces the configured+ # fallback for that half.+ if models:+ _models_cache[half] = _Listing(models=models, at=now)+ _models_cache["key"] = key++ result = cached_models() or ClusterModels(chat=None, embedding=None)+ logger.info(+ "Listed models on OpenShift AI",+ chat=len(result.chat) if result.chat is not None else "fallback",+ embedding=len(result.embedding) if result.embedding is not None else "fallback",+ )+ return result+++# --------------------------------------------------------------------------+# Health+# --------------------------------------------------------------------------+++async def lightweight_health_check(credentials: Mapping[str, Any]) -> None:+ """Validate connectivity and the token without selecting a model.++ `GET /v1/models` needs no model, no prompt and no tokens billed, which is+ what makes it a check the provider can pass before anything has been chosen+ in Settings. Both endpoints are checked when they differ — an operator who+ got the embedding URL wrong should find out here, not on the first ingest.++ For that to happen the caller has to pass the *stored* form, which still+ carries `embedding_api_base`; `validate_provider_setup(stored_credentials=...)`+ is how it gets here. Given only the LiteLLM form, the one endpoint it was+ narrowed to is checked.+ """+ import httpx++ values = _values(credentials)+ chat_base, embedding_base = endpoints(values)+ if not chat_base:+ raise Exception("No chat endpoint is configured for Red Hat OpenShift AI")+ api_key = values.get("api_key", "")+ if not api_key:+ raise Exception(+ "No ServiceAccount token is configured for Red Hat OpenShift AI. "+ "Create one with `oc create token <serviceaccount> -n <namespace>`."+ )++ targets = [("chat", chat_base)]+ if embedding_base and embedding_base != chat_base:+ targets.append(("embeddings", embedding_base))++ logger.info("Checking the OpenShift AI endpoints", endpoints=len(targets))+ try:+ # `asyncio.timeout` raises the builtin `TimeoutError`, not httpx's, so+ # both are caught below.+ async with (+ httpx.AsyncClient(verify=ssl_verify_for(values)) as client,+ asyncio.timeout(HEALTH_TOTAL_BUDGET_SECONDS),+ ):+ for label, api_base in targets:+ await _check_endpoint(client, label, api_base, api_key)+ except (httpx.TimeoutException, TimeoutError):+ logger.error("OpenShift AI health check timed out")+ raise Exception("The OpenShift AI endpoint did not respond in time") from None+ logger.info("OpenShift AI health check passed", endpoints=len(targets))+++async def _check_endpoint(client: Any, label: str, api_base: str, api_key: str) -> None:+ """One endpoint's `GET /v1/models`, with a message that names the fix."""+ import httpx++ url = models_url(api_base)+ try:+ response = await _http_request_with_retry(+ "GET",+ url,+ client=client,+ max_retries=HEALTH_MAX_RETRIES,+ headers=_auth_headers(api_key),+ timeout=HEALTH_TIMEOUT_SECONDS,+ )+ except (httpx.TimeoutException, TimeoutError):+ raise+ except Exception as error:+ logger.error(+ "Could not reach the OpenShift AI endpoint",+ endpoint=label,+ url=url,+ error=str(error),+ )+ raise Exception(+ f"Could not reach the OpenShift AI {label} endpoint at {api_base}: {error}"+ ) from error++ if response.status_code == 200:+ logger.debug("OpenShift AI endpoint answered", endpoint=label, url=url)+ return++ details = _error_details(response)+ logger.error(+ "The OpenShift AI endpoint rejected the request",+ endpoint=label,+ url=url,+ status_code=response.status_code,+ details=details,+ )+ if response.status_code in (401, 403):+ raise Exception(+ f"The OpenShift AI {label} endpoint rejected the token. Check that its subject "+ "has `get` on inferenceservices in the model namespace, and that the token has "+ f"not expired. {details}".strip()+ )+ if response.status_code == 404:+ raise Exception(+ f"The OpenShift AI {label} endpoint has no /v1/models at {api_base}. Check the "+ f"URL points at the predictor Service and ends in /v1. {details}".strip()+ )+ raise Exception(+ f"The OpenShift AI {label} endpoint returned {response.status_code}: {details}".strip()+ )diff --git a/enhancements/providers/registry.py b/enhancements/providers/registry.pyindex dd0297c9d..e5683a1d1 100644--- a/enhancements/providers/registry.py+++ b/enhancements/providers/registry.py@@ -7,12 +7,18 @@ or validation protocol.from __future__ import annotations+import functools+import inspect+from collections.abc import Mappingfrom types import ModuleType+from typing import Any+from enhancements.providers.redhat import openshift_aifrom enhancements.providers.watsonx import onprem_ENHANCEMENTS: dict[str, ModuleType] = {onprem.PROVIDER_KEY: onprem,+ openshift_ai.PROVIDER_KEY: openshift_ai,}@@ -42,3 +48,54 @@ def credential_field_overrides() -> dict[str, list[dict[str, object]]]:def enhancements() -> tuple[ModuleType, ...]:"""All registered enhancements, for generic catalogue refreshes."""return tuple(_ENHANCEMENTS.values())+++def credentials_for(+ enhancement: ModuleType,+ stored: Mapping[str, Any],+ kind: str = "chat",+) -> dict[str, Any]:+ """Translate stored form values into LiteLLM kwargs for one kind of call.++ `kind` is the *optional* half of the enhancement contract. Most providers+ reach one API with one set of credentials and take the stored values alone;+ Red Hat OpenShift AI serves chat and embeddings from two separate+ `InferenceService`s, so it has to know which endpoint the caller wants.++ `kind` is passed only to a module whose `litellm_credentials` declares it,+ so every other enhancement — and any future one written against the simpler+ signature — keeps working unchanged. The flip side is that a module which+ *needs* `kind` but drops it from its signature is called without it and+ silently answers for chat; that is a contract error the module's own tests+ have to catch, not something this function can detect.+ """+ translate = enhancement.litellm_credentials+ if _accepts_kind(enhancement):+ return dict(translate(stored, kind=kind))+ return dict(translate(stored))+++def _accepts_kind(enhancement: ModuleType) -> bool:+ """Whether the module's `litellm_credentials` takes a `kind` keyword.++ Cached per module: the answer is fixed for the life of the process, and+ this sits on the path of every gateway request.+ """+ try:+ return "kind" in inspect.signature(enhancement.litellm_credentials).parameters+ except (TypeError, ValueError):+ # A C-implemented or otherwise unintrospectable callable. Assume the+ # base contract rather than risk a TypeError on a real call.+ _log_signature_fallback(getattr(enhancement, "__name__", str(enhancement)))+ return False+++def _log_signature_fallback(module_name: str) -> None:+ from utils.logging_config import get_logger++ get_logger(__name__).debug(+ "Could not inspect a provider enhancement's credential translation; "+ "calling it without a call kind",+ module=module_name,+ )diff --git a/frontend/components/icons/openshift-ai-logo.tsx b/frontend/components/icons/openshift-ai-logo.tsxnew file mode 100644index 000000000..d85791e82--- /dev/null+++ b/frontend/components/icons/openshift-ai-logo.tsx@@ -0,0 +1,20 @@+export default function OpenShiftAILogo(props: React.SVGProps<SVGSVGElement>) {+ return (+ <svg+ xmlns="http://www.w3.org/2000/svg"+ viewBox="0 0 24 24"+ fill="currentColor"+ {...props}+ >+ <title>Red Hat OpenShift AI</title>+ {/* The OpenShift mark: a hexagonal ring around a solid core. Drawn with+ evenodd so the ring reads as a ring at 16px rather than filling in. */}+ <path+ fillRule="evenodd"+ clipRule="evenodd"+ d="M12 1.5 2.9 6.75v10.5L12 22.5l9.1-5.25V6.75L12 1.5Zm0 2.31 7.1 4.1v8.18l-7.1 4.1-7.1-4.1V7.91l7.1-4.1Z"+ />+ <circle cx="12" cy="12" r="3.35" />+ </svg>+ );+}diff --git a/frontend/components/models/model-features.tsx b/frontend/components/models/model-features.tsxindex b395ff5b9..f5df078b1 100644--- a/frontend/components/models/model-features.tsx+++ b/frontend/components/models/model-features.tsx@@ -7,6 +7,7 @@ import AzureOpenAILogo from "@/components/icons/azure-openai-logo";import IBMLogo from "@/components/icons/ibm-logo";import OllamaLogo from "@/components/icons/ollama-logo";import OpenAILogo from "@/components/icons/openai-logo";+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";import { CAPABILITY_ICONS } from "./capability-icons";import type { CatalogModel } from "./catalog-models";import {@@ -35,6 +36,7 @@ const PROVIDER_LOGOS: Record<anthropic: AnthropicLogo,watsonx: IBMLogo,watsonx_onprem: IBMLogo,+ rhoai: OpenShiftAILogo,ollama: OllamaLogo,// Both Azure rows: `azure_ai` is Foundry, `azure` is Azure OpenAI Service,// and Microsoft draws the two differently.diff --git a/frontend/components/models/model-helpers.test.ts b/frontend/components/models/model-helpers.test.tsindex e56a84998..89f05c219 100644--- a/frontend/components/models/model-helpers.test.ts+++ b/frontend/components/models/model-helpers.test.ts@@ -9,10 +9,13 @@*/import assert from "node:assert/strict";+import type { ReactElement } from "react";import { describe, it } from "vitest";+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";import {canRemoveProvider,EMBEDDING_PROVIDER_ORDER,+ getModelLogo,getProviderChrome,isProviderConfigured,LLM_PROVIDER_ORDER,@@ -82,6 +85,19 @@ describe("getProviderChrome", () => {});});+describe("getModelLogo", () => {+ it("draws the OpenShift AI mark for the rhoai provider", () => {+ // RHOAI (KServe + vLLM, PROVIDER_CHROME's `rhoai` row) is a distinct+ // branch from the watsonx/azure/local ones above and below it, so it+ // needs its own pin rather than relying on the neighbors.+ const icon = getModelLogo(+ "granite-3.1-8b-instruct",+ "rhoai",+ ) as ReactElement;+ assert.equal(icon.type, OpenShiftAILogo);+ });+});+describe("isProviderConfigured", () => {it("returns true when configured in custom or legacy fields", () => {assert.equal(diff --git a/frontend/components/models/model-helpers.tsx b/frontend/components/models/model-helpers.tsxindex d518180e7..a376afea2 100644--- a/frontend/components/models/model-helpers.tsx+++ b/frontend/components/models/model-helpers.tsx@@ -6,6 +6,7 @@ import GenericProviderLogo from "@/components/icons/generic-provider-logo";import IBMLogo from "@/components/icons/ibm-logo";import OllamaLogo from "@/components/icons/ollama-logo";import OpenAILogo from "@/components/icons/openai-logo";+import OpenShiftAILogo from "@/components/icons/openshift-ai-logo";/*** A provider key as the backend names it.@@ -24,6 +25,7 @@ export const KNOWN_PROVIDERS = ["ollama","watsonx","watsonx_onprem",+ "rhoai","azure_ai","azure","local",@@ -163,6 +165,14 @@ const PROVIDER_CHROME: Record<string, ProviderChrome> = {logoColor: "text-white",logoBgColor: "bg-[#1063FE]",},+ // Models served by KServe + vLLM inside the customer's own OpenShift+ // cluster. One card, though it is two InferenceServices behind the scenes.+ rhoai: {+ name: "Red Hat OpenShift AI",+ logo: OpenShiftAILogo,+ logoColor: "text-white",+ logoBgColor: "bg-[#EE0000]",+ },// Microsoft draws the two Azure model services differently, and neither is// the generic Azure logo. AiFoundryLogo paints its own gradient, so// logoColor does nothing for it and the tile stays white.@@ -217,6 +227,8 @@ export function getModelLogo(modelValue: string, provider?: string) {return <OllamaLogo className="w-4 h-4" />;} else if (provider === "watsonx" || provider === "watsonx_onprem") {return <IBMLogo className="w-4 h-4" />;+ } else if (provider === "rhoai") {+ return <OpenShiftAILogo className="w-4 h-4" />;} else if (provider === "azure") {return <AzureOpenAILogo className="w-4 h-4" />;} else if (provider === "azure_ai") {diff --git a/frontend/components/models/model-selector.test.tsx b/frontend/components/models/model-selector.test.tsxindex 4ae0c6a71..6b1bb5dfd 100644--- a/frontend/components/models/model-selector.test.tsx+++ b/frontend/components/models/model-selector.test.tsx@@ -1,6 +1,9 @@+import { HttpResponse, http } from "msw";import { describe, expect, it, vi } from "vitest";+import { server } from "@/test-utils/msw/server";import { renderWithProviders, screen, userEvent } from "@/test-utils/render";import { ModelSelector } from "./model-selector";+import type { GroupedModelOption } from "./types";describe("ModelSelector custom entries", () => {it("offers a typed model under every configured provider group", async () => {@@ -95,3 +98,82 @@ describe("ModelSelector flat previews", () => {expect(screen.getByRole("option", { name: "gpt-4.1" })).toBeInTheDocument();});});++/**+ * Provider badges in the group heading.+ *+ * The badge text comes from `GET /api/models/providers` (`config/model_providers.yaml`+ * on the backend), keyed by provider name — the same list Settings and+ * Onboarding already read. This pins that a provider which declares a badge+ * gets it rendered next to its group heading, and that one which doesn't+ * renders no badge at all.+ */++const rhoaiGroup: GroupedModelOption = {+ group: "OpenShift AI Models",+ provider: "rhoai",+ options: [{ value: "granite-3.1-8b-instruct", label: "Granite 3.1 8B" }],+};++const onpremGroup: GroupedModelOption = {+ group: "watsonx.ai On-Prem Models",+ provider: "watsonx_onprem",+ options: [{ value: "ibm/granite-3-8b-instruct", label: "Granite 3 8B" }],+};++function setupBadges(groupedOptions: GroupedModelOption[]) {+ return renderWithProviders(+ <ModelSelector+ groupedOptions={groupedOptions}+ value=""+ onValueChange={vi.fn()}+ defaultOpen+ />,+ );+}++function mockProviders(+ providers: Array<{ name: string; display_name: string; badge?: string }>,+) {+ server.use(+ http.get("/api/models/providers", () =>+ HttpResponse.json({ run_mode: "oss", providers }),+ ),+ );+}++describe("ModelSelector provider badges", () => {+ it("shows the badge the providers API sent for the group's provider", async () => {+ mockProviders([+ {+ name: "rhoai",+ display_name: "Red Hat OpenShift AI",+ badge: "Tech Preview",+ },+ ]);++ setupBadges([rhoaiGroup]);++ expect(await screen.findByText("Tech Preview")).toBeInTheDocument();+ });++ it("renders no badge when the provider declares none", async () => {+ // A second provider that *does* declare a badge is the proof the providers+ // response has landed: group headings render synchronously from+ // `groupedOptions`, so waiting on one of those would resolve before the+ // query does and the negative assertion below could never fail.+ mockProviders([+ { name: "rhoai", display_name: "Red Hat OpenShift AI" },+ {+ name: "watsonx_onprem",+ display_name: "watsonx.ai On-Prem",+ badge: "On-prem",+ },+ ]);++ setupBadges([rhoaiGroup, onpremGroup]);++ expect(await screen.findByText("On-prem")).toBeInTheDocument();+ expect(screen.queryByText("Tech Preview")).not.toBeInTheDocument();+ });+});diff --git a/frontend/components/models/model-selector.tsx b/frontend/components/models/model-selector.tsxindex 4323ca5cc..8ba6927fa 100644--- a/frontend/components/models/model-selector.tsx+++ b/frontend/components/models/model-selector.tsx@@ -2,6 +2,7 @@import { CheckIcon, ChevronsUpDownIcon } from "lucide-react";import { useDeferredValue, useId, useMemo, useState } from "react";+import { useGetModelProvidersQuery } from "@/app/api/queries/useGetModelProvidersQuery";import { Badge } from "@/components/ui/badge";import { Button } from "@/components/ui/button";import {@@ -100,6 +101,20 @@ export function ModelSelector({variant = "outline",...props}: ModelSelectorProps) {+ // Badges come from `config/model_providers.yaml` via the providers API, so a+ // provider that declares one gets it in the picker heading without a+ // frontend change — the same list Settings and Onboarding already read.+ const { data: modelProviders } = useGetModelProvidersQuery();+ const providerBadges = useMemo(() => {+ const badges: Record<string, string> = {};+ for (const provider of modelProviders?.providers ?? []) {+ if (provider.badge) {+ badges[provider.name] = provider.badge;+ }+ }+ return badges;+ }, [modelProviders]);+const [open, setOpen] = useState(defaultOpen);const [prevDefaultOpen, setPrevDefaultOpen] = useState(defaultOpen);if (defaultOpen !== prevDefaultOpen) {@@ -356,9 +371,9 @@ export function ModelSelector({<div className="w-4 h-4">{group.icon}</div>)}<span>{group.group}</span>- {groupProvider === "watsonx_onprem" && (+ {groupProvider && providerBadges[groupProvider] && (<Badge variant="secondary" className="text-xs">- On-prem+ {providerBadges[groupProvider]}</Badge>)}</div>diff --git a/frontend/scripts/check-diff-coverage.mjs b/frontend/scripts/check-diff-coverage.mjsindex 6405d5340..b88d5e00a 100644--- a/frontend/scripts/check-diff-coverage.mjs+++ b/frontend/scripts/check-diff-coverage.mjs@@ -36,9 +36,13 @@ const COVERAGE_FILE = path.resolve("coverage/lcov.info");/** Source files we gate on. Mirrors coverage.include in vitest.config.mts. */const INCLUDED =/^(app|components|contexts|hooks|lib|enhancements)\/.*\.(ts|tsx)$/;-/** Never gate on these: tests themselves, types, and Next route plumbing. */+/**+ * Never gate on these: tests themselves, types, Next route plumbing, and the+ * pure presentational SVG wrappers under components/icons/. Mirrors+ * coverage.exclude in vitest.config.mts.+ */const EXCLUDED =- /(\.test\.tsx?$|\.d\.ts$|\/(layout|error|global-error|loading)\.tsx$|\/route\.ts$)/;+ /(\.test\.tsx?$|\.d\.ts$|\/(layout|error|global-error|loading)\.tsx$|\/route\.ts$|^components\/icons\/)/;function git(...a) {return execFileSync("git", a, {diff --git a/frontend/vitest.config.mts b/frontend/vitest.config.mtsindex 37b319331..ec16d5111 100644--- a/frontend/vitest.config.mts+++ b/frontend/vitest.config.mts@@ -57,6 +57,8 @@ export default defineConfig({"app/**/global-error.tsx","app/**/loading.tsx","app/**/route.ts",+ // Pure presentational SVG wrappers, no logic to cover.+ "components/icons/**",],},},diff --git a/src/api/provider_health.py b/src/api/provider_health.pyindex e5ccfb901..579e989bd 100644--- a/src/api/provider_health.py+++ b/src/api/provider_health.py@@ -77,7 +77,6 @@ async def check_provider_health(api_key = getattr(provider_config, "api_key", None)endpoint = getattr(provider_config, "endpoint", None)project_id = getattr(provider_config, "project_id", None)- credentials = current_config.providers.credential_values(provider)# Check if this provider is used for LLM or embeddingllm_model = (@@ -98,6 +97,19 @@ async def check_provider_health(if model or embedding_model_override:llm_model = model or Noneembedding_model = embedding_model_override or None++ # One credential set per role, because the endpoint a provider+ # is checked against depends on which kind of call it is being+ # checked for (Red Hat OpenShift AI serves the two from+ # different endpoints). Each role is probed on its own below.+ role_credentials = {+ kind: current_config.providers.credential_values(provider, kind=kind)+ for kind in ("chat", "embedding")+ }+ # The untranslated form as well: a provider enhancement's+ # lightweight check needs every endpoint the operator entered,+ # and each entry above has been narrowed to one of them.+ stored_credentials = current_config.providers.stored_credentials(provider)except ValueError:# Provider not found in configurationreturn JSONResponse(@@ -124,8 +136,14 @@ async def check_provider_health(embedding_endpoint = getattr(embedding_provider_config, "endpoint", None)embedding_project_id = getattr(embedding_provider_config, "project_id", None)embedding_model = current_config.knowledge.embedding_model- credentials = current_config.providers.credential_values(provider)- embedding_credentials = current_config.providers.credential_values(embedding_provider)+ credentials = current_config.providers.credential_values(provider, kind="chat")+ embedding_credentials = current_config.providers.credential_values(+ embedding_provider, kind="embedding"+ )+ stored_credentials = current_config.providers.stored_credentials(provider)+ embedding_stored_credentials = current_config.providers.stored_credentials(+ embedding_provider+ )# Short-circuit identical concurrent polls from the provider-health# banner so we don't fan out N watsonx round-trips per poll cycle.@@ -185,16 +203,35 @@ async def check_provider_health(# rather than the dedicated api_key/endpoint/project_id fields, so# this must be forwarded or validating one from the providers page# runs with no credentials at all.- await validate_provider_setup(- provider=provider,- api_key=api_key,- embedding_model=embedding_model,- llm_model=llm_model,- endpoint=endpoint,- project_id=project_id,- test_completion=test_completion,- credentials=credentials,- )+ #+ # One probe per role the provider is selected for. The validator+ # tests a single model per call, so a provider that is both the+ # LLM and the embedding provider needs two calls, each with the+ # credentials for that role — otherwise the chat model is never+ # checked and the response below claims it was. With no model at+ # all, one lightweight check runs.+ probes = [+ (kind, model_name)+ for kind, model_name in (("chat", llm_model), ("embedding", embedding_model))+ if model_name+ ] or [("chat", None)]+ for index, (kind, model_name) in enumerate(probes):+ # Same spacing the polled branch applies between the two+ # watsonx tests, so back-to-back calls don't trip its rate limit.+ if index and test_completion and provider == "watsonx":+ logger.info("Waiting 2 seconds before WatsonX embedding test")+ await asyncio.sleep(2)+ await validate_provider_setup(+ provider=provider,+ api_key=api_key,+ embedding_model=model_name if kind == "embedding" else None,+ llm_model=model_name if kind == "chat" else None,+ endpoint=endpoint,+ project_id=project_id,+ test_completion=test_completion,+ credentials=role_credentials[kind],+ stored_credentials=stored_credentials,+ )return JSONResponse({@@ -226,6 +263,7 @@ async def check_provider_health(project_id=project_id,test_completion=test_completion,credentials=credentials,+ stored_credentials=stored_credentials,)except httpx.TimeoutException as e:# Timeout means provider is busy, not misconfigured@@ -261,6 +299,7 @@ async def check_provider_health(project_id=embedding_project_id,test_completion=test_completion,credentials=embedding_credentials,+ stored_credentials=embedding_stored_credentials,)except httpx.TimeoutException as e:# Timeout means provider is busy, not misconfigureddiff --git a/src/api/provider_validation.py b/src/api/provider_validation.pyindex 3f72852b8..67e227692 100644--- a/src/api/provider_validation.py+++ b/src/api/provider_validation.py@@ -3,6 +3,7 @@import asyncioimport jsonimport re+from collections.abc import Mappingfrom typing import Anyfrom urllib.parse import urlparse@@ -272,6 +273,8 @@ def _provider_probe_inputs(config: Any,provider: str,provider_config: Any,+ *,+ kind: str = "chat",) -> tuple[str | None, str | None, str | None, dict[str, str]]:"""Return legacy fields plus the complete LiteLLM credential map.@@ -280,11 +283,15 @@ def _provider_probe_inputs(every LiteLLM keyword argument in ``ProvidersConfig.credentials`` instead.Recovery probes must use both representations because they run specificallywhen Langflow has hidden the provider's real failure.++ ``kind`` (``"chat"`` or ``"embedding"``) selects the endpoint for a provider+ that serves the two from different places (Red Hat OpenShift AI), so an+ embedding probe hits the endpoint the real embedding call will."""credentials: dict[str, str] = {}credential_values = getattr(getattr(config, "providers", None), "credential_values", None)if callable(credential_values):- values = credential_values(provider)+ values = credential_values(provider, kind=kind)if isinstance(values, dict):credentials = dict(values)@@ -334,6 +341,7 @@ async def probe_provider_credential_error() -> str | None:config,provider,provider_config,+ kind="embedding" if embedding_model else "chat",)if provider == "ollama":if not endpoint:@@ -424,6 +432,7 @@ async def probe_embedding_error() -> str | None:config,provider,provider_config,+ kind="embedding",)if provider == "ollama":if not endpoint:@@ -662,6 +671,7 @@ async def validate_provider_setup(project_id: str = None,test_completion: bool = False,credentials: dict[str, str] | None = None,+ stored_credentials: Mapping[str, Any] | None = None,) -> None:"""Validate provider setup by testing completion with tool calling and embedding.@@ -675,6 +685,11 @@ async def validate_provider_setup(project_id: Project ID (required for watsonx)test_completion: If True, performs full validation with completion/embedding tests (consumes credits).If False, performs lightweight validation (no credits consumed). Default: False.+ credentials: LiteLLM kwargs for the provider, as the real call will be made.+ stored_credentials: The provider's credentials as the operator entered them, untranslated.+ Only a provider enhancement's lightweight check reads it: a provider with+ separate chat and embedding endpoints has ``credentials`` narrowed to one+ of them, and the check has to see both to probe both.Raises:Exception: If validation fails, raises the original exception with the actual error message.@@ -729,6 +744,7 @@ async def validate_provider_setup(endpoint=endpoint,project_id=project_id,credentials=supplied,+ stored_credentials=stored_credentials,)return# Full validation with completion/embedding tests (consumes credits)@@ -758,6 +774,7 @@ async def validate_provider_setup(endpoint=endpoint,project_id=project_id,credentials=supplied,+ stored_credentials=stored_credentials,)logger.info(f"Validation successful for provider: {provider_lower}")@@ -813,8 +830,14 @@ async def test_lightweight_health(endpoint: str = None,project_id: str = None,credentials: dict[str, str] | None = None,+ stored_credentials: Mapping[str, Any] | None = None,) -> None:- """Test provider health with lightweight check (no credits consumed)."""+ """Test provider health with lightweight check (no credits consumed).++ ``stored_credentials`` is the untranslated form and is what a provider+ enhancement's check is given when the caller has it; ``credentials`` is the+ LiteLLM form and the fallback. See ``validate_provider_setup``.+ """if provider == "openai":await _test_openai_lightweight_health(api_key)@@ -829,7 +852,9 @@ async def test_lightweight_health(elif provider == "anthropic":await _test_anthropic_lightweight_health(api_key)elif enhancement := get_provider_enhancement(provider):- await enhancement.lightweight_health_check(credentials or {})+ await enhancement.lightweight_health_check(+ stored_credentials if stored_credentials is not None else (credentials or {})+ )else:raise ValueError(f"Unknown provider: {provider}")diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.pyindex 773e6dd37..94e998823 100644--- a/src/api/settings/endpoints.py+++ b/src/api/settings/endpoints.py@@ -14,6 +14,7 @@ from fastapi import Depends, HTTPException, Requestfrom fastapi.responses import JSONResponsefrom api.provider_validation import (+ get_provider_enhancement,sanitize_provider_error_content,validate_provider_setup,)@@ -463,6 +464,10 @@ async def update_settings(# A generic provider save normally has no selected model to# probe. Azure is the exception: both OpenAI Service and# Foundry resource endpoints have a read-only auth probe.+ # Provider enhancements are the other exception: each carries+ # a model-free `lightweight_health_check`, so a bad URL or+ # token is rejected here rather than on the first chat or+ # ingest.for provider, submitted in (body.provider_credentials or {}).items():provider_key = _provider_key(provider)credentials = current_config.providers.pending_credentials(@@ -489,6 +494,20 @@ async def update_settings(endpoint=credentials.get("api_base"),credentials=credentials,)+ elif get_provider_enhancement(provider_key) is not None:+ # No model is passed, so the validator runs the+ # enhancement's lightweight check. It is given the+ # untranslated pending form because that is the only+ # one carrying every endpoint the operator entered —+ # both OpenShift AI `InferenceService` URLs, not the+ # single one `credentials` was narrowed to.+ await validate_provider_setup(+ provider=provider_key,+ credentials=credentials,+ stored_credentials=current_config.providers.pending_stored_credentials(+ provider_key, submitted+ ),+ )# Validate LLM provider if being changedif body.llm_provider is not None or body.llm_model is not None:@@ -539,6 +558,9 @@ async def update_settings(endpoint=endpoint,project_id=project_id,credentials=credentials,+ stored_credentials=current_config.providers.pending_stored_credentials(+ llm_provider_key, submitted_credentials.get(llm_provider_key, {})+ ),)logger.info(f"LLM provider validation successful for {llm_provider}")@@ -572,6 +594,7 @@ async def update_settings(credentials = current_config.providers.pending_credentials(embedding_provider_key,submitted_credentials.get(embedding_provider_key, {}),+ kind="embedding",)api_key = credentials.get("api_key", api_key)endpoint = credentials.get("api_base", endpoint)@@ -594,6 +617,10 @@ async def update_settings(endpoint=endpoint,project_id=project_id,credentials=credentials,+ stored_credentials=current_config.providers.pending_stored_credentials(+ embedding_provider_key,+ submitted_credentials.get(embedding_provider_key, {}),+ ),)logger.info(f"Embedding provider validation successful for {embedding_provider}"@@ -1333,7 +1360,9 @@ async def onboarding(endpoint=getattr(embedding_provider_config, "endpoint", None),project_id=getattr(embedding_provider_config, "project_id", None),test_completion=True, # Full validation with completion test - ensures provider health- credentials=current_config.providers.credential_values(embedding_provider),+ credentials=current_config.providers.credential_values(+ embedding_provider, kind="embedding"+ ),)logger.info(f"Embedding provider setup validation completed successfully for {embedding_provider}"diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 99562406e..0d6660b7d 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -3,6 +3,7 @@import jsonimport osimport re+from collections.abc import Mapping, Sequencefrom dataclasses import asdict, dataclass, fieldfrom pathlib import Pathfrom typing import Any, Optional, Protocol@@ -245,7 +246,18 @@ class ProvidersConfig:}previous.auth_method = auth_methodprevious.credentials.update(clean)- previous.configured = True+ # Complete against the form's required fields, not merely non-empty:+ # a submission of just `ssl_verify` must not make a provider look+ # callable. Same gate as the environment seed.+ required = _required_credential_keys(key)+ previous.configured = _credentials_complete(previous.credentials, required)+ if not previous.configured:+ logger.warning(+ "Model provider credentials are incomplete; it is left unconfigured "+ "until every required field is set",+ provider=key,+ missing=[name for name in required if not previous.credentials.get(name)],+ )self.custom[key] = previousif key == "openai":self.openai.api_key = clean.get("api_key", self.openai.api_key)@@ -262,9 +274,36 @@ class ProvidersConfig:self.ollama.endpoint = clean.get("api_base", self.ollama.endpoint)self.ollama.configured = bool(self.ollama.endpoint)- def pending_credentials(+ def stored_credentials(self, provider: str) -> dict[str, str]:+ """The provider's credentials exactly as the operator entered them.++ Untranslated, and therefore complete: `credential_values()` narrows a+ multi-endpoint provider down to the one endpoint a given call needs, so+ anything that has to see *all* of them — model discovery across an+ OpenShift AI deployment's chat and embedding `InferenceService`s — has+ to read the stored form instead.+ """+ return dict(self.custom.get(provider.strip().lower(), GenericProviderConfig()).credentials)++ def pending_stored_credentials(self, provider: str, submitted: dict[str, str] | None = None) -> dict[str, str]:+ """`stored_credentials()` as it would read once `submitted` is saved.++ The untranslated counterpart of `pending_credentials()`, for the pre-save+ health check of a provider whose check needs every field — both of an+ OpenShift AI deployment's endpoints — rather than the one LiteLLM call+ the translated form is narrowed to.+ """+ return {**self.stored_credentials(provider), **_clean_submitted(submitted)}++ def pending_credentials(+ self,+ provider: str,+ submitted: dict[str, str] | None = None,+ *,+ kind: str = "chat",+ ) -> dict[str, Any]:"""LiteLLM kwargs for `provider` as it would be once `submitted` is saved.Validation runs before the write, so it has to reason about the union of@@ -274,29 +313,37 @@ class ProvidersConfig:halves have to be merged *before* the translation, or a request thatchanges the API key would be validated against a stale credential builtfrom the old one.++ `kind` selects which endpoint a multi-endpoint provider is validated+ against, so the pre-save probe hits the same one the real call will."""+ from enhancements.providers.registry import credentials_forfrom enhancements.providers.registry import get as get_provider_enhancementkey = provider.strip().lower()- clean = {- str(name): str(value).strip()- for name, value in (submitted or {}).items()- if str(name).strip() and str(value).strip()- }+ clean = _clean_submitted(submitted)enhancement = get_provider_enhancement(key)if enhancement:stored = self.custom.get(key, GenericProviderConfig()).credentials- return enhancement.litellm_credentials({**stored, **clean})- values = self.credential_values(key)+ return credentials_for(enhancement, {**stored, **clean}, kind)+ values = self.credential_values(key, kind=kind)values.update(clean)return values- def credential_values(self, provider: str) -> dict[str, str]:- """Return LiteLLM keyword arguments for a configured provider."""+ def credential_values(self, provider: str, *, kind: str = "chat") -> dict[str, Any]:+ """Return LiteLLM keyword arguments for a configured provider.++ `kind` is `"chat"` or `"embedding"`. It matters only to a provider whose+ two kinds of call go to different endpoints — Red Hat OpenShift AI, where+ `vLLM` serves one model per `InferenceService` — and is ignored by every+ other provider. Callers that need the untranslated form should use+ `stored_credentials()`.+ """+ from enhancements.providers.registry import credentials_forfrom enhancements.providers.registry import get as get_provider_enhancementkey = provider.strip().lower()- custom = dict(self.custom.get(key, GenericProviderConfig()).credentials)+ custom: dict[str, Any] = dict(self.custom.get(key, GenericProviderConfig()).credentials)if key == "openai":if self.openai.api_key:custom.setdefault("api_key", self.openai.api_key)@@ -306,7 +353,7 @@ class ProvidersConfig:custom.setdefault("api_key", self.anthropic.api_key)return customif key == "watsonx":- legacy = {+ legacy: dict[str, Any] = {name: valuefor name, value in {"api_key": self.watsonx.api_key,@@ -327,10 +374,54 @@ class ProvidersConfig:# (cluster URL, username, API key); LiteLLM wants a ZenApiKey. The# translation lives with the provider so the gateway, the health# check and the validator all issue the same call.- return enhancement.litellm_credentials(custom)+ return credentials_for(enhancement, custom, kind)return custom+def _clean_submitted(submitted: dict[str, str] | None) -> dict[str, str]:+ """Submitted credential fields, trimmed, with blank names and values dropped."""+ return {+ str(name): str(value).strip()+ for name, value in (submitted or {}).items()+ if str(name).strip() and str(value).strip()+ }+++def _required_credential_keys(provider: str) -> tuple[str, ...]:+ """The fields `provider`'s form marks required, per the catalogue's spec.++ Empty for a provider without a spec, or when the catalogue cannot be read+ (LiteLLM absent, or a context where `services` is not importable): callers+ then fall back to "any credential at all", which is what `set_credentials`+ always did.+ """+ try:+ from services.model_catalog import required_field_keys++ return tuple(required_field_keys(provider))+ except Exception:+ logger.debug(+ "Could not read the required credential fields for a provider",+ provider=provider,+ exc_info=True,+ )+ return ()+++def _credentials_complete(stored: Mapping[str, Any], required: Sequence[str]) -> bool:+ """Whether a stored credential set is enough to call the provider at all.++ This is what `configured` means for both write paths — the environment+ seed and a settings save — so a provider that got half its fields cannot+ satisfy `any_configured()`, be picked as a fallback, and then be called with+ nothing useful (or, for `hosted_vllm`, with LiteLLM falling back to an+ `HOSTED_VLLM_API_BASE` from the environment that points somewhere else).+ """+ if not required:+ return any(str(value or "").strip() for value in stored.values())+ return all(str(stored.get(name) or "").strip() for name in required)++@dataclassclass KnowledgeConfig:"""Knowledge/ingestion configuration."""@@ -595,6 +686,47 @@ class ConfigManager:credentials["api_version"] = api_versionentry["configured"] = bool(credentials.get("api_key") and credentials.get("api_base"))+ @staticmethod+ def _seed_custom_provider_credentials(+ config_data: dict[str, Any],+ provider: str,+ credentials: dict[str, str | None],+ *,+ required: tuple[str, ...],+ ) -> None:+ """Fill a custom provider's credentials from an arbitrary field map.++ The sibling `_seed_custom_provider` understands exactly `api_key` /+ `api_base` / `api_version`, which is all Azure needs. A provider with its+ own form — OpenShift AI carries a second endpoint URL and a TLS setting —+ needs every field it declares, so this takes the map instead.++ `required` names the fields without which the provider cannot be called+ at all. It gates `configured` through the same `_credentials_complete`+ check `set_credentials` applies to a settings save, so a provider that+ got half its fields on either path cannot satisfy `any_configured()`,+ be picked as a fallback, and be called with nothing useful.+ """+ supplied = {+ name: str(value).strip()+ for name, value in credentials.items()+ if str(value or "").strip()+ }+ if not supplied:+ return+ custom_providers = config_data.setdefault("providers", {}).setdefault("custom", {})+ entry = custom_providers.setdefault(provider, {})+ stored = entry.setdefault("credentials", {})+ stored.update(supplied)+ entry["configured"] = _credentials_complete(stored, required)+ if not entry["configured"]:+ logger.warning(+ "Environment variables for a model provider are incomplete; it is left "+ "unconfigured until every required field is set",+ provider=provider,+ missing=[name for name in required if not stored.get(name)],+ )+def _load_env_overrides(self, config_data: dict[str, Any], temp_config: Optional["OpenRAGConfig"] = None) -> None:@@ -675,6 +807,29 @@ class ConfigManager:config_data, "azure_ai", azure_ai_key, azure_ai_endpoint, azure_ai_version)+ # Red Hat OpenShift AI. Two endpoints rather than one, because vLLM+ # serves a single model per InferenceService — see+ # enhancements/providers/redhat/openshift_ai.py. Seeding these is what+ # lets a Helm (`llmProviders.rhoai.*`) or operator (`spec.rhoai`) install+ # come up configured with no human clicking through Settings, which is+ # the point on an air-gapped cluster.+ rhoai_endpoint = os.getenv("RHOAI_ENDPOINT")+ rhoai_embeddings_endpoint = os.getenv("RHOAI_EMBEDDINGS_ENDPOINT")+ rhoai_api_key = os.getenv("RHOAI_API_KEY")+ rhoai_tls_verify = os.getenv("RHOAI_TLS_VERIFY")+ if rhoai_endpoint or rhoai_embeddings_endpoint or rhoai_api_key or rhoai_tls_verify:+ self._seed_custom_provider_credentials(+ config_data,+ "rhoai",+ {+ "api_base": rhoai_endpoint,+ "embedding_api_base": rhoai_embeddings_endpoint,+ "api_key": rhoai_api_key,+ "ssl_verify": rhoai_tls_verify,+ },+ required=("api_base", "api_key"),+ )+# Knowledge settingsif os.getenv("EMBEDDING_PROVIDER"):config_data["knowledge"]["embedding_provider"] = os.getenv("EMBEDDING_PROVIDER")diff --git a/src/config/model_providers.yaml b/src/config/model_providers.yamlindex 55d83ba4f..5f344fc96 100644--- a/src/config/model_providers.yaml+++ b/src/config/model_providers.yaml@@ -114,6 +114,36 @@ providers:embedding_models:- ibm/slate-30m-english-rtrvr+ # Red Hat OpenShift AI: models served by KServe + vLLM inside the customer's+ # own OpenShift cluster. vLLM speaks the OpenAI API, so OpenRAG routes this+ # through LiteLLM's `hosted_vllm` provider — see+ # `enhancements/providers/redhat/openshift_ai.py`, which also explains why it+ # is not `openai_like` (that key silently drops the per-provider TLS setting a+ # cluster CA needs).+ #+ # Off in every mode as shipped. It needs two endpoint URLs and a cluster token+ # that only the deployment knows, so it is enabled per deployment with+ # OPENRAG_MODEL_PROVIDERS_CONFIG rather than offered to everyone.+ #+ # The two lists below are a *fallback*, not the catalogue. OpenRAG asks each+ # InferenceService what it serves (`GET /v1/models`) and shows that in the+ # pickers, so a model the cluster does not have is never offered. These rows+ # are what it falls back to while the endpoints are unreachable or before any+ # credentials are entered.+ - name: rhoai+ display_name: Red Hat OpenShift AI+ modes:+ oss: false+ on_prem: false+ saas: false+ # Verified on an RHOAI 3.5.0 cluster running the x86 CPU vLLM runtime.+ # These are `--served-model-name` values, not image tags: whatever the+ # InferenceService was started with is the id that has to be sent.+ models:+ - granite-3.3-2b-instruct+ embedding_models:+ - granite-embedding-english-r2+- name: anthropicdisplay_name: Anthropicmodes:diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.pyindex 6acf8e026..530951b01 100644--- a/src/services/llm_gateway.py+++ b/src/services/llm_gateway.py@@ -147,7 +147,9 @@ def legacy_embedding_provider(model: str, config: Any | None = None) -> str | Noreturn provider or None-def provider_credentials(provider: str, config=None) -> dict[str, Any]:+def provider_credentials(+ provider: str, config=None, *, kind: Literal["chat", "embedding"] = "chat"+) -> dict[str, Any]:"""LiteLLM kwargs for any configured OpenRAG provider. Never logs secrets."""cfg = config or _get_config()key = (provider or "").strip().lower()@@ -157,7 +159,7 @@ def provider_credentials(provider: str, config=None) -> dict[str, Any]:raise LlmGatewayError("LLM providers are not configured", 400) from excif hasattr(prov, "credential_values"):- credentials = prov.credential_values(key)+ credentials = prov.credential_values(key, kind=kind)else:provider_config = getattr(prov, key, None)if provider_config is None:@@ -232,7 +234,7 @@ def resolve_call(if provider is None:provider = default_provider(kind, cfg)name = requested- credentials = provider_credentials(provider, cfg)+ credentials = provider_credentials(provider, cfg, kind=kind)# An OpenRAG provider that LiteLLM does not know by that name is routed# under the key it aliases (`watsonx_onprem` -> `watsonx`). The OpenRAG key# is still what the caller sees and what credentials are stored under.diff --git a/src/services/model_catalog.py b/src/services/model_catalog.pyindex 5fa6fe376..ed1575ae1 100644--- a/src/services/model_catalog.py+++ b/src/services/model_catalog.py@@ -412,7 +412,9 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:`models:` rows in `model_providers.yaml` can only ever be a guess. Once thecluster has said what it actually has, that wins — a model it does notserve must not sit in the picker waiting to be chosen. The configured rows- stay as the fallback for when it cannot be reached.+ stay as the fallback for when it cannot be reached — per list, since a+ provider with separate chat and embedding endpoints can have an answer+ from one and not the other.Entries are the `_catalog()` cache key, so substituting here rebuilds thepayload rather than serving a stale one.@@ -424,7 +426,10 @@ def _catalog_entries() -> tuple[ProviderEntry, ...]:if hasattr(enhancement, "cached_models")}return tuple(- entry._replace(models=live.chat, embedding_models=live.embedding)+ entry._replace(+ models=entry.models if live.chat is None else live.chat,+ embedding_models=entry.embedding_models if live.embedding is None else live.embedding,+ )if (live := live_models.get(entry.name)) is not Noneelse entryfor entry in entries@@ -447,7 +452,15 @@ async def refresh_live_models() -> None:):continuetry:- credentials = get_openrag_config().providers.credential_values(enhancement.PROVIDER_KEY)+ # The stored form, not the LiteLLM one: a provider whose chat and+ # embedding endpoints differ has its credentials narrowed to one of+ # them by `credential_values()`, and listing models needs both.+ providers = get_openrag_config().providers+ credentials = (+ providers.stored_credentials(enhancement.PROVIDER_KEY)+ if hasattr(providers, "stored_credentials")+ else providers.credential_values(enhancement.PROVIDER_KEY)+ )except Exception:logger.debug("Could not read credentials for %s", enhancement.PROVIDER_KEY, exc_info=Truediff --git a/tests/unit/api/test_settings_endpoints.py b/tests/unit/api/test_settings_endpoints.pyindex 2702be8f0..ba72b11a4 100644--- a/tests/unit/api/test_settings_endpoints.py+++ b/tests/unit/api/test_settings_endpoints.py@@ -304,6 +304,89 @@ async def test_update_settings_rejects_invalid_azure_foundry_key_before_saving()save.assert_not_called()+async def test_update_settings_validates_enhancement_credentials_before_saving():+ """A credential-only save runs a provider enhancement's model-free check.++ `ProviderSettingsDialog` submits nothing but `provider_credentials`, so the+ `llm_provider` / `embedding_provider` branches never fire. Without a branch+ of its own, a bad OpenShift AI URL or token was stored and the UI reported+ success; the check is handed the untranslated pending form so it sees both+ endpoints, not the one LiteLLM was narrowed to.+ """+ from api.settings.endpoints import update_settings+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict({})+ config.edited = True+ submitted = {+ "api_base": "https://granite-chat.openrag.svc:8443/v1",+ "embedding_api_base": "https://granite-embed.openrag.svc:8443/v1",+ "api_key": "expired-token",+ }+ body = SettingsUpdateBody(provider_credentials={"rhoai": submitted})+ rbac = MagicMock()+ rbac.has_permission = AsyncMock(return_value=True)++ with (+ patch("api.settings.endpoints.get_openrag_config", return_value=config),+ patch(+ "api.settings.endpoints.validate_provider_setup",+ new_callable=AsyncMock,+ side_effect=Exception("The OpenShift AI chat endpoint rejected the token"),+ ) as validate,+ patch("api.settings.endpoints.config_manager.save_config_file") as save,+ ):+ response = await update_settings(+ body=body,+ session_manager=AsyncMock(),+ user=MagicMock(spec=User),+ models_service=MagicMock(),+ rbac=rbac,+ )++ assert response.status_code == 400+ assert b"rejected the token" in response.body+ validate.assert_awaited_once()+ kwargs = validate.await_args.kwargs+ assert kwargs["provider"] == "rhoai"+ assert kwargs.get("llm_model") is None and kwargs.get("embedding_model") is None+ assert kwargs["stored_credentials"] == submitted+ save.assert_not_called()+++async def test_update_settings_skips_pre_save_check_for_litellm_only_providers():+ """Providers without an enhancement have no model-free probe to run."""+ from api.settings.endpoints import update_settings+ from config.config_manager import OpenRAGConfig++ config = OpenRAGConfig.from_dict({})+ config.edited = True+ body = SettingsUpdateBody(+ provider_credentials={"openai_like": {"api_base": "https://llm.example", "api_key": "k"}}+ )+ rbac = MagicMock()+ rbac.has_permission = AsyncMock(return_value=True)++ with (+ patch("api.settings.endpoints.get_openrag_config", return_value=config),+ patch("api.settings.endpoints.validate_provider_setup", new_callable=AsyncMock) as validate,+ patch("api.settings.endpoints.config_manager.save_config_file"),+ patch("api.settings.endpoints._update_langflow_global_variables", new_callable=AsyncMock),+ ):+ response = await update_settings(+ body=body,+ session_manager=AsyncMock(),+ user=MagicMock(spec=User),+ models_service=MagicMock(),+ rbac=rbac,+ )++ assert getattr(response, "status_code", 200) == 200+ validate.assert_not_awaited()++@pytest.mark.parametrize("provider",["openai", "watsonx", "anthropic", "local", "ollama", "azure", "azure_ai", "openai_like"],diff --git a/tests/unit/config/test_generic_provider_config.py b/tests/unit/config/test_generic_provider_config.pyindex 881ee57f7..2cf3f1202 100644--- a/tests/unit/config/test_generic_provider_config.py+++ b/tests/unit/config/test_generic_provider_config.py@@ -136,3 +136,197 @@ def test_azure_ai_foundry_env_configures_only_foundry(monkeypatch, tmp_path):"api_base": "https://test.services.ai.azure.com/models",}assert "azure" not in config.providers.custom+++def _clear_rhoai_env(monkeypatch):+ """`.env` is loaded by the root conftest, so competing values must be dropped."""+ for name in (+ "RHOAI_ENDPOINT",+ "RHOAI_EMBEDDINGS_ENDPOINT",+ "RHOAI_API_KEY",+ "RHOAI_TLS_VERIFY",+ "OPENAI_API_KEY",+ "LLM_PROVIDER",+ "EMBEDDING_PROVIDER",+ "LLM_MODEL",+ "EMBEDDING_MODEL",+ ):+ monkeypatch.delenv(name, raising=False)+++def test_rhoai_env_seeds_both_endpoints_and_the_token(monkeypatch, tmp_path):+ """A Helm/operator install has to come up configured without a human clicking+ through Settings, which is the point on an air-gapped cluster."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")++ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ assert config.providers.custom["rhoai"].configured is True+ assert config.providers.stored_credentials("rhoai") == {+ "api_base": "https://chat.svc:8443/v1",+ "embedding_api_base": "https://embed.svc:8443/v1",+ "api_key": "sha256~token",+ "ssl_verify": "/var/run/secrets/service-ca.crt",+ }+++def test_rhoai_env_without_a_token_stays_unconfigured(monkeypatch, tmp_path):+ """A provider that reports configured with half a credential set satisfies+ `any_configured()` and can then be picked as a fallback and called with+ nothing useful."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")++ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ assert config.providers.custom["rhoai"].configured is False+++def test_rhoai_tls_env_alone_reaches_a_stored_config(monkeypatch, tmp_path):+ """A CA path mounted in a later rollout arrives as `RHOAI_TLS_VERIFY` on its+ own, against a config that already holds the endpoint and token. It has to+ land, and it must not disturb `configured`."""+ import yaml++ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ config_file = tmp_path / "config.yaml"+ config_file.write_text(+ yaml.safe_dump(+ {+ "providers": {+ "custom": {+ "rhoai": {+ "credentials": {+ "api_base": "https://chat.svc:8443/v1",+ "api_key": "sha256~token",+ },+ "configured": True,+ }+ }+ }+ }+ ),+ encoding="utf-8",+ )+ monkeypatch.setenv("RHOAI_TLS_VERIFY", "/var/run/secrets/service-ca.crt")++ config = ConfigManager(config_file=config_file).load_config()++ assert config.providers.custom["rhoai"].configured is True+ assert config.providers.stored_credentials("rhoai") == {+ "api_base": "https://chat.svc:8443/v1",+ "api_key": "sha256~token",+ "ssl_verify": "/var/run/secrets/service-ca.crt",+ }+++def test_rhoai_env_is_ignored_once_settings_have_been_edited(monkeypatch, tmp_path):+ """The trap worth an explicit test: the first Settings save sets+ `config.edited` and silently freezes every environment override, which on a+ declaratively seeded cluster looks like the variables stopped working."""+ import yaml++ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ config_file = tmp_path / "config.yaml"+ config_file.write_text(yaml.safe_dump({"edited": True}), encoding="utf-8")+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")++ config = ConfigManager(config_file=config_file).load_config()++ assert "rhoai" not in config.providers.custom+++def test_pending_stored_credentials_overlays_the_submission_on_the_stored_form(+ monkeypatch, tmp_path+):+ """The pre-save health check of a two-endpoint provider needs the raw form+ as it will read once saved: what is stored, with the request's non-blank+ fields on top, and nothing translated away."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ monkeypatch.setenv("RHOAI_EMBEDDINGS_ENDPOINT", "https://embed.svc:8443/v1")+ monkeypatch.setenv("RHOAI_API_KEY", "sha256~token")+ config = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ pending = config.providers.pending_stored_credentials(+ "rhoai", {"embedding_api_base": " https://new-embed.svc:8443/v1 ", "ssl_verify": " "}+ )++ assert pending == {+ "api_base": "https://chat.svc:8443/v1",+ "embedding_api_base": "https://new-embed.svc:8443/v1",+ "api_key": "sha256~token",+ }+ # Pending, not saved.+ assert config.providers.stored_credentials("rhoai")["embedding_api_base"] == (+ "https://embed.svc:8443/v1"+ )+++def test_a_partial_save_does_not_mark_a_provider_configured():+ """`configured` gates `any_configured()` and the fallback-provider pick, so+ a direct API save of just `ssl_verify` must not make `rhoai` look callable:+ `litellm_credentials()` would return `{}` for it and LiteLLM would fall back+ to whatever `HOSTED_VLLM_API_BASE` is in the environment."""+ config = OpenRAGConfig.from_dict({})++ config.providers.set_credentials("rhoai", {"ssl_verify": "false"})++ assert config.providers.custom["rhoai"].configured is False+ assert config.providers.any_configured() is False+ # The value is kept, so a later save that completes the set builds on it.+ assert config.providers.stored_credentials("rhoai") == {"ssl_verify": "false"}+++def test_a_save_that_completes_the_required_fields_configures_the_provider():+ """The check is against the merged credentials, not the one submission, so+ a form saved in two steps ends up configured once every required field is in."""+ config = OpenRAGConfig.from_dict({})++ config.providers.set_credentials("rhoai", {"api_base": "https://chat.svc:8443/v1"})+ assert config.providers.custom["rhoai"].configured is False++ config.providers.set_credentials("rhoai", {"api_key": "sha256~token"})+ assert config.providers.custom["rhoai"].configured is True+++def test_a_provider_without_required_fields_is_configured_by_any_credential():+ """Providers whose spec marks nothing required keep the old "any non-empty+ credential" rule — Anthropic's form, for one, marks neither field."""+ config = OpenRAGConfig.from_dict({})++ config.providers.set_credentials("anthropic", {"api_base": "https://proxy.example"})++ assert config.providers.custom["anthropic"].configured is True+++def test_the_settings_save_and_the_env_seed_agree_on_configured(monkeypatch, tmp_path):+ """Both write paths run the same completeness check, so a provider cannot+ be unconfigured from the environment yet configured from the same fields+ entered in Settings, or the reverse."""+ from config.config_manager import ConfigManager++ _clear_rhoai_env(monkeypatch)+ monkeypatch.setenv("RHOAI_ENDPOINT", "https://chat.svc:8443/v1")+ seeded = ConfigManager(config_file=tmp_path / "config.yaml").load_config()++ saved = OpenRAGConfig.from_dict({})+ saved.providers.set_credentials("rhoai", {"api_base": "https://chat.svc:8443/v1"})++ assert seeded.providers.custom["rhoai"].configured is False+ assert saved.providers.custom["rhoai"].configured is Falsediff --git a/tests/unit/enhancements/providers/redhat/test_openshift_ai.py b/tests/unit/enhancements/providers/redhat/test_openshift_ai.pynew file mode 100644index 000000000..e6019f4a5--- /dev/null+++ b/tests/unit/enhancements/providers/redhat/test_openshift_ai.py@@ -0,0 +1,1179 @@+"""Red Hat OpenShift AI: KServe + vLLM endpoints, routed through LiteLLM's hosted_vllm."""++import time+from typing import Any++import pytest++import config.model_providers as model_providers+from config.config_manager import (+ AnthropicConfig,+ GenericProviderConfig,+ OllamaConfig,+ OpenAIConfig,+ ProvidersConfig,+ WatsonXConfig,+)+from enhancements.providers import registry+from enhancements.providers.redhat import openshift_ai as rhoai+from services import model_catalog+from services.llm_gateway import provider_credentials, resolve_call, split_model_id++PROVIDER = rhoai.PROVIDER_KEY++CHAT_BASE = "https://openrag-chat-predictor.openrag-models.svc.cluster.local:8443/v1"+EMBED_BASE = "https://openrag-embed-predictor.openrag-models.svc.cluster.local:8443/v1"+TOKEN = "sha256~not-a-real-token"++CHAT_MODEL = "granite-3.3-2b-instruct"+EMBED_MODEL = "granite-embedding-english-r2"++[email protected](autouse=True)+def _reload_provider_config():+ model_providers.reload()+ yield+ model_providers.reload()++[email protected](autouse=True)+def _forget_module_state():+ """Model cache and TLS warnings are process-global; no test may inherit them."""+ rhoai.forget_models()+ rhoai.forget_tls_warnings()+ yield+ rhoai.forget_models()+ rhoai.forget_tls_warnings()+++def _stored(**overrides: Any) -> dict[str, Any]:+ values: dict[str, Any] = {+ "api_base": CHAT_BASE,+ "embedding_api_base": EMBED_BASE,+ "api_key": TOKEN,+ }+ values.update(overrides)+ return {name: value for name, value in values.items() if value is not None}+++def _providers(**credentials: Any) -> ProvidersConfig:+ return ProvidersConfig(+ openai=OpenAIConfig(),+ anthropic=AnthropicConfig(),+ watsonx=WatsonXConfig(),+ ollama=OllamaConfig(),+ custom={PROVIDER: GenericProviderConfig(credentials=dict(credentials), configured=True)},+ )+++def _enable_in_oss(tmp_path, monkeypatch) -> None:+ """Point the run mode at a config file that offers this provider.++ The shipped `model_providers.yaml` has every mode false — the endpoints and+ token are deployment-specific, so it is turned on per deployment. A test that+ needs the catalogue has to do the same thing an operator does.+ """+ config = tmp_path / "model_providers.yaml"+ config.write_text(+ "providers:\n"+ f" - name: {PROVIDER}\n"+ " display_name: Red Hat OpenShift AI\n"+ " modes:\n"+ " oss: true\n"+ f" models:\n - {CHAT_MODEL}\n"+ f" embedding_models:\n - {EMBED_MODEL}\n",+ encoding="utf-8",+ )+ monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")+ monkeypatch.setenv("OPENRAG_MODEL_PROVIDERS_CONFIG", str(config))+ model_providers.reload()+++# ---------------------------------------------------------------------------+# Credential form+# ---------------------------------------------------------------------------+++def test_the_form_asks_for_two_endpoints_a_token_and_a_ca() -> None:+ """vLLM serves one model per InferenceService, so one endpoint is not enough."""+ fields = {field["key"]: field for field in model_catalog.credential_fields(PROVIDER)}++ assert set(fields) == {"api_base", "embedding_api_base", "api_key", "ssl_verify"}+ assert fields["api_base"]["field_type"] == "text"+ assert fields["embedding_api_base"]["field_type"] == "text"+ assert fields["api_key"]["field_type"] == "password"+ assert fields["ssl_verify"]["field_type"] == "text"+++def test_only_the_token_is_encrypted_at_rest() -> None:+ """`field_type` drives encryption. A CA path is not a secret; a token is."""+ assert model_catalog.secret_field_keys(PROVIDER) == {"api_key"}+++def test_an_endpoint_and_a_token_are_both_required() -> None:+ assert model_catalog.required_field_keys(PROVIDER) == ["api_base", "api_key"]+++# ---------------------------------------------------------------------------+# Endpoint selection — the reason this module exists+# ---------------------------------------------------------------------------+++def test_a_chat_call_goes_to_the_chat_endpoint() -> None:+ assert rhoai.litellm_credentials(_stored(), kind="chat")["api_base"] == CHAT_BASE+++def test_an_embedding_call_goes_to_the_embedding_endpoint() -> None:+ """The whole point: one credential bag, two InferenceServices."""+ assert rhoai.litellm_credentials(_stored(), kind="embedding")["api_base"] == EMBED_BASE+++def test_chat_is_the_default_kind() -> None:+ """An older caller that does not know about kinds must still reach a model."""+ assert rhoai.litellm_credentials(_stored())["api_base"] == CHAT_BASE+++def test_one_endpoint_serves_both_when_no_embedding_url_is_given() -> None:+ """The second field is optional, so a single-endpoint gateway still works."""+ stored = _stored(embedding_api_base=None)++ assert rhoai.litellm_credentials(stored, kind="chat")["api_base"] == CHAT_BASE+ assert rhoai.litellm_credentials(stored, kind="embedding")["api_base"] == CHAT_BASE++[email protected]("kind", ["chat", "embedding"])+def test_the_embedding_url_never_reaches_litellm(kind) -> None:+ """LiteLLM forwards kwargs it does not recognise, so a stray field lands in the body."""+ assert "embedding_api_base" not in rhoai.litellm_credentials(_stored(), kind=kind)+++def test_no_endpoint_means_no_credentials_at_all() -> None:+ """Not tidiness: LiteLLM falls back to HOSTED_VLLM_API_BASE from the process+ environment when `api_base` is absent, so returning a bare token could send a+ cluster ServiceAccount token to whatever unrelated host is configured there.+ """+ assert rhoai.litellm_credentials({"api_key": TOKEN}) == {}+++def test_the_token_is_carried_through_as_the_api_key() -> None:+ """RHOAI's kube-rbac-proxy wants `Authorization: Bearer <token>`, which is+ exactly what LiteLLM does with `api_key`."""+ assert rhoai.litellm_credentials(_stored())["api_key"] == TOKEN+++def test_the_tls_setting_is_kept_off_the_wire() -> None:+ """LiteLLM reads `ssl_verify` for TLS and, on the hosted_vllm chat path, also+ copies it into the request body. `additional_drop_params` is how the bag+ tells LiteLLM to strip it again."""+ credentials = rhoai.litellm_credentials(_stored(ssl_verify="false"))++ assert credentials["ssl_verify"] is False+ assert "ssl_verify" in credentials["additional_drop_params"]+++async def test_litellm_does_not_leak_the_tls_setting_into_the_request_body() -> None:+ """The real thing, against a local OpenAI-compatible stub: found with a+ capturing proxy in front of a live cluster, where the body arrived as+ `{"model": ..., "messages": ..., "ssl_verify": false}`. vLLM ignored it; a+ stricter server would not."""+ import json+ import threading+ from http.server import BaseHTTPRequestHandler, HTTPServer++ import litellm++ bodies: list[dict[str, Any]] = []++ class _Stub(BaseHTTPRequestHandler):+ def log_message(self, *args: Any) -> None:+ pass++ def do_POST(self) -> None:+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ bodies.append(body)+ if self.path.endswith("/embeddings"):+ reply = {+ "object": "list",+ "model": body["model"],+ "data": [{"index": 0, "object": "embedding", "embedding": [0.0] * 4}],+ "usage": {"prompt_tokens": 1, "total_tokens": 1},+ }+ else:+ reply = {+ "id": "chatcmpl-stub",+ "object": "chat.completion",+ "model": body["model"],+ "choices": [+ {+ "index": 0,+ "finish_reason": "stop",+ "message": {"role": "assistant", "content": "ok"},+ }+ ],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ }+ raw = json.dumps(reply).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(raw)))+ self.end_headers()+ self.wfile.write(raw)++ server = HTTPServer(("127.0.0.1", 0), _Stub)+ threading.Thread(target=server.serve_forever, daemon=True).start()+ try:+ base = f"http://127.0.0.1:{server.server_port}/v1"+ stored = _stored(api_base=base, embedding_api_base=base, ssl_verify="false")+ await litellm.acompletion(+ model=f"hosted_vllm/{CHAT_MODEL}",+ messages=[{"role": "user", "content": "hi"}],+ drop_params=True,+ **rhoai.litellm_credentials(stored, kind="chat"),+ )+ await litellm.aembedding(+ model=f"hosted_vllm/{EMBED_MODEL}",+ input=["hi"],+ **rhoai.litellm_credentials(stored, kind="embedding"),+ )+ finally:+ server.shutdown()++ assert len(bodies) == 2+ for body in bodies:+ assert "ssl_verify" not in body+ assert "additional_drop_params" not in body+ assert "embedding_api_base" not in body+++# ---------------------------------------------------------------------------+# Endpoint URL normalization+# ---------------------------------------------------------------------------+++ "given, expected",+ [+ # `oc get svc` prints an address with no path; pasting it is the common case.+ ("https://chat.svc:8443", "https://chat.svc:8443/v1"),+ ("https://chat.svc:8443/", "https://chat.svc:8443/v1"),+ ("https://chat.svc:8443/v1", "https://chat.svc:8443/v1"),+ ("https://chat.svc:8443/v1/", "https://chat.svc:8443/v1"),+ # An endpoint behind a path-routing proxy keeps its prefix.+ ("https://gw.example.com/openrag/v1", "https://gw.example.com/openrag/v1"),+ ("", ""),+ (None, ""),+ ],+)+def test_the_openai_path_is_added_but_never_doubled(given, expected) -> None:+ assert rhoai.normalized_api_base(given) == expected+++def test_a_pasted_service_address_still_reaches_the_models_route() -> None:+ assert rhoai.models_url("https://chat.svc:8443") == "https://chat.svc:8443/v1/models"+++# ---------------------------------------------------------------------------+# TLS+# ---------------------------------------------------------------------------+++def test_no_ca_configured_verifies_normally() -> None:+ assert rhoai.resolve_ssl_verify("") is True+ assert rhoai.resolve_ssl_verify(None) is True++[email protected]("value", ["false", "False", "0", "no", "off", " FALSE "])+def test_verification_can_be_turned_off_for_a_port_forward(value) -> None:+ assert rhoai.resolve_ssl_verify(value) is False++[email protected]("value", ["true", "1", "yes", "on"])+def test_an_explicit_true_verifies_normally(value) -> None:+ assert rhoai.resolve_ssl_verify(value) is True+++def test_a_ca_bundle_path_is_passed_through(tmp_path) -> None:+ ca = tmp_path / "service-ca.crt"+ ca.write_text("-----BEGIN CERTIFICATE-----", encoding="utf-8")++ assert rhoai.resolve_ssl_verify(str(ca)) == str(ca)+++def _recorded_warnings(monkeypatch) -> list[str]:+ """Warnings the module emits. Read off the module logger rather than caplog:+ logging goes through structlog here, which does not route to caplog's handler.+ """+ messages: list[str] = []++ class _Recorder:+ def warning(self, message, **fields):+ messages.append(message)++ def __getattr__(self, _name):+ return lambda *args, **kwargs: None++ monkeypatch.setattr(rhoai, "logger", _Recorder())+ return messages+++def test_a_missing_ca_path_is_warned_about_and_still_used(monkeypatch) -> None:+ """Falling back to the system trust store would verify against roots the+ operator never asked for, and succeed — hiding the misconfiguration. Better+ to fail on the missing file.+ """+ warnings = _recorded_warnings(monkeypatch)+ missing = "/no/such/service-ca.crt"++ assert rhoai.resolve_ssl_verify(missing) == missing+ assert any("CA bundle path does not exist" in message for message in warnings)+++def test_the_disabled_tls_warning_is_not_repeated_per_request(monkeypatch) -> None:+ """`litellm_credentials` runs on every chat message and every embedded chunk."""+ warnings = _recorded_warnings(monkeypatch)++ for _ in range(5):+ rhoai.resolve_ssl_verify("false")++ disabled = [m for m in warnings if "TLS verification is disabled" in m]+ assert len(disabled) == 1+++def test_a_plain_http_endpoint_is_warned_about_but_still_used(monkeypatch) -> None:+ """The token goes out in the clear, which deserves the same treatment as+ disabled verification — a warning, not a rejection, since a port-forward+ and a cluster-local raw deployment are legitimately plain HTTP."""+ warnings = _recorded_warnings(monkeypatch)++ chat, embedding = rhoai.endpoints(+ _stored(api_base="http://chat.svc:8080/v1", embedding_api_base="http://embed.svc:8080")+ )++ assert chat == "http://chat.svc:8080/v1"+ assert embedding == "http://embed.svc:8080/v1"+ cleartext = [m for m in warnings if "plain HTTP" in m]+ assert len(cleartext) == 2 # one per distinct endpoint+++def test_the_cleartext_warning_is_not_repeated_per_request(monkeypatch) -> None:+ """`endpoints()` runs inside `litellm_credentials`, i.e. on every chat+ message and every embedded chunk."""+ warnings = _recorded_warnings(monkeypatch)++ for _ in range(5):+ rhoai.litellm_credentials(_stored(api_base="http://chat.svc:8080/v1"))++ cleartext = [m for m in warnings if "plain HTTP" in m]+ assert len(cleartext) == 1+++def test_https_endpoints_draw_no_cleartext_warning(monkeypatch) -> None:+ warnings = _recorded_warnings(monkeypatch)++ rhoai.endpoints(_stored())++ assert not [m for m in warnings if "plain HTTP" in m]+++def test_the_tls_setting_rides_in_the_credential_bag(tmp_path) -> None:+ """Per-provider, unlike SSL_CERT_FILE, which would replace certifi's roots+ process-wide and break every public provider in the same deployment."""+ ca = tmp_path / "service-ca.crt"+ ca.write_text("-----BEGIN CERTIFICATE-----", encoding="utf-8")++ credentials = rhoai.litellm_credentials(_stored(ssl_verify=str(ca)))++ assert credentials["ssl_verify"] == str(ca)+ assert rhoai.litellm_credentials(_stored(ssl_verify="false"))["ssl_verify"] is False+++def test_a_translated_false_is_not_mistaken_for_a_blank() -> None:+ """The health check and model discovery are handed the LiteLLM form by some+ callers, where `ssl_verify` is already a bool. `False` is falsy, and a+ truth-test on the value used to drop it — turning "do not verify" back+ into "verify" for the one setting where that inverts the operator's intent."""+ assert rhoai._values({"ssl_verify": False})["ssl_verify"] == "false"+ assert rhoai._values({"ssl_verify": True})["ssl_verify"] == "true"+ # Not a credential; must not be stringified into one.+ assert "additional_drop_params" not in rhoai._values({"additional_drop_params": ["x"]})+++def test_verification_stays_off_when_the_health_check_gets_the_litellm_form() -> None:+ """The regression: `RHOAI_TLS_VERIFY=false` for a port-forward, and the+ pre-save check verifying anyway while real traffic did not."""+ translated = rhoai.litellm_credentials(_stored(ssl_verify="false"), kind="chat")++ assert translated["ssl_verify"] is False+ assert rhoai.ssl_verify_for(translated) is False+++# ---------------------------------------------------------------------------+# Routing+# ---------------------------------------------------------------------------+++def test_it_routes_as_hosted_vllm() -> None:+ """`rhoai/<model>` is not a prefix LiteLLM can resolve. `hosted_vllm` is+ chosen over `openai_like` because only it threads a per-call `ssl_verify` —+ see the module docstring.+ """+ assert model_catalog.litellm_provider_key(PROVIDER) == "hosted_vllm"+ assert rhoai.LITELLM_PROVIDER == "hosted_vllm"+++def test_an_alias_is_a_provider_the_gateway_can_resolve() -> None:+ assert model_catalog.is_known_provider(PROVIDER)+++def test_its_model_ids_carry_the_openrag_provider_tag() -> None:+ assert split_model_id(f"{PROVIDER}:{CHAT_MODEL}") == (PROVIDER, CHAT_MODEL)+ assert model_catalog.public_model_id(PROVIDER, CHAT_MODEL) == f"{PROVIDER}:{CHAT_MODEL}"+++def test_a_chat_call_resolves_to_the_chat_endpoint() -> None:+ config = _config(_providers(**_stored()))++ litellm_model, provider, credentials = resolve_call(+ f"{PROVIDER}:{CHAT_MODEL}", kind="chat", config=config+ )++ assert litellm_model == f"hosted_vllm/{CHAT_MODEL}"+ assert provider == PROVIDER+ assert credentials["api_base"] == CHAT_BASE+++def test_an_embedding_call_resolves_to_the_embedding_endpoint() -> None:+ """End to end through the gateway: the kind has to survive `resolve_call`,+ `provider_credentials` and `credential_values` to get here."""+ config = _config(_providers(**_stored()))++ litellm_model, provider, credentials = resolve_call(+ f"{PROVIDER}:{EMBED_MODEL}", kind="embedding", config=config+ )++ assert litellm_model == f"hosted_vllm/{EMBED_MODEL}"+ assert provider == PROVIDER+ assert credentials["api_base"] == EMBED_BASE+++def _config(providers: ProvidersConfig):+ from types import SimpleNamespace++ return SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_provider=PROVIDER, llm_model=CHAT_MODEL),+ knowledge=SimpleNamespace(+ embedding_provider=PROVIDER,+ embedding_model=EMBED_MODEL,+ legacy_embedding_provider_map={},+ ),+ )+++def test_credential_values_narrows_to_the_requested_endpoint() -> None:+ providers = _providers(**_stored())++ assert providers.credential_values(PROVIDER, kind="chat")["api_base"] == CHAT_BASE+ assert providers.credential_values(PROVIDER, kind="embedding")["api_base"] == EMBED_BASE+++def test_stored_credentials_keeps_both_endpoints() -> None:+ """Model discovery has to see both, which the narrowed form cannot give it."""+ stored = _providers(**_stored()).stored_credentials(PROVIDER)++ assert stored["api_base"] == CHAT_BASE+ assert stored["embedding_api_base"] == EMBED_BASE+++def test_the_gateway_asks_the_config_for_the_kind_it_needs() -> None:+ """`credential_values(provider, kind=...)` is the contract: the gateway+ passes the keyword outright, so a config object has to declare it."""++ from types import SimpleNamespace++ class _Config:+ def credential_values(self, provider, *, kind="chat"):+ return {"api_key": "legacy", "kind": kind}++ credentials = provider_credentials(+ "openai", SimpleNamespace(providers=_Config()), kind="embedding"+ )++ assert credentials == {"api_key": "legacy", "kind": "embedding"}+++def test_a_kindless_enhancement_is_still_callable() -> None:+ """watsonx.ai on-prem reaches one API and takes the one-argument form."""+ onprem = registry.get("watsonx_onprem")+ credentials = registry.credentials_for(+ onprem,+ {"api_base": "https://cpd.example.com", "username": "u", "api_key": "k"},+ "embedding",+ )++ assert credentials["api_base"] == "https://cpd.example.com"+++# ---------------------------------------------------------------------------+# Model discovery+# ---------------------------------------------------------------------------+++class _Response:+ def __init__(self, status_code: int, payload: Any = None, text: str = "") -> None:+ self.status_code = status_code+ self._payload = payload+ self.text = text++ def json(self) -> Any:+ if self._payload is None:+ raise ValueError("no JSON body")+ return self._payload+++def _models_body(*ids: str) -> dict[str, Any]:+ return {"object": "list", "data": [{"id": model, "object": "model"} for model in ids]}+++def _client_returning(responses: dict[str, Any], seen: dict[str, Any] | None = None):+ """An httpx.AsyncClient stand-in that answers per URL."""++ class _Client:+ def __init__(self, **kwargs):+ if seen is not None:+ seen["verify"] = kwargs.get("verify")++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ async def request(self, method, url, **kwargs):+ if seen is not None:+ seen.setdefault("urls", []).append(url)+ seen["headers"] = kwargs.get("headers")+ answer = responses[url]+ if isinstance(answer, Exception):+ raise answer+ return answer++ return _Client+++async def test_each_endpoint_is_asked_what_it_serves(monkeypatch) -> None:+ """Chat and embedding are told apart by which endpoint answered, so no+ capability sniffing is needed and the split cannot be wrong."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ },+ seen,+ ),+ )++ models = await rhoai.fetch_models(_stored())++ assert models.chat == (CHAT_MODEL,)+ assert models.embedding == (EMBED_MODEL,)+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models"]+ assert seen["headers"]["Authorization"] == f"Bearer {TOKEN}"+++async def test_one_endpoint_is_asked_once_and_fills_both_pickers(monkeypatch) -> None:+ """`GET /v1/models` does not say what a model is for. Emptying a picker on a+ guess is worse than offering an id that fails loudly if misused."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning({f"{CHAT_BASE}/models": _Response(200, _models_body("solo"))}, seen),+ )++ models = await rhoai.fetch_models(_stored(embedding_api_base=None))++ assert models.chat == ("solo",)+ assert models.embedding == ("solo",)+ assert seen["urls"] == [f"{CHAT_BASE}/models"]+++async def test_the_configured_tls_setting_is_used_for_discovery_too(monkeypatch) -> None:+ """Discovery talks to the endpoint directly, not through LiteLLM, so without+ this it could fail on a certificate the gateway accepts, or the reverse."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ },+ seen,+ ),+ )++ await rhoai.fetch_models(_stored(ssl_verify="false"))++ assert seen["verify"] is False+++async def test_one_dead_endpoint_does_not_empty_the_other_picker(monkeypatch) -> None:+ """A rolling embedding deployment must not take the chat picker with it."""+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ models = await rhoai.fetch_models(_stored())++ assert models.chat == (CHAT_MODEL,)+ assert models.embedding is None # that half keeps its configured fallback+++async def test_a_dead_endpoint_does_not_empty_its_own_picker_either(monkeypatch) -> None:+ """The failed half is None, not an empty list, so the catalogue keeps the+ configured rows for it rather than showing nothing for five minutes."""+ responses = {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),+ }+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses))+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ await rhoai.fetch_models(_stored())++ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)+++async def test_a_failed_listing_keeps_the_previous_answer_for_that_half(monkeypatch) -> None:+ """An endpoint mid-rollout answered fine a minute ago; that answer is+ better than the configured guess, so it stays until it ages out."""+ seen: dict[str, Any] = {}+ responses = {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ }+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses, seen))+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)+ await rhoai.fetch_models(_stored())++ # The embedding answer ages out; the endpoint is then down.+ rhoai._models_cache["embedding"] = rhoai._models_cache["embedding"]._replace(+ at=time.monotonic() - rhoai.MODELS_TTL_SECONDS - 1+ )+ responses[f"{EMBED_BASE}/models"] = _Response(503, None, "Service Unavailable")+ models = await rhoai.fetch_models(_stored())++ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)+ # Only the stale half was asked again; the fresh chat answer was reused.+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models", f"{EMBED_BASE}/models"]+++async def test_a_failed_half_is_asked_again_next_time(monkeypatch) -> None:+ """A failure must not be parked behind a fresh timestamp: the next refresh+ tries that endpoint again, and a recovered answer fills the picker."""+ seen: dict[str, Any] = {}+ responses = {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),+ }+ monkeypatch.setattr("httpx.AsyncClient", _client_returning(responses, seen))+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)+ await rhoai.fetch_models(_stored())++ responses[f"{EMBED_BASE}/models"] = _Response(200, _models_body(EMBED_MODEL))+ models = await rhoai.fetch_models(_stored())++ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=(EMBED_MODEL,))+ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models", f"{EMBED_BASE}/models"]+++async def _no_retry(method, url, *, client, **kwargs):+ """Skip the backoff so a failure test does not sleep."""+ return await client.request(method, url, **kwargs)+++async def test_an_unreachable_cluster_keeps_the_configured_fallback(monkeypatch) -> None:+ """Returning None leaves whatever the catalogue had; raising would fail the+ whole catalogue request because one endpoint is down."""+ import httpx++ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": httpx.ConnectError("no route to host"),+ f"{EMBED_BASE}/models": httpx.ConnectError("no route to host"),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ assert await rhoai.fetch_models(_stored()) is None+++async def test_a_listing_that_cannot_be_read_keeps_the_fallback(monkeypatch) -> None:+ """A proxy answering with an HTML error page must not put garbage in a picker."""+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, None, "<html>gateway timeout</html>"),+ f"{EMBED_BASE}/models": _Response(200, {"unexpected": "shape"}),+ }+ ),+ )++ assert await rhoai.fetch_models(_stored()) is None+++async def test_an_empty_listing_on_one_half_keeps_that_fallback(monkeypatch) -> None:+ """An endpoint that answers `{"data": []}` while the other half is fine must+ not have an empty tuple cached for it: that would swap the configured rows+ for a blank picker until the TTL expires."""+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, {"object": "list", "data": []}),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ models = await rhoai.fetch_models(_stored())++ assert models == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)+ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)+++async def test_an_unreadable_listing_on_one_half_keeps_that_fallback(monkeypatch) -> None:+ """Same for a JSON body of the wrong shape: it is no answer for that half,+ not an answer of "nothing"."""+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, {"unexpected": "shape"}),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ await rhoai.fetch_models(_stored())++ assert rhoai.cached_models() == rhoai.ClusterModels(chat=(CHAT_MODEL,), embedding=None)+++async def test_incomplete_credentials_are_not_taken_to_the_network(monkeypatch) -> None:+ def _explode(**kwargs):+ raise AssertionError("no HTTP call should be made")++ monkeypatch.setattr("httpx.AsyncClient", _explode)++ assert await rhoai.fetch_models({"api_key": TOKEN}) is None+ assert await rhoai.fetch_models({"api_base": CHAT_BASE}) is None+++ "body, expected",+ [+ (_models_body("a", "b"), ("a", "b")),+ # Order preserved, duplicates dropped.+ (_models_body("a", "b", "a"), ("a", "b")),+ ({"object": "list", "data": []}, ()),+ ({"object": "list", "data": [{"no": "id"}, {"id": " "}, "junk"]}, ()),+ ({"error": {"message": "nope"}}, ()),+ ("not a mapping", ()),+ (None, ()),+ ],+)+def test_only_well_formed_model_entries_are_kept(body, expected) -> None:+ assert rhoai.model_ids(body) == expected+++async def test_a_fresh_listing_is_reused_rather_than_refetched(monkeypatch) -> None:+ """The set of served models changes when an administrator deploys one, not+ once per catalogue request."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ },+ seen,+ ),+ )++ await rhoai.fetch_models(_stored())+ await rhoai.fetch_models(_stored())++ assert len(seen["urls"]) == 2 # one round, not two+ assert rhoai.cached_models().chat == (CHAT_MODEL,)+++async def test_changing_an_endpoint_invalidates_the_cache(monkeypatch) -> None:+ """A cached list belongs to the endpoints and token it was fetched with."""+ other = "https://other-chat.svc:8443/v1"+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ f"{other}/models": _Response(200, _models_body("other-model")),+ },+ seen,+ ),+ )++ await rhoai.fetch_models(_stored())+ models = await rhoai.fetch_models(_stored(api_base=other))++ assert models.chat == ("other-model",)+++async def test_forget_models_drops_the_cache(monkeypatch) -> None:+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ }+ ),+ )++ await rhoai.fetch_models(_stored())+ assert rhoai.cached_models() is not None++ rhoai.forget_models()+ assert rhoai.cached_models() is None+++# ---------------------------------------------------------------------------+# Health+# ---------------------------------------------------------------------------+++async def test_a_healthy_deployment_passes_with_no_model_selected(monkeypatch) -> None:+ """`GET /v1/models` needs no model and bills no tokens, which is what makes+ it a check the provider can pass during onboarding."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(200, _models_body(EMBED_MODEL)),+ },+ seen,+ ),+ )++ await rhoai.lightweight_health_check(_stored())++ assert seen["urls"] == [f"{CHAT_BASE}/models", f"{EMBED_BASE}/models"]+++async def test_the_litellm_form_still_checks_with_the_configured_tls_setting(monkeypatch) -> None:+ """Handed the narrowed bag, the check covers the one endpoint in it — but+ with the TLS setting the operator chose, not verification switched back on."""+ seen: dict[str, Any] = {}+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning({f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL))}, seen),+ )+ translated = rhoai.litellm_credentials(_stored(ssl_verify="false"), kind="chat")++ await rhoai.lightweight_health_check(translated)++ assert seen["verify"] is False+ assert seen["urls"] == [f"{CHAT_BASE}/models"]+++async def test_validation_hands_the_check_the_stored_form(monkeypatch) -> None:+ """`validate_provider_setup` gets the LiteLLM form for the probe and the+ stored form for the enhancement check; the check must get the latter, or+ the embedding endpoint is never looked at before the first ingest."""+ from api import provider_validation++ received: list[dict[str, Any]] = []++ async def _capture(credentials):+ received.append(dict(credentials))++ monkeypatch.setattr(rhoai, "lightweight_health_check", _capture)+ stored = _stored()+ translated = rhoai.litellm_credentials(stored, kind="chat")++ await provider_validation.validate_provider_setup(+ provider=PROVIDER, credentials=translated, stored_credentials=stored+ )+ await provider_validation.validate_provider_setup(provider=PROVIDER, credentials=translated)++ assert received[0]["embedding_api_base"] == EMBED_BASE+ assert "embedding_api_base" not in received[1] # no stored form given: falls back+++async def test_a_wrong_embedding_url_is_found_before_the_first_ingest(monkeypatch) -> None:+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body(CHAT_MODEL)),+ f"{EMBED_BASE}/models": _Response(404, {"error": {"message": "Not Found"}}),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ with pytest.raises(Exception, match="no /v1/models"):+ await rhoai.lightweight_health_check(_stored())++[email protected]("status", [401, 403])+async def test_a_rejected_token_names_the_rolebinding(monkeypatch, status) -> None:+ """RHOAI runs a SubjectAccessReview, so the usual cause is a missing grant+ rather than a bad token — and the message has to say so or the operator+ hunts for the wrong thing."""+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {f"{CHAT_BASE}/models": _Response(status, {"error": {"message": "forbidden"}})}+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ with pytest.raises(Exception, match="inferenceservices"):+ await rhoai.lightweight_health_check(_stored())+++async def test_an_unreachable_endpoint_says_which_one(monkeypatch) -> None:+ import httpx++ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning({f"{CHAT_BASE}/models": httpx.ConnectError("no route to host")}),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ with pytest.raises(Exception, match="chat endpoint"):+ await rhoai.lightweight_health_check(_stored())+++async def test_a_slow_endpoint_is_reported_as_a_timeout(monkeypatch) -> None:+ import httpx++ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning({f"{CHAT_BASE}/models": httpx.ConnectTimeout("too slow")}),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)++ with pytest.raises(Exception, match="did not respond in time"):+ await rhoai.lightweight_health_check(_stored())+++async def test_the_whole_check_is_bounded_however_slow_the_endpoints_are(monkeypatch) -> None:+ """Per-request timeouts, retries and backoff across two endpoints add up to+ about a minute; a human is waiting on the settings save, so the check as a+ whole has a deadline of its own."""+ import asyncio++ class _Hanging:+ def __init__(self, **kwargs):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ async def request(self, method, url, **kwargs):+ await asyncio.sleep(60)++ monkeypatch.setattr("httpx.AsyncClient", _Hanging)+ monkeypatch.setattr(rhoai, "HEALTH_TOTAL_BUDGET_SECONDS", 0.05)++ with pytest.raises(Exception, match="did not respond in time"):+ await asyncio.wait_for(rhoai.lightweight_health_check(_stored()), timeout=5)+++async def test_the_health_check_retries_less_than_discovery(monkeypatch) -> None:+ """Discovery can afford to wait out a rollout; the pre-save check cannot."""+ calls: list[int] = []++ async def _count(method, url, *, client, max_retries=2, **kwargs):+ calls.append(max_retries)+ return _Response(200, _models_body(CHAT_MODEL))++ monkeypatch.setattr("httpx.AsyncClient", _client_returning({}))+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _count)++ await rhoai.lightweight_health_check(_stored(embedding_api_base=""))++ assert calls == [rhoai.HEALTH_MAX_RETRIES]+ assert rhoai.HEALTH_MAX_RETRIES < 2+++async def test_missing_credentials_say_what_is_missing(monkeypatch) -> None:+ """Distinct messages, because the fix is different: one is a URL, the other+ is an `oc create token`."""++ def _explode(**kwargs):+ raise AssertionError("no HTTP call should be made")++ monkeypatch.setattr("httpx.AsyncClient", _explode)++ with pytest.raises(Exception, match="No chat endpoint"):+ await rhoai.lightweight_health_check({"api_key": TOKEN})++ with pytest.raises(Exception, match="No ServiceAccount token"):+ await rhoai.lightweight_health_check({"api_base": CHAT_BASE})+++# ---------------------------------------------------------------------------+# Visibility and the catalogue+# ---------------------------------------------------------------------------++[email protected]("run_mode", ["oss", "on_prem", "saas"])+def test_it_is_hidden_everywhere_as_shipped(monkeypatch, run_mode) -> None:+ """Its endpoints and token are deployment-specific, so it is turned on with+ OPENRAG_MODEL_PROVIDERS_CONFIG rather than offered to everyone."""+ monkeypatch.delenv("OPENRAG_MODEL_PROVIDERS_CONFIG", raising=False)+ monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)+ model_providers.reload()++ assert PROVIDER not in model_providers.visible_provider_keys()+++def test_a_deployment_can_offer_it_with_a_config_override(tmp_path, monkeypatch) -> None:+ _enable_in_oss(tmp_path, monkeypatch)++ assert PROVIDER in model_providers.visible_provider_keys()+++def test_the_configured_models_are_the_fallback_picker(tmp_path, monkeypatch) -> None:+ """LiteLLM's bundled table has no hosted_vllm models at all, so without the+ config rows the pickers would be empty until the cluster answers."""+ _enable_in_oss(tmp_path, monkeypatch)++ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]++ assert [m["model"] for m in entry["models"]] == [CHAT_MODEL]+ assert [m["model"] for m in entry["embedding_models"]] == [EMBED_MODEL]+ assert entry["name"] == "Red Hat OpenShift AI"+++async def test_what_the_cluster_serves_wins_over_the_configured_fallback(+ tmp_path, monkeypatch+) -> None:+ """A model the InferenceServices do not serve must not sit in the picker+ waiting to be chosen."""+ _enable_in_oss(tmp_path, monkeypatch)+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body("deployed-chat")),+ f"{EMBED_BASE}/models": _Response(200, _models_body("deployed-embed")),+ }+ ),+ )+ await rhoai.fetch_models(_stored())++ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]++ assert [m["model"] for m in entry["models"]] == ["deployed-chat"]+ assert [m["model"] for m in entry["embedding_models"]] == ["deployed-embed"]+++async def test_the_picker_for_a_dead_endpoint_keeps_the_configured_fallback(+ tmp_path, monkeypatch+) -> None:+ """One endpoint down empties neither picker: the live answer wins where+ there is one, and the configured rows stand in for the half without."""+ _enable_in_oss(tmp_path, monkeypatch)+ monkeypatch.setattr(+ "httpx.AsyncClient",+ _client_returning(+ {+ f"{CHAT_BASE}/models": _Response(200, _models_body("deployed-chat")),+ f"{EMBED_BASE}/models": _Response(503, None, "Service Unavailable"),+ }+ ),+ )+ monkeypatch.setattr(rhoai, "_http_request_with_retry", _no_retry)+ await rhoai.fetch_models(_stored())++ entry = {e["key"]: e for e in model_catalog.catalog()["providers"]}[PROVIDER]++ assert [m["model"] for m in entry["models"]] == ["deployed-chat"]+ assert [m["model"] for m in entry["embedding_models"]] == [EMBED_MODEL]+++async def test_the_catalogue_refresh_passes_both_endpoints(tmp_path, monkeypatch) -> None:+ """`refresh_live_models` reads the stored form, not the narrowed LiteLLM one:+ the kind-specific bag carries only one endpoint, and discovery needs both."""+ _enable_in_oss(tmp_path, monkeypatch)+ captured: dict[str, Any] = {}++ async def _capture(credentials):+ captured.update(credentials)+ return None++ monkeypatch.setattr(rhoai, "fetch_models", _capture)+ monkeypatch.setattr(+ "config.settings.get_openrag_config", lambda: _config(_providers(**_stored()))+ )++ await model_catalog.refresh_live_models()++ assert captured["api_base"] == CHAT_BASE+ assert captured["embedding_api_base"] == EMBED_BASEdiff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.pyindex d475ce642..2f872ee4d 100644--- a/tests/unit/test_provider_error_formatting.py+++ b/tests/unit/test_provider_error_formatting.py@@ -241,7 +241,7 @@ async def test_resolve_ingest_error_message_probes_generic_embedding_with_credenwatsonx = Noneollama = None- def credential_values(self, provider):+ def credential_values(self, provider, *, kind="chat"):assert provider == "azure"return dict(credentials)@@ -281,6 +281,84 @@ async def test_resolve_ingest_error_message_probes_generic_embedding_with_credenassert len(calls) == 1+async def test_ingest_recovery_probes_request_embedding_credentials(monkeypatch):+ """The embedding probes must ask for the *embedding* endpoint's credentials.++ A provider that serves chat and embeddings from separate endpoints (Red Hat+ OpenShift AI) narrows ``credential_values`` by ``kind``; probing the+ embedding model against the chat endpoint would report a bogus "model not+ found" and hide the real embedding-endpoint failure.+ """++ by_kind = {+ "chat": {"api_key": "token", "api_base": "https://chat.example/v1"},+ "embedding": {"api_key": "token", "api_base": "https://embed.example/v1"},+ }++ class GenericProvider:+ configured = True++ class FakeProviders:+ custom = {"rhoai": GenericProvider()}+ openai = None+ anthropic = None+ watsonx = None+ ollama = None++ def credential_values(self, provider, *, kind="chat"):+ assert provider == "rhoai"+ return dict(by_kind[kind])++ class FakeConfig:+ class knowledge:+ embedding_provider = "rhoai"+ embedding_model = "granite-embedding"++ class agent:+ llm_provider = "rhoai"+ llm_model = "granite-chat"++ providers = FakeProviders()++ def get_embedding_provider_config(self):+ return self.providers.custom["rhoai"]++ def get_llm_provider_config(self):+ return self.providers.custom["rhoai"]++ calls = []++ async def fake_validate(**kwargs):+ calls.append(kwargs)++ monkeypatch.setattr("config.settings.get_openrag_config", lambda: FakeConfig())+ monkeypatch.setattr("api.provider_validation.validate_provider_setup", fake_validate)++ from api.provider_validation import (+ probe_chat_llm_error,+ probe_embedding_error,+ probe_provider_credential_error,+ )++ assert await probe_embedding_error() is None+ assert await probe_chat_llm_error() is None+ assert await probe_provider_credential_error() is None++ # probe_embedding_error, probe_chat_llm_error, then the credential probe's+ # single rhoai candidate (the embedding row is checked first, so it wins).+ assert [c["credentials"]["api_base"] for c in calls] == [+ "https://embed.example/v1",+ "https://chat.example/v1",+ "https://embed.example/v1",+ ]+ assert [c["endpoint"] for c in calls] == [+ "https://embed.example/v1",+ "https://chat.example/v1",+ "https://embed.example/v1",+ ]++@pytest.mark.asyncioasync def test_probe_chat_llm_error_uses_generic_provider_credentials(monkeypatch):credentials = {@@ -295,7 +373,7 @@ async def test_probe_chat_llm_error_uses_generic_provider_credentials(monkeypatcclass FakeProviders:custom = {"azure": GenericProvider()}- def credential_values(self, provider):+ def credential_values(self, provider, *, kind="chat"):assert provider == "azure"return dict(credentials)diff --git a/tests/unit/test_provider_health_generic.py b/tests/unit/test_provider_health_generic.pyindex b6c064481..e18c1c656 100644--- a/tests/unit/test_provider_health_generic.py+++ b/tests/unit/test_provider_health_generic.py@@ -20,13 +20,15 @@ async def test_provider_health_accepts_configured_azure_provider(endpoint=None,project_id=None,)+ stored = {+ "api_key": "azure-secret",+ "api_base": "https://example.openai.azure.com",+ "api_version": "2024-10-21",+ }providers = SimpleNamespace(get_provider_config=lambda provider: azure,- credential_values=lambda provider: {- "api_key": "azure-secret",- "api_base": "https://example.openai.azure.com",- "api_version": "2024-10-21",- },+ credential_values=lambda provider, kind="chat": dict(stored),+ stored_credentials=lambda provider: dict(stored),)config = SimpleNamespace(providers=providers,@@ -57,14 +59,112 @@ async def test_provider_health_accepts_configured_azure_provider(endpoint=None,project_id=None,test_completion=True,- credentials={- "api_key": "azure-secret",- "api_base": "https://example.openai.azure.com",- "api_version": "2024-10-21",- },+ credentials=stored,+ stored_credentials=stored,)+async def test_provider_health_forwards_the_stored_form_for_multi_endpoint_providers(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """OpenShift AI keeps two endpoints; `credential_values` narrows to one of+ them for the LiteLLM probe, so the enhancement's health check is handed the+ stored form separately, or the embedding endpoint is never checked."""+ stored = {+ "api_base": "https://chat.svc:8443/v1",+ "embedding_api_base": "https://embed.svc:8443/v1",+ "api_key": "sha256~token",+ "ssl_verify": "false",+ }+ translated = {"api_key": "sha256~token", "api_base": stored["api_base"], "ssl_verify": False}+ providers = SimpleNamespace(+ get_provider_config=lambda provider: SimpleNamespace(+ api_key=None, endpoint=None, project_id=None+ ),+ credential_values=lambda provider, kind="chat": dict(translated),+ stored_credentials=lambda provider: dict(stored),+ )+ config = SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_provider="openai", llm_model="gpt-4o-mini"),+ knowledge=SimpleNamespace(embedding_provider="openai", embedding_model="ada"),+ )+ validate = AsyncMock()+ monkeypatch.setattr("api.provider_health.get_openrag_config", lambda: config)+ monkeypatch.setattr("api.provider_health.validate_provider_setup", validate)+ monkeypatch.setattr("api.provider_health.is_known_provider", lambda provider: True)++ response = await check_provider_health(provider="rhoai", user=None)++ assert response.status_code == 200+ # Not selected for either role, so there is no model to probe with: one+ # lightweight check, handed both forms.+ validate.assert_awaited_once()+ kwargs = validate.await_args.kwargs+ assert kwargs["llm_model"] is None+ assert kwargs["embedding_model"] is None+ assert kwargs["credentials"] == translated+ assert kwargs["stored_credentials"] == stored+++async def test_provider_health_probes_each_role_of_a_dual_role_provider(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A provider selected for both roles is probed once per role, each with+ that role's credentials. The validator tests one model per call, so a single+ call would probe the embedding model only and report the chat model as+ validated without ever touching its endpoint."""+ by_kind = {+ "chat": {"api_key": "sha256~token", "api_base": "https://chat.svc:8443/v1"},+ "embedding": {"api_key": "sha256~token", "api_base": "https://embed.svc:8443/v1"},+ }+ stored = {+ "api_key": "sha256~token",+ "api_base": by_kind["chat"]["api_base"],+ "embedding_api_base": by_kind["embedding"]["api_base"],+ }+ providers = SimpleNamespace(+ get_provider_config=lambda provider: SimpleNamespace(+ api_key=None, endpoint=None, project_id=None+ ),+ credential_values=lambda provider, kind="chat": dict(by_kind[kind]),+ stored_credentials=lambda provider: dict(stored),+ )+ config = SimpleNamespace(+ providers=providers,+ agent=SimpleNamespace(llm_provider="rhoai", llm_model="granite-3.3-2b-instruct"),+ knowledge=SimpleNamespace(+ embedding_provider="rhoai", embedding_model="granite-embedding-english-r2"+ ),+ )+ validate = AsyncMock()+ monkeypatch.setattr("api.provider_health.get_openrag_config", lambda: config)+ monkeypatch.setattr("api.provider_health.validate_provider_setup", validate)+ monkeypatch.setattr("api.provider_health.is_known_provider", lambda provider: True)++ response = await check_provider_health(provider="rhoai", user=None)++ assert response.status_code == 200+ body = json.loads(response.body)+ assert body["details"] == {+ "llm_model": "granite-3.3-2b-instruct",+ "embedding_model": "granite-embedding-english-r2",+ "endpoint": None,+ }+ assert validate.await_count == 2+ chat, embedding = (call.kwargs for call in validate.await_args_list)+ assert (chat["llm_model"], chat["embedding_model"]) == ("granite-3.3-2b-instruct", None)+ assert chat["credentials"] == by_kind["chat"]+ assert (embedding["llm_model"], embedding["embedding_model"]) == (+ None,+ "granite-embedding-english-r2",+ )+ assert embedding["credentials"] == by_kind["embedding"]+ assert all(call.kwargs["stored_credentials"] == stored for call in validate.await_args_list)++@pytest.mark.asyncioasync def test_provider_page_catalog_contract_includes_azure(monkeypatch: pytest.MonkeyPatch,